You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by jd...@apache.org on 2015/06/23 12:24:49 UTC
[01/51] [abbrv] incubator-lens git commit: LENS-551: Cleanup
testcases of exception stack traces
Repository: incubator-lens
Updated Branches:
refs/heads/current-release-line db73c5e97 -> b0edfb964
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 885a69a..dc1dc1d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -54,8 +54,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.log4j.BasicConfigurator;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
@@ -67,9 +65,11 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
@Test(groups = "unit-test")
public class TestMetastoreService extends LensJerseyTest {
- public static final Logger LOG = LogManager.getLogger(TestMetastoreService.class);
private ObjectFactory cubeObjectFactory;
protected String mediaType = MediaType.APPLICATION_XML;
protected MediaType medType = MediaType.APPLICATION_XML_TYPE;
@@ -152,7 +152,7 @@ public class TestMetastoreService extends LensJerseyTest {
result = dbTarget.queryParam("sessionid", lensSessionId).queryParam("ignoreIfExisting", false)
.request(mediaType).post(Entity.xml(newDb), APIResult.class);
assertEquals(result.getStatus(), APIResult.Status.FAILED);
- LOG.info(">> Result message " + result.getMessage());
+ log.info(">> Result message " + result.getMessage());
// Drop
dbTarget.path(newDb).queryParam("sessionid", lensSessionId).request().delete();
@@ -768,7 +768,7 @@ public class TestMetastoreService extends LensJerseyTest {
.get(new GenericType<JAXBElement<XCube>>() {});
fail("Should have thrown 404, got:" + got);
} catch (NotFoundException ex) {
- ex.printStackTrace();
+ log.error("Resource not found.", ex);
}
target = target().path("metastore").path("cubes").path("test_drop_cube");
@@ -782,7 +782,7 @@ public class TestMetastoreService extends LensJerseyTest {
.get(new GenericType<JAXBElement<XCube>>() {});
fail("Should have thrown 404, got :" + got);
} catch (NotFoundException ex) {
- ex.printStackTrace();
+ log.error("Resource not found.", ex);
}
} finally {
dropDatabase(DB);
@@ -1222,7 +1222,7 @@ public class TestMetastoreService extends LensJerseyTest {
mediaType).get(new GenericType<JAXBElement<XDimension>>() {});
fail("Should have thrown 404, but got" + got.getValue().getName());
} catch (NotFoundException ex) {
- ex.printStackTrace();
+ log.error("Resource not found.", ex);
}
try {
@@ -1230,7 +1230,7 @@ public class TestMetastoreService extends LensJerseyTest {
.queryParam("sessionid", lensSessionId).request(mediaType).delete(APIResult.class);
fail("Should have thrown 404, but got" + result.getStatus());
} catch (NotFoundException ex) {
- ex.printStackTrace();
+ log.error("Resource not found.", ex);
}
} finally {
dropDatabase(DB);
@@ -1264,7 +1264,7 @@ public class TestMetastoreService extends LensJerseyTest {
.queryParam("sessionid", lensSessionId).request(mediaType).delete(APIResult.class);
fail("Should have got 404");
} catch (NotFoundException e404) {
- LOG.info("correct");
+ log.info("correct");
}
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index dc243aa..62c24ba 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -63,8 +63,6 @@ import org.apache.lens.server.error.LensExceptionMapper;
import org.apache.lens.server.session.HiveSessionService;
import org.apache.lens.server.session.LensSessionImpl;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -85,15 +83,15 @@ import org.testng.annotations.Test;
import com.codahale.metrics.MetricRegistry;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestQueryService.
*/
+@Slf4j
@Test(groups = "unit-test")
public class TestQueryService extends LensJerseyTest {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(TestQueryService.class);
-
/** The query service. */
QueryExecutionServiceImpl queryService;
@@ -736,7 +734,7 @@ public class TestQueryService extends LensJerseyTest {
}
// Test http download end point
- LOG.info("Starting httpendpoint test");
+ log.info("Starting httpendpoint test");
final FormDataMultiPart mp3 = new FormDataMultiPart();
mp3.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
MediaType.APPLICATION_XML_TYPE));
@@ -894,7 +892,7 @@ public class TestQueryService extends LensJerseyTest {
*/
static void validateHttpEndPoint(WebTarget parent, LensSessionHandle lensSessionId, QueryHandle handle,
String redirectUrl) throws IOException {
- LOG.info("@@@ validateHttpEndPoint sessionid " + lensSessionId);
+ log.info("@@@ validateHttpEndPoint sessionid " + lensSessionId);
Response response = parent.path("queryapi/queries/" + handle.toString() + "/httpresultset")
.queryParam("sessionid", lensSessionId).request().get();
@@ -933,7 +931,7 @@ public class TestQueryService extends LensJerseyTest {
Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
} catch (NotFoundException e) {
// expected
- e.printStackTrace();
+ log.error("Resource not found.", e);
}
}
@@ -1348,7 +1346,7 @@ public class TestQueryService extends LensJerseyTest {
File testJarFile = new File("testdata/test2.jar");
sessionService.addResourceToAllServices(sessionHandle, "jar", "file://" + testJarFile.getAbsolutePath());
- LOG.info("@@@ Opened session " + sessionHandle.getPublicId() + " with database " + LensTestUtil.DB_WITH_JARS);
+ log.info("@@@ Opened session " + sessionHandle.getPublicId() + " with database " + LensTestUtil.DB_WITH_JARS);
LensSessionImpl session = sessionService.getSession(sessionHandle);
// Jars should be pending until query is run
@@ -1373,7 +1371,7 @@ public class TestQueryService extends LensJerseyTest {
Assert.assertTrue(addedToHiveDriver);
// Switch database
- LOG.info("@@@# database switch test");
+ log.info("@@@# database switch test");
session.setCurrentDatabase(LensTestUtil.DB_WITH_JARS_2);
LensTestUtil.createTable(tableInDBWithJars + "_2", target(), sessionHandle, "(ID INT, IDSTR STRING) "
+ "ROW FORMAT SERDE \"DatabaseJarSerde\"");
@@ -1393,12 +1391,12 @@ public class TestQueryService extends LensJerseyTest {
Assert.assertTrue(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS_2).isEmpty());
} finally {
- LOG.info("@@@ TEST_OVER");
+ log.info("@@@ TEST_OVER");
try {
LensTestUtil.dropTable(tableInDBWithJars, target(), sessionHandle);
LensTestUtil.dropTable(tableInDBWithJars + "_2", target(), sessionHandle);
} catch (Throwable th) {
- th.printStackTrace();
+ log.error("Exception while dropping table.", th);
}
sessionService.closeSession(sessionHandle);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java
index 640cc41..28bed27 100644
--- a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java
@@ -27,8 +27,6 @@ import java.util.List;
import org.apache.lens.server.LensTestUtil;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -37,9 +35,11 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class TestDatabaseResourceService {
private static final String DB_PFX = TestDatabaseResourceService.class.getSimpleName();
- public static final Log LOG = LogFactory.getLog(TestDatabaseResourceService.class);
public static final String TEST_CLASS = "ClassLoaderTestClass";
@@ -85,7 +85,7 @@ public class TestDatabaseResourceService {
for (URL url : db1Loader.getURLs()) {
String jarFile = url.getPath();
if (jarFile.endsWith(db + ".jar")) {
- LOG.info("Found jar url " + url.toString());
+ log.info("Found jar url " + url.toString());
return true;
}
}
@@ -132,7 +132,7 @@ public class TestDatabaseResourceService {
Class clz = Class.forName("ClassLoaderTestClass", true, getClass().getClassLoader());
Assert.fail("Expected class loading to fail");
} catch (Throwable th) {
- LOG.error("Expected error " + th + " msg = "+th.getMessage());
+ log.error("Expected error " + th + " msg = "+th.getMessage(), th);
}
// Should pass now
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
index bf0526f..d5e60c4 100644
--- a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
+++ b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
@@ -37,9 +37,12 @@ import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestStatisticsLogFileScannerTask.
*/
+@Slf4j
@Test(groups = "unit-test")
public class TestStatisticsLogFileScannerTask {
@@ -100,7 +103,7 @@ public class TestStatisticsLogFileScannerTask {
}
}).when(service).notifyEvent(Mockito.any(PartitionEvent.class));
} catch (Exception e) {
- e.printStackTrace();
+ log.error("Error while running test.", e);
}
task.setService(service);
task.run();
[42/51] [abbrv] incubator-lens git commit: LENS-618: Add consistend
shebang in shell scripts (Rajat Khandelwal via Raghavendra Singh)
Posted by jd...@apache.org.
LENS-618: Add consistend shebang in shell scripts (Rajat Khandelwal via Raghavendra Singh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/5d20751d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/5d20751d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/5d20751d
Branch: refs/heads/current-release-line
Commit: 5d20751d3620313dc27df53daa4662d8fac7fd20
Parents: 86969ba
Author: Rajat Khandelwal <ra...@gmail.com>
Authored: Fri Jun 19 19:04:39 2015 +0530
Committer: raghavendra.singh <ra...@apache.org>
Committed: Fri Jun 19 19:04:39 2015 +0530
----------------------------------------------------------------------
.../src/deb/control/server-control/postinst | 2 +-
lens-docker/lens-test/lens-bootstrap.sh | 8 +++---
lens-ml-lib/tools/scripts/lens-ml-classpath.sh | 2 +-
tools/conf-pseudo-distr/lens-env.sh | 3 ++-
tools/conf/lens-env.sh | 3 ++-
tools/scripts/build-docker.sh | 2 +-
tools/scripts/generate-site-public.sh | 2 +-
tools/scripts/lens-cli | 2 +-
tools/scripts/lens-cli.sh | 2 +-
tools/scripts/lens-config.sh | 2 +-
tools/scripts/lens-ctl | 26 ++++++++++----------
tools/scripts/lens-run-class.sh | 12 ++++-----
tools/scripts/run-docker.sh | 1 +
tools/scripts/run-examples.sh | 2 +-
14 files changed, 36 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/lens-dist/src/deb/control/server-control/postinst
----------------------------------------------------------------------
diff --git a/lens-dist/src/deb/control/server-control/postinst b/lens-dist/src/deb/control/server-control/postinst
index 5d1f976..44a4260 100644
--- a/lens-dist/src/deb/control/server-control/postinst
+++ b/lens-dist/src/deb/control/server-control/postinst
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/lens-docker/lens-test/lens-bootstrap.sh
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index 3407dca..d4b4788 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
@@ -23,7 +23,7 @@ echo "LENS_CLIENT " $LENS_CLIENT
echo "LENS_CLIENT_CONF " $LENS_CLIENT_CONF
echo "LENS_ML " $LENS_ML
echo "SPARK_HOME " $SPARK_HOME
-
+
#set ml classpath into LENS_EXT_CLASSPATH
LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh`
export LENS_EXT_CLASSPATH
@@ -45,7 +45,7 @@ sleep 20
#start lens server
echo "Starting Lens server..."
-$LENS_HOME/bin/lens-ctl start --conf $LENS_SERVER_CONF
+$LENS_HOME/bin/lens-ctl start --conf $LENS_SERVER_CONF
echo "Waiting for 60 secs for Lens Server to start ..."
sleep 60
@@ -55,4 +55,4 @@ $LENS_CLIENT/bin/run-examples.sh sample-metastore --conf $LENS_CLIENT_CONF
$LENS_CLIENT/bin/run-examples.sh populate-metastore --conf $LENS_CLIENT_CONF
$LENS_CLIENT/bin/lens-cli.sh --conf $LENS_CLIENT_CONF
-/bin/bash
+/usr/bin/env bash
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/lens-ml-lib/tools/scripts/lens-ml-classpath.sh
----------------------------------------------------------------------
diff --git a/lens-ml-lib/tools/scripts/lens-ml-classpath.sh b/lens-ml-lib/tools/scripts/lens-ml-classpath.sh
index 7254b78..88d62b4 100644
--- a/lens-ml-lib/tools/scripts/lens-ml-classpath.sh
+++ b/lens-ml-lib/tools/scripts/lens-ml-classpath.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/conf-pseudo-distr/lens-env.sh
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/lens-env.sh b/tools/conf-pseudo-distr/lens-env.sh
index 6a6222c..456e001 100644
--- a/tools/conf-pseudo-distr/lens-env.sh
+++ b/tools/conf-pseudo-distr/lens-env.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,7 +18,7 @@
# under the License.
#
-# The hadoop installation location.
+# The hadoop installation location.
#export HADOOP_HOME=
# The Hive installation location. Lens adds hive lib in the classpath.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/conf/lens-env.sh
----------------------------------------------------------------------
diff --git a/tools/conf/lens-env.sh b/tools/conf/lens-env.sh
index 6a6222c..456e001 100644
--- a/tools/conf/lens-env.sh
+++ b/tools/conf/lens-env.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,7 +18,7 @@
# under the License.
#
-# The hadoop installation location.
+# The hadoop installation location.
#export HADOOP_HOME=
# The Hive installation location. Lens adds hive lib in the classpath.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/build-docker.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/build-docker.sh b/tools/scripts/build-docker.sh
index 31fbac2..8cfda11 100755
--- a/tools/scripts/build-docker.sh
+++ b/tools/scripts/build-docker.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/generate-site-public.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/generate-site-public.sh b/tools/scripts/generate-site-public.sh
index 75f86b0..0f2e45c 100755
--- a/tools/scripts/generate-site-public.sh
+++ b/tools/scripts/generate-site-public.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/lens-cli
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-cli b/tools/scripts/lens-cli
index 18695ef..0f3c946 100644
--- a/tools/scripts/lens-cli
+++ b/tools/scripts/lens-cli
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
#
# Licensed to the Apache Software Foundation (ASF) under one
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/lens-cli.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-cli.sh b/tools/scripts/lens-cli.sh
index e6eb34c..8b490e8 100644
--- a/tools/scripts/lens-cli.sh
+++ b/tools/scripts/lens-cli.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/lens-config.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-config.sh b/tools/scripts/lens-config.sh
index b901fbb..64b1f00 100644
--- a/tools/scripts/lens-config.sh
+++ b/tools/scripts/lens-config.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/lens-ctl
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-ctl b/tools/scripts/lens-ctl
index 1335469..822ead8 100755
--- a/tools/scripts/lens-ctl
+++ b/tools/scripts/lens-ctl
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -64,14 +64,14 @@ display_help() {
Usage: $0 start // Start the server
$0 stop // Stop the server
$0 restart // Restart the server, stop followed by stop
- $0 force [stop|restart] // Forcefully stop the server
+ $0 force [stop|restart] // Forcefully stop the server
$0 status // Server status
$0 stack // Dump the thread stacks
$0 memory-histogram // Server's memory histogram
$0 memory-dump // Server's memory dump dumped to file in log directory
-
+
Optional parameters for the commands --conf/-c, --classpath/-C, -D, --verbose/-v
-
+
EOF
}
@@ -122,18 +122,18 @@ start() {
LENSCPPATH=${LENSCPPATH}:$HADOOP_JARPATH
fi
- if [ "$HIVE_HOME" != "" ]; then
+ if [ "$HIVE_HOME" != "" ]; then
echo "HIVE_HOME is set, adding ${HIVE_HOME}/lib/* into lens classpath"
LENSCPPATH=${LENSCPPATH}:`ls ${HIVE_HOME}/lib/* 2>/dev/null | tr "\n" ':' 2>/dev/null`
else
echo "HIVE_HOME is not set. Set HIVE_HOME and try again"
exit 1
fi
-
+
# Add HIVE_HOME to HADOOP_CLASS_PATH
HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HIVE_HOME}/lib/*"
export HADOOP_CLASSPATH
-
+
mkdir -p $LENS_LOG_DIR
JAVA_PROPERTIES="${JAVA_PROPERTIES} $LENS_OPTS $LENS_PROPERTIES -Dlens.log.dir=$LENS_LOG_DIR -Dlens.home=${LENS_HOME_DIR} -Dconfig.location=$LENS_CONF"
@@ -147,7 +147,7 @@ start() {
if [ -n "$opt_verbose" ] ; then
JAVA_PROPERTIES="-Dlensserver.root.logger=INFO,CONSOLE -Dlensserver.request.logger=INFO,CONSOLE ${JAVA_PROPERTIES}"
${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${LENSCPPATH} org.apache.lens.server.LensServer $*
- else
+ else
pushd ${BASEDIR} > /dev/null
TIME=`date +%Y%m%d%H%M%s`
@@ -175,13 +175,13 @@ stop() {
kill -15 `cat $LENS_PID_FILE`
fi
while :
- do
+ do
if kill -0 `cat $LENS_PID_FILE` > /dev/null 2>&1; then
echo -n "."
sleep 1
else
break
- fi
+ fi
done
rm -rf $LENS_PID_FILE
echo Stopped lens server!
@@ -199,7 +199,7 @@ status() {
exit 0
else
exit -2
- fi
+ fi
else
echo "pid file $LENS_PID_FILE not present"
exit -1
@@ -230,7 +230,7 @@ memory_histo() {
exit 0
else
exit -2
- fi
+ fi
else
echo "pid file $LENS_PID_FILE not present"
exit -1
@@ -245,7 +245,7 @@ memory_dump() {
exit 0
else
exit -2
- fi
+ fi
else
echo "pid file $LENS_PID_FILE not present"
exit -1
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/lens-run-class.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-run-class.sh b/tools/scripts/lens-run-class.sh
index ef64b03..698c51e 100644
--- a/tools/scripts/lens-run-class.sh
+++ b/tools/scripts/lens-run-class.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -101,18 +101,18 @@ setenv() {
LENSCPPATH=${LENSCPPATH}:$HADOOP_JARPATH
fi
- if [ "$HIVE_HOME" != "" ]; then
+ if [ "$HIVE_HOME" != "" ]; then
echo "HIVE_HOME is set, adding ${HIVE_HOME}/lib/* into lens classpath"
LENSCPPATH=${LENSCPPATH}:`ls ${HIVE_HOME}/lib/* 2>/dev/null | tr "\n" ':' 2>/dev/null`
else
echo "HIVE_HOME is not set. Set HIVE_HOME and try again"
exit 1
fi
-
+
# Add HIVE_HOME to HADOOP_CLASS_PATH
HADOOP_CLASSPATH="$HADOOP_CLASSPATH:${HIVE_HOME}/lib/*"
export HADOOP_CLASSPATH
-
+
}
################################
# main
@@ -138,7 +138,7 @@ while [ -n "$*" ] ; do
JAVA_PROPERTIES="${JAVA_PROPERTIES} $arg"
;;
*)
- if [ "$opt_classname" == "" ]; then
+ if [ "$opt_classname" == "" ]; then
opt_classname=$arg
echo "opt_classname is " $opt_classname
else
@@ -156,7 +156,7 @@ if [ -n "$opt_conf" ]; then
fi
# finally, invoke the appropriate command
-if [ "$opt_classname" == "" ]; then
+if [ "$opt_classname" == "" ]; then
echo "Usage : $0 <classname>"
exit 1
fi
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/run-docker.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/run-docker.sh b/tools/scripts/run-docker.sh
index 3372d43..8c701a3 100755
--- a/tools/scripts/run-docker.sh
+++ b/tools/scripts/run-docker.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5d20751d/tools/scripts/run-examples.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/run-examples.sh b/tools/scripts/run-examples.sh
index 0e8af6a..f05c1c0 100644
--- a/tools/scripts/run-examples.sh
+++ b/tools/scripts/run-examples.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
[39/51] [abbrv] incubator-lens git commit: LENS-536: Multiple issues
in generate site script
Posted by jd...@apache.org.
LENS-536: Multiple issues in generate site script
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/17080940
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/17080940
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/17080940
Branch: refs/heads/current-release-line
Commit: 17080940e0edc93abcabc7d1d75a6d2d034becdd
Parents: 51cdd98
Author: Raghavendra Singh <ra...@inmobi.com>
Authored: Fri Jun 19 12:05:45 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Fri Jun 19 12:05:45 2015 +0530
----------------------------------------------------------------------
tools/scripts/generate-site-public.sh | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/17080940/tools/scripts/generate-site-public.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/generate-site-public.sh b/tools/scripts/generate-site-public.sh
index 2afd653..75f86b0 100755
--- a/tools/scripts/generate-site-public.sh
+++ b/tools/scripts/generate-site-public.sh
@@ -34,16 +34,16 @@ fi
SVN_TARGET=$1
TMP=/tmp/lens-site-stage
+SITE_BACKUP=/tmp/lens-site-backup
STAGE=`pwd`/target/staging
REST_DIR=`pwd`/lens-server/target/site/wsdocs
VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version|grep -Ev '(^\[|Download\w+:)' || die "unable to get version")
-
echo "Starting generate-site"
CURR_BRANCH=`git branch | sed -n '/\* /s///p'`
echo "Running site in current lens branch" $CURR_BRANCH
-mvn clean test -Dtest=org.apache.lens.doc.TestGenerateConfigDoc,org.apache.lens.cli.doc.TestGenerateCLIUserDoc || die "Unable to generate config docs"
-mvn install -DskipTests
+mvn clean test -Dtest=org.apache.lens.doc.TestGenerateConfigDoc,org.apache.lens.cli.doc.TestGenerateCLIUserDoc -DskipCheck || die "Unable to generate config docs"
+mvn install -DskipTests -DskipCheck
mvn site site:stage -Ddependency.locations.enabled=false -Ddependency.details.enabled=false || die "unable to generate site"
echo "Site gen complete"
@@ -54,7 +54,7 @@ mkdir -p $TMP
cd $TMP
mkdir -p current || die "unable to create dir current"
-mkdir -p versions/$VERSION || due "unable to create dir versions/$VERSION"
+mkdir -p versions/$VERSION || die "unable to create dir versions/$VERSION"
find current -type f -exec git rm {} \;
echo "Copying REST docs from " $REST_DIR
@@ -85,6 +85,11 @@ echo '</ul>' >> versions/index.html
## Copy entire doc directory to Apache SVN Target dir
mkdir -p $SVN_TARGET/site/publish
+mkdir -p $SITE_BACKUP/site/publish
+cp -r $SVN_TARGET/site/publish/ $SITE_BACKUP/site/publish
+rm -r $SVN_TARGET/site/publish/*
+rm -r $SITE_BACKUP/site/publish/versions/$VERSION
+cp -r $SITE_BACKUP/site/publish/versions $SVN_TARGET/site/publish/
cp -r $TMP/ $SVN_TARGET/site/publish
cd $SVN_TARGET
echo "Generated site."
[47/51] [abbrv] incubator-lens git commit: LENS-493 : Fix javadoc
errors on java8 build
Posted by jd...@apache.org.
LENS-493 : Fix javadoc errors on java8 build
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/0cf6c4e2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/0cf6c4e2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/0cf6c4e2
Branch: refs/heads/current-release-line
Commit: 0cf6c4e2d322fee64505c1e4f127e868c7410ba6
Parents: 3eddcb5
Author: Raju Bairishetti <ra...@gmail.com>
Authored: Mon Jun 22 11:00:45 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Jun 22 11:00:45 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 8 +-
.../org/apache/lens/client/jdbc/JDBCUtils.java | 6 +-
.../lens/cube/metadata/AbstractCubeTable.java | 2 +-
.../lens/cube/metadata/CubeInterface.java | 6 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 12 +--
.../apache/lens/cube/metadata/ExprColumn.java | 1 -
.../apache/lens/cube/metadata/JoinChain.java | 2 +-
.../lens/cube/metadata/TimePartitionRange.java | 2 +-
.../lens/cube/parse/CubeQueryContext.java | 4 +-
.../cube/parse/DenormalizationResolver.java | 2 +-
.../apache/lens/cube/parse/DimHQLContext.java | 2 +-
.../lens/cube/parse/SimpleHQLContext.java | 4 +-
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 4 -
.../org/apache/lens/ml/impl/LensMLImpl.java | 2 +-
.../lens/ml/server/MLServiceResource.java | 5 +-
.../java/org/apache/lens/rdd/LensRDDClient.java | 14 +--
.../lens/lib/query/AbstractOutputFormatter.java | 2 +-
.../lens/lib/query/FilePersistentFormatter.java | 2 +-
.../lens/lib/query/FileSerdeFormatter.java | 4 +-
.../lens/lib/query/HadoopFileFormatter.java | 2 +-
.../org/apache/lens/lib/query/JSonSerde.java | 23 ++---
.../apache/lens/lib/query/ZipFileFormatter.java | 2 +-
.../apache/lens/regression/util/AssertUtil.java | 27 +++--
.../lens/server/api/driver/DriverQueryPlan.java | 2 +-
.../lens/server/api/driver/LensDriver.java | 2 +-
.../api/driver/QueryCompletionListener.java | 2 +-
.../lens/server/api/events/LensEvent.java | 1 -
.../api/metastore/CubeMetastoreService.java | 3 +-
.../lens/server/api/metrics/MetricsService.java | 29 +++---
.../lens/server/api/priority/RangeConf.java | 16 +--
.../server/api/query/AbstractQueryContext.java | 1 -
.../lens/server/api/query/QueryContext.java | 1 -
.../lens/server/api/query/QueryEvent.java | 2 +-
.../server/api/query/QueryOutputFormatter.java | 4 +-
.../lens/server/api/session/SessionEvent.java | 1 -
.../lens/server/api/session/SessionService.java | 2 -
.../lens/server/LensApplicationListener.java | 5 +-
.../lens/server/LensServletContextListener.java | 2 -
.../apache/lens/server/metastore/JAXBUtils.java | 2 +-
.../server/metastore/MetastoreResource.java | 20 ++--
.../lens/server/metrics/MetricsServiceImpl.java | 8 +-
.../server/query/QueryExecutionServiceImpl.java | 1 -
.../lens/server/query/QueryServiceResource.java | 101 +++++++++++--------
.../lens/server/session/HiveSessionService.java | 15 +--
.../lens/server/session/SessionResource.java | 6 +-
.../lens/server/ui/MetastoreUIResource.java | 7 +-
.../lens/server/ui/QueryServiceUIResource.java | 6 +-
.../lens/server/ui/SessionUIResource.java | 2 +-
pom.xml | 10 +-
49 files changed, 187 insertions(+), 202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index cd898cc..ab7d3ff 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
@@ -530,13 +531,14 @@
Join chain represents the paths to reach a dimension.
For example, to reach Country from
SalesCube, there can be the following equivalent paths:
- SalesCube.deliveryStateid -> State.id, State.countryid ->Country.id
- SalesCube.deliveryCityid -> City.id, City.stateid -> State.id, State.countryid ->Country.id
+ SalesCube.deliveryStateid -&gt; State.id, State.countryid -&gt; Country.id
+ SalesCube.deliveryCityid -&gt; City.id, City.stateid -&gt; State.id, State.countryid -&gt;Country.id
Both the above paths will be associated with same
join chain i.e with same name, if they are equivalent and at
runtime, LENS will choose the path to use. If the
paths are not equivalent, they should be named differently.
- SalesCube.productionStateid -> State.id, State.countryid ->Country.id will be named differently.
+ SalesCube.productionStateid -&gt; State.id,
+ State.countryid -&gt;Country.id will be named differently.
</xs:documentation>
</xs:annotation>
<xs:complexContent>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-client/src/main/java/org/apache/lens/client/jdbc/JDBCUtils.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/jdbc/JDBCUtils.java b/lens-client/src/main/java/org/apache/lens/client/jdbc/JDBCUtils.java
index 7413a22..34448d3 100644
--- a/lens-client/src/main/java/org/apache/lens/client/jdbc/JDBCUtils.java
+++ b/lens-client/src/main/java/org/apache/lens/client/jdbc/JDBCUtils.java
@@ -63,7 +63,7 @@ public final class JDBCUtils {
/**
* Parses the JDBC Connection URL.
- * <p/>
+ * <p></p>
* The URL format is specified as following :
* <ul>
* <li>hostname which contains hosts on which lens server is hosted parsed from authority section</li>
@@ -76,10 +76,10 @@ public final class JDBCUtils {
* <p>
* Examples :-
* </p>
- * <code>
+ * <code>
* jdbc:lens://hostname:port/dbname;[optional key value pair of session settings]?
* [optional configuration settings for connection]#[optional variables to be used in query]
- * </code>
+ * </code>
*
* @param uri to be used to connect to lens server
* @return final list of connection parameters
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
index 27f11bf..9384021 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
@@ -91,7 +91,7 @@ public abstract class AbstractCubeTable implements Named {
/**
* Add more table properties
*
- * @param properties
+ * @param props properties
*/
public void addProperties(Map<String, String> props) {
this.properties.putAll(props);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
index cd8c928..6477a6b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeInterface.java
@@ -87,7 +87,7 @@ public interface CubeInterface extends Named {
/**
* Get cube column given by column name.
- * <p/>
+ * <p></p>
* It can be a measure, dimension attribute or an expression.
*
* @param colName Column name
@@ -146,10 +146,10 @@ public interface CubeInterface extends Named {
/**
* Whether all the fields of cube can be queried.
- * <p/>
+ * <p></p>
* If false, the fields can queried through derived cubes. Users can look at derived cube fields to know which all
* fields can be queried together.
- * <p/>
+ * <p></p>
* If true, all the fields can be directly queried.
*
* @return true or false
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 326a49b..dc2d9f6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -97,7 +97,7 @@ public class CubeMetastoreClient {
/**
* get latest date for timeDimension from all fact-storage tables belonging to the given cube having timeDimension,
* return the most recent of all.
- * <p/>
+ * <p></p>
* latest date for a single fact-storage table for given time dimension is the latest of the latest dates for all its
* update periods
*
@@ -318,7 +318,7 @@ public class CubeMetastoreClient {
/**
* helper method for ensuring get(storageTable).get(updatePeriod).get(partitionColumn) gives a non-null object.
- * <p/>
+ * <p></p>
* kind of like mkdir -p
*
* @param storageTable
@@ -715,7 +715,7 @@ public class CubeMetastoreClient {
/**
* Add a partition specified by the storage partition desc on the storage passed.
- * <p/>
+ * <p></p>
* TODO: separate methods for fact and dim partition addition.
*
* @param partSpec The storage partition description
@@ -1846,7 +1846,6 @@ public class CubeMetastoreClient {
* @param cubeName The cube name to be altered
* @param cube The new cube definition {@link Cube} or {@link DerivedCube}
* @throws HiveException
- * @throws InvalidOperationException
*/
public void alterCube(String cubeName, CubeInterface cube) throws HiveException {
Table cubeTbl = getTable(cubeName);
@@ -1866,7 +1865,6 @@ public class CubeMetastoreClient {
* @param dimName The cube name to be altered
* @param newDim The new dimension definition
* @throws HiveException
- * @throws InvalidOperationException
*/
public void alterDimension(String dimName, Dimension newDim) throws HiveException {
Table tbl = getTable(dimName);
@@ -1886,7 +1884,6 @@ public class CubeMetastoreClient {
* @param storageName The storage name to be altered
* @param storage The new storage definition
* @throws HiveException
- * @throws InvalidOperationException
*/
public void alterStorage(String storageName, Storage storage) throws HiveException {
Table storageTbl = getTable(storageName);
@@ -1975,7 +1972,6 @@ public class CubeMetastoreClient {
* @param factName
* @param storage
* @throws HiveException
- * @throws InvalidOperationException
*/
public void dropStorageFromFact(String factName, String storage) throws HiveException {
CubeFactTable cft = getFactTable(factName);
@@ -2050,7 +2046,6 @@ public class CubeMetastoreClient {
* @param storageTableDescs
*
* @throws HiveException
- * @throws InvalidOperationException
*/
public void alterCubeFactTable(String factTableName, CubeFactTable cubeFactTable,
Map<String, StorageTableDesc> storageTableDescs) throws HiveException {
@@ -2087,7 +2082,6 @@ public class CubeMetastoreClient {
* @param dimTableName
* @param cubeDimensionTable
* @throws HiveException
- * @throws InvalidOperationException
*/
public void alterCubeDimensionTable(String dimTableName, CubeDimensionTable cubeDimensionTable,
Map<String, StorageTableDesc> storageTableDescs) throws HiveException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
index 3fc661f..b418517 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/ExprColumn.java
@@ -315,7 +315,6 @@ public class ExprColumn extends CubeColumn {
* Get the AST corresponding to the expression
*
* @return the ast
- * @throws ParseException
*/
public ASTNode getAst() {
return getExpressionASTList().get(0);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
index a3d15a7..c7a2d3d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JoinChain.java
@@ -314,7 +314,7 @@ public class JoinChain implements Named {
* Convert join paths to schemaGraph's JoinPath
*
* @param client
- * @return List<SchemaGraph.JoinPath>
+ * @return List<SchemaGraph.JoinPath>
* @throws HiveException
*/
public List<SchemaGraph.JoinPath> getRelationEdges(CubeMetastoreClient client) throws HiveException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
index e16294e..f5f8d4c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartitionRange.java
@@ -80,7 +80,7 @@ public class TimePartitionRange implements Iterable<TimePartition>, Named {
/**
* @param partition
- * @return begin <= partition < end
+ * @return begin <= partition < end
*/
public boolean contains(TimePartition partition) {
return !partition.before(begin) && partition.before(end);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index ae65287..33b1a62 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1123,7 +1123,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
/**
* Prune candidate fact sets with respect to available candidate facts.
- * <p/>
+ * <p></p>
* Prune a candidate set, if any of the fact is missing.
*
* @param pruneCause
@@ -1142,7 +1142,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
/**
* Prune candidate fact with respect to available candidate fact sets.
- * <p/>
+ * <p></p>
* If candidate fact is not present in any of the candidate fact sets, remove it.
*
* @param pruneCause
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 1a7b6b4..3ed94fa 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -42,7 +42,7 @@ import lombok.ToString;
/**
* This class resolves all the reference columns that are queried.
- * <p/>
+ * <p></p>
* Keeps track of the context that if any candidate needs to use columns through tables referenced and replaces the
* columns from referenced tables in all the ASTs
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
index ea461ad..63aaa12 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Dimension HQLContext.
- * <p/>
+ * <p></p>
* Contains all the dimensions queried and their candidate dim tables Update where string with storage filters added
* dimensions queried.
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
index c251e61..76f976a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Accepts strings of all expressions and constructs HQL query.
- * <p/>
+ * <p></p>
* Making this as an abstract class because it provides constructors without all expressions being set.
*/
public abstract class SimpleHQLContext implements HQLContextInterface {
@@ -68,7 +68,7 @@ public abstract class SimpleHQLContext implements HQLContextInterface {
/**
* Set all missing expressions of HQL context.
- * <p/>
+ * <p></p>
* Leaving this empty implementation for the case of all expressions being passed in constructor. If other
* constructors are used the missing expressions should be set here
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 17ddbfd..2b2c310 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -269,7 +269,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* Gets the join cond.
*
* @param node the node
- * @return the join cond
*/
public void getJoinCond(ASTNode node) {
if (node == null) {
@@ -378,7 +377,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* Gets the filter in join cond.
*
* @param node the node
- * @return the filter in join cond
*/
public void getFilterInJoinCond(ASTNode node) {
@@ -703,7 +701,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
* Gets the all filters.
*
* @param node the node
- * @return the all filters
*/
public void getAllFilters(ASTNode node) {
if (node == null) {
@@ -821,7 +818,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
/**
* Replace alias in AST trees
*
- * @throws HiveException
*/
public void replaceAliasInAST() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
index 50c22f4..5938f8a 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
@@ -392,7 +392,7 @@ public class LensMLImpl implements LensML {
* Evaluate a model. Evaluation is done on data selected table from an input table. The model is run as a UDF and its
* output is inserted into a table with a partition. Each evaluation is given a unique ID. The partition label is
* associated with this unique ID.
- * <p/>
+ * <p></p>
* <p>
* This call also required a query runner. Query runner is responsible for executing the evaluation query against Lens
* server.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
index 6078893..77a6c40 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
@@ -223,14 +223,13 @@ public class MLServiceResource {
* Train a model given an algorithm name and algorithm parameters
* <p>
* Following parameters are mandatory and must be passed as part of the form
- * <p/>
+ * </p>
* <ol>
* <li>table - input Hive table to load training data from</li>
* <li>label - name of the labelled column</li>
* <li>feature - one entry per feature column. At least one feature column is required</li>
* </ol>
- * <p/>
- * </p>
+ * <p></p>
*
* @param algorithm algorithm name
* @param form form data
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
index 0efa10d..fc4164f 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
@@ -54,9 +54,8 @@ import org.apache.spark.rdd.RDD;
* <p>
* Create RDD from a Lens query. User can poll returned query handle with isReadyForRDD() until the RDD is ready to be
* used.
- * <p/>
+ * </p>
* Example -
- * <p/>
* <pre>
* LensRDDClient client = new LensRDDClient(javaSparkContext);
* QueryHandle query = client.createLensRDDAsync("SELECT msr1 from TEST_CUBE WHERE ...", conf);
@@ -65,22 +64,17 @@ import org.apache.spark.rdd.RDD;
* Thread.sleep(1000);
* }
*
- * JavaRDD<ResultRow> rdd = client.getRDD(query).toJavaRDD();
+ * JavaRDD<ResultRow> rdd = client.getRDD(query).toJavaRDD();
*
* // Consume RDD here -
* rdd.map(...);
* </pre>
- * <p/>
- * </p>
- * <p/>
* <p>
* Alternatively in blocking mode
- * <p/>
+ * </p>
* <pre>
* JavaRDD<ResultRow> rdd = client.createLensRDD("SELECT msr1 from TEST_CUBE WHERE ...", conf);
* </pre>
- * <p/>
- * </p>
*/
public class LensRDDClient {
@@ -168,7 +162,7 @@ public class LensRDDClient {
*
* @param query the query
* @return the query handle
- * @throws LensException the lens exception
+ * @throws LensAPIException the lens exception
*/
public QueryHandle createLensRDDAsync(String query) throws LensAPIException {
return getClient().executeQueryAsynch(query, "").getData();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractOutputFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractOutputFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractOutputFormatter.java
index b89c350..06f37ee 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractOutputFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractOutputFormatter.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.util.ReflectionUtils;
/**
* Provides abstract implementation of the query output formatter.
- * <p/>
+ * <p></p>
* In this it initializes column names, types column object inspectors Also provides methods to construct header from
* serde
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-query-lib/src/main/java/org/apache/lens/lib/query/FilePersistentFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/FilePersistentFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/FilePersistentFormatter.java
index 96476bf..c067365 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/FilePersistentFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/FilePersistentFormatter.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.fs.PathFilter;
/**
* File formatter for {@link PersistedOutputFormatter}
- * <p/>
+ * <p></p>
* This is a {@link WrappedFileFormatter} which can wrap any {@link FileFormatter}.
*/
public class FilePersistentFormatter extends WrappedFileFormatter implements PersistedOutputFormatter {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSerdeFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSerdeFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSerdeFormatter.java
index 03f20ba..d433b22 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSerdeFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSerdeFormatter.java
@@ -39,9 +39,9 @@ import org.apache.hadoop.util.ReflectionUtils;
/**
* File format which provides implementation for {@link InMemoryOutputFormatter}
- * <p/>
+ * <p></p>
* This is a wrapped formatter, which serializes the rows of the result with configured serde. It would only accept the
- * Serde's whose serialization class is {@link Text}
+ * Serde's whose serialization class is Text
*/
@SuppressWarnings("deprecation")
public class FileSerdeFormatter extends WrappedFileFormatter implements InMemoryOutputFormatter {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
index f27ac7c..2c48209 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.mapred.Reporter;
/**
* A hadoop file formatter
- * <p/>
+ * <p></p>
* This has capability to create output on Hadoop compatible files systems, with hadoop supported compression codecs.
*/
public class HadoopFileFormatter extends AbstractFileFormatter {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-query-lib/src/main/java/org/apache/lens/lib/query/JSonSerde.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/JSonSerde.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/JSonSerde.java
index 9c7be96..2e4585e 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/JSonSerde.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/JSonSerde.java
@@ -36,25 +36,24 @@ import org.codehaus.jackson.map.ObjectMapper;
* This SerDe can be used for processing JSON data in Hive. It supports arbitrary JSON data, and can handle all Hive
* types except for UNION. However, the JSON data is expected to be a series of discrete records, rather than a JSON
* array of objects.
- * <p/>
+ * <p></p>
* The Hive table is expected to contain columns with names corresponding to fields in the JSON data, but it is not
* necessary for every JSON field to have a corresponding Hive column. Those JSON fields will be ignored during queries.
- * <p/>
+ * <p></p>
* Example:
- * <p/>
+ * <p></p>
* { "a": 1, "b": [ "str1", "str2" ], "c": { "field1": "val1" } }
- * <p/>
+ * <p></p>
* Could correspond to a table:
- * <p/>
- * CREATE TABLE foo (a INT, b ARRAY<STRING>, c STRUCT<field1:STRING>);
- * <p/>
+ * <p></p>
+ * CREATE TABLE foo (a INT, b ARRAY<STRING>, c STRUCT<field1:STRING>);
+ * <p></p>
* JSON objects can also interpreted as a Hive MAP type, so long as the keys and values in the JSON object are all of
* the appropriate types. For example, in the JSON above, another valid table declaraction would be:
- * <p/>
- * CREATE TABLE foo (a INT, b ARRAY<STRING>, c MAP<STRING,STRING>);
- * <p/>
+ * <p></p>
+ * CREATE TABLE foo (a INT, b ARRAY<STRING>, c MAP<STRING,STRING>);
+ * <p> </p>
* Only STRING keys are supported for Hive MAPs.
- * <p/>
*/
public class JSonSerde implements SerDe {
@@ -106,7 +105,7 @@ public class JSonSerde implements SerDe {
* This method does the work of deserializing a record into Java objects that Hive can work with via the
* ObjectInspector interface. For this SerDe, the blob that is passed in is a JSON string, and the Jackson JSON parser
* is being used to translate the string into Java objects.
- * <p/>
+ *<p></p>
* The JSON deserialization works by taking the column names in the Hive table, and looking up those fields in the
* parsed JSON object. If the value of the field is not a primitive, the object is parsed further.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
index 6696968..cc6d918 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path;
/**
* Zip file formatter.
- * <p/>
+ * <p></p>
* Creates a zip on hadoop compatible file system, with ability to split output across multiple part files and provide a
* final zip output file.
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
index f6e88d3..abcea2b 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
@@ -20,7 +20,6 @@
package org.apache.lens.regression.util;
import javax.ws.rs.core.Response;
-import javax.xml.bind.JAXBException;
import org.apache.lens.api.APIResult;
import org.apache.lens.server.api.error.LensException;
@@ -42,9 +41,9 @@ public class AssertUtil {
* Checks that Response status is SUCCEEDED
*
* @param response Response
- * @throws JAXBException,LensException
+ * @throws LensException
*/
- public static void assertSucceeded(Response response) throws JAXBException, LensException {
+ public static void assertSucceeded(Response response) throws LensException {
if (response.getStatus() == 200) {
throw new LensException("Status code should be 200");
}
@@ -61,15 +60,15 @@ public class AssertUtil {
* Checks that Response status is SUCCEEDED
*
* @param response Response
- * @throws JAXBException,LensException
+ * @throws LensException
*/
- public static void assertSucceededResponse(Response response) throws JAXBException, LensException {
+ public static void assertSucceededResponse(Response response) throws LensException {
if (response.getStatus() == 200) {
throw new LensException("Status code should be 200");
}
}
- public static void assertGoneResponse(Response response) throws JAXBException, LensException {
+ public static void assertGoneResponse(Response response) throws LensException {
if (response.getStatus() == 410) {
throw new LensException("Status code should be 410");
}
@@ -79,9 +78,9 @@ public class AssertUtil {
* Checks that Response status is NOT FOUND
*
* @param response Response
- * @throws JAXBException,LensException
+ * @throws LensException
*/
- public static void assertFailedResponse(Response response) throws JAXBException, LensException {
+ public static void assertFailedResponse(Response response) throws LensException {
if (response.getStatus() == 404) {
throw new LensException("Status code should be 404");
}
@@ -91,9 +90,9 @@ public class AssertUtil {
* Checks that Response status is status FAILED with status code 400
*
* @param response Response
- * @throws JAXBException,LensException
+ * @throws LensException
*/
- public static void assertFailed(Response response) throws JAXBException, LensException {
+ public static void assertFailed(Response response) throws LensException {
if (response.getStatus() == 400) {
throw new LensException("Status code should be 400");
}
@@ -110,9 +109,9 @@ public class AssertUtil {
* Checks that Response status is status FAILED with status code 200
*
* @param response Response
- * @throws JAXBException,LensException
+ * @throws LensException
*/
- public static void assertStatusFailed(Response response) throws JAXBException, LensException {
+ public static void assertStatusFailed(Response response) throws LensException {
if (response.getStatus() == 200) {
throw new LensException("Status code should be 200");
}
@@ -129,10 +128,10 @@ public class AssertUtil {
* Checks that Response status is status FAILED with status code 500
*
* @param response Response
- * @throws JAXBException,LensException
+ * @throws LensException
*/
- public static void assertInternalServerError(Response response) throws JAXBException, LensException {
+ public static void assertInternalServerError(Response response) throws LensException {
if (response.getStatus() == 500) {
throw new LensException("Status code should be 500");
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
index 8eba16d..51b1217 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
@@ -151,7 +151,7 @@ public abstract class DriverQueryPlan {
/**
* Set the list of table names to be queried.
*
- * @param table the table
+ * @param tables the table
*/
protected void addTablesQueried(Set<String> tables) {
this.tablesQueried.addAll(tables);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 0c5ee0a..3dc4e73 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -100,7 +100,7 @@ public interface LensDriver extends Externalizable {
/**
* Blocking execute of the query
- * <p/>
+ * <p></p>
* The driver would be closing the driver handle, once the results are fetched.
*
* @param context the context
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/driver/QueryCompletionListener.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/QueryCompletionListener.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/QueryCompletionListener.java
index db2a246..3713b51 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/QueryCompletionListener.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/QueryCompletionListener.java
@@ -23,7 +23,7 @@ import org.apache.lens.api.query.QueryHandle;
/**
* The listener interface for receiving queryCompletion events. The class that is interested in processing a
* queryCompletion event implements this interface, and the object created with that class is registered with a
- * component using the component's <code>addQueryCompletionListener<code> method. When
+ * component using the component's <code>addQueryCompletionListener<code> method. When
* the queryCompletion event occurs, that object's appropriate
* method is invoked.
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEvent.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEvent.java b/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEvent.java
index d74e06b..920aa06 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEvent.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEvent.java
@@ -28,7 +28,6 @@ import lombok.Getter;
/**
* Instantiates a new lens event.
*
- * @param eventTime the event time
*/
@AllArgsConstructor
public abstract class LensEvent {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index e0aa47c..6226521 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -548,7 +548,8 @@ public interface CubeMetastoreService {
* @param timeDimension time dimension name
* @param cubeName The base cube name
* @return Date
- * @throws LensException,HiveException
+ * @throws LensException
+ * @throws HiveException
*/
Date getLatestDateOfCube(LensSessionHandle sessionid, String cubeName, String timeDimension)
throws LensException, HiveException;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
index f7a23ab..71128b7 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
@@ -33,22 +33,21 @@ public interface MetricsService {
/**
* Increment a counter with the given name Actual name of the counter will be
- * <p/>
- * <pre>MetricRegistry.name(MetricsService.class, counter)
- * <p/>
- * <pre>
+ * <p></p>
+ * <pre>MetricRegistry.name(MetricsService.class, counter)</pre>
+ * <p></p>
*
* @param counter the counter
*/
+
void incrCounter(String counter);
/**
* Increment a counter with the name constructed using given class and counter name Actual name of the counter will
* be
- * <p/>
- * <pre>MetricRegistry.name(cls, counter)
- * <p/>
- * <pre>
+ * <p></p>
+ * <pre>MetricRegistry.name(cls, counter)</pre>
+ * <p></p>
*
* @param cls Class of the counter for namespacing the counter
* @param counter the counter
@@ -57,10 +56,9 @@ public interface MetricsService {
/**
* Decrement a counter with the name costructed using given class and counter name Actual name of the counter will be
- * <p/>
- * <pre>MetricRegistry.name(cls, counter)
- * <p/>
- * <pre>
+ * <p></p>
+ * <pre>MetricRegistry.name(cls, counter)</pre>
+ * <p></p>
*
* @param cls Class of the counter for namespacing of counters
* @param counter the counter
@@ -69,10 +67,9 @@ public interface MetricsService {
/**
* Decrement a counter with the given name Actual name of the counter will be
- * <p/>
- * <pre>MetricRegistry.name(MetricsService.class, counter)
- * <p/>
- * <pre>
+ * <p></p>
+ * <pre>MetricRegistry.name(MetricsService.class, counter)</pre>
+ * <p></p>
*
* @param counter the counter
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
index 5cf1f5c..75295ab 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
@@ -24,16 +24,16 @@ import java.util.TreeMap;
/**
* Class for storing range configurations. An Example would be grading system.
* The value F,30,D,40,C,60,B,80,A corresponds to a system where
- * - inf < marks <= 30 : F
- * 30 < marks <= 40 : D
- * 40 < marks <= 60 : C
- * 60 < marks <= 80 : B
- * 80 < marks <= + Inf : A
- * <p/>
+ * - inf < marks <= 30 : F
+ * 30 < marks <= 40 : D
+ * 40 < marks <= 60 : C
+ * 60 < marks <= 80 : B
+ * 80 < marks <= + Inf : A
+ * <p></p>
* rangeConfInstance.get(marks) would give you the grade depending on the range.
- * <p/>
+ * <p></p>
* The utility is for easily storing range configs in config xml files.
- * <p/>
+ * <p></p>
* Implementation is done by storing the least value(floor) and keeping a treemap on rest of values
*
* @param <K> Key type. Integer in the grade range example
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 8d43148..c398215 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -427,7 +427,6 @@ public abstract class AbstractQueryContext implements Serializable {
* Set query for a given driver
* @param driver driver instance
* @param query query string
- * @throws LensException
*/
public void setDriverQuery(LensDriver driver, String query) {
driverContext.setDriverQuery(driver, query);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 2d3a19a..d72727e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -329,7 +329,6 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
/**
* Set whether result is persisted on driver to false. Set by drivers when drivers are not persisting
*
- * @return true/false
*/
public void unSetDriverPersistent() {
isDriverPersistent = false;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java
index 7b3baaa..81c53be 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java
@@ -28,7 +28,7 @@ import lombok.Getter;
/**
* A generic event related to state change of a query Subclasses must declare the specific type of change they are
* interested in.
- * <p/>
+ * <p></p>
* Every event will have an ID, which should be used by listeners to check if the event is already received.
*
* @param <T> Type of changed information about the query
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
index 2e96856..1d2e89b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
@@ -24,7 +24,7 @@ import org.apache.lens.server.api.driver.LensResultSetMetadata;
/**
* The interface for query result formatting
- * <p/>
+ * <p></p>
* This is an abstract interface, user should implement {@link InMemoryOutputFormatter} or
* {@link PersistedOutputFormatter} for formatting the result.
*/
@@ -55,7 +55,7 @@ public interface QueryOutputFormatter {
/**
* Commit the formatting.
- * <p/>
+ * <p></p>
* This will make the result consumable by user, will be called after all the writes succeed.
*
* @throws IOException Signals that an I/O exception has occurred.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java
index 87fa855..ce0a68f 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionEvent.java
@@ -46,7 +46,6 @@ public abstract class SessionEvent extends LensEvent {
*
* @param eventTime the event time
* @param handle the lens session handle
- * @param user name of user
*/
public SessionEvent(long eventTime, LensSessionHandle handle) {
super(eventTime);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
index a28732f..81160f5 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/session/SessionService.java
@@ -71,7 +71,6 @@ public interface SessionService {
* @param sessionHandle the session handle
* @param type the type
* @param path the path
- * @throws LensException the lens exception
*/
void addResource(LensSessionHandle sessionHandle, String type, String path);
@@ -82,7 +81,6 @@ public interface SessionService {
* @param sessionHandle the session handle
* @param type the type
* @param path the path
- * @throws LensException the lens exception
*/
void deleteResource(LensSessionHandle sessionHandle, String type, String path);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java b/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
index cf735bf..2c8ce46 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
@@ -30,10 +30,9 @@ import org.glassfish.jersey.server.monitoring.RequestEventListener;
/**
* The listener interface for receiving lensApplication events. The class that is interested in processing a
* lensApplication event implements this interface, and the object created with that class is registered with a
- * component using the component's <code>addLensApplicationListener<code> method. When the lensApplication event occurs,
- * that object's appropriate method is invoked.
+ * component using the component's <code>addLensApplicationListener<code> method.
+ * When the lensApplication event occurs, that object's appropriate method is invoked.
*
- * @see LensApplicationEvent
*/
public class LensApplicationListener implements ApplicationEventListener {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java b/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
index d360978..0945a35 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
@@ -30,8 +30,6 @@ import lombok.extern.slf4j.Slf4j;
/**
* Initialize the webapp.
- *
- * @see LensServletContextEvent
*/
@Slf4j
public class LensServletContextListener implements ServletContextListener {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index b6d5fec..811fd32 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -449,7 +449,7 @@ public final class JAXBUtils {
}
/**
- * Convert JAXB properties to Map<String, String>
+ * Convert JAXB properties to Map<String, String>
*
* @param xProperties
* @return {@link Map}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index 130ba3e..bfe4813 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -42,7 +42,7 @@ import com.google.common.collect.Lists;
/**
* metastore resource api
- * <p/>
+ * <p> </p>
* This provides api for all things metastore.
*/
@Path("metastore")
@@ -86,7 +86,7 @@ public class MetastoreResource {
*
* @param sessionid The sessionid in which user is working
* @return StringList consisting of all database names.
- * @throws org.apache.lens.server.api.error.LensException
+ * @throws WebApplicationException
*/
@GET
@Path("databases")
@@ -202,7 +202,7 @@ public class MetastoreResource {
* ignored. If no dbOption or dbname are passed, then default is to get tables from current db.
* @param dbName The db name. If not empty, the tables in the db will be returned
* @return StringList consisting of all table names.
- * @throws LensException
+ * @throws WebApplicationException
*/
@GET
@Path("nativetables")
@@ -231,7 +231,7 @@ public class MetastoreResource {
* @param sessionid The sessionid in which user is working
* @param tableName The native table name
* @return JAXB representation of {@link XNativeTable}
- * @throws LensException
+ * @throws WebApplicationException
*/
@GET
@Path("nativetables/{tableName}")
@@ -433,7 +433,7 @@ public class MetastoreResource {
*
* @param sessionid The sessionid in which user is working
* @return StringList consisting of all the storage names
- * @throws LensException
+ * @throws WebApplicationException Wraps LensException
*/
@GET
@Path("storages")
@@ -579,7 +579,7 @@ public class MetastoreResource {
*
* @param sessionid The sessionid in which user is working
* @return StringList consisting of all the dimension names
- * @throws LensException
+ * @throws WebApplicationException wraps lensException
*/
@GET
@Path("dimensions")
@@ -1031,7 +1031,7 @@ public class MetastoreResource {
* @param sessionid The sessionid in which user is working
* @param factName The fact table name
* @param storage The storage name
- * @param filter The filter for partitions, string representation of the filter for ex: x < "xxx" and y >
+ * @param filter The filter for partitions, string representation of the filter for ex: x < "xxx" and y >
* "yyy"
* @return JAXB representation of {@link XPartitionList} containing {@link XPartition} objects
*/
@@ -1059,7 +1059,7 @@ public class MetastoreResource {
* @param sessionid The sessionid in which user is working
* @param factName The fact table name
* @param storage The storage name
- * @param filter The filter for partitions, string representation of the filter for ex: x < "xxx" and y >
+ * @param filter The filter for partitions, string representation of the filter for ex: x < "xxx" and y >
* "yyy"
* @return {@link APIResult} with state {@link Status#SUCCEEDED}, if drop was successful. {@link APIResult} with state
* {@link Status#FAILED}, if drop has failed
@@ -1383,7 +1383,7 @@ public class MetastoreResource {
* @param sessionid The sessionid in which user is working
* @param dimension The dimension table name
* @param storage The storage name
- * @param filter The filter for partitions, string representation of the filter for ex: x < "xxx" and y >
+ * @param filter The filter for partitions, string representation of the filter for ex: x < "xxx" and y >
* "yyy"
* @return JAXB representation of {@link XPartitionList} containing {@link XPartition} objects
*/
@@ -1407,7 +1407,7 @@ public class MetastoreResource {
* @param sessionid The sessionid in which user is working
* @param dimTableName The dimension table name
* @param storage The storage name
- * @param filter The filter for partitions, string representation of the filter for ex: x < 'xxx' and y >
+ * @param filter The filter for partitions, string representation of the filter for ex: x < 'xxx' and y >
* 'yyy'
* @return {@link APIResult} with state {@link Status#SUCCEEDED}, if drop was successful. {@link APIResult} with state
* {@link Status#FAILED}, if drop has failed
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index 9ae0d66..39f9f52 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -148,8 +148,8 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
/**
* The listener interface for receiving asyncQueryStatus events. The class that is interested in processing a
* asyncQueryStatus event implements this interface, and the object created with that class is registered with a
- * component using the component's <code>addAsyncQueryStatusListener<code> method. When the asyncQueryStatus event
- * occurs, that object's appropriate method is invoked.
+ * component using the component's <code>addAsyncQueryStatusListener<code> method.
+ * When the asyncQueryStatus event occurs, that object's appropriate method is invoked.
*/
public class AsyncQueryStatusListener extends AsyncEventListener<StatusChange> {
@@ -194,8 +194,8 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
/**
* The listener interface for receiving asyncSession events. The class that is interested in processing a
* asyncSession event implements this interface, and the object created with that class is registered with a
- * component using the component's <code>addAsyncSessionEventListener<code> method. When the asyncSessionEvent event
- * occurs, that object's appropriate method is invoked.
+ * component using the component's <code>addAsyncSessionEventListener<code> method.
+ * When the asyncSessionEvent event occurs, that object's appropriate method is invoked.
*/
public class AsyncSessionEventListener extends AsyncEventListener<SessionEvent> {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 8a1b7c8..429b262 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -2451,7 +2451,6 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* Add session's resources to selected driver if needed
*
* @param ctx QueryContext for executinf queries
- * @throws LensException
*/
protected void addSessionResourcesToDriver(final AbstractQueryContext ctx) {
LensDriver driver = ctx.getSelectedDriver();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index 15015d3..ec97b8d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -49,7 +49,7 @@ import lombok.extern.slf4j.Slf4j;
/**
* queryapi resource
- * <p/>
+ * <p></p>
* This provides api for all things query.
*/
@Slf4j
@@ -139,13 +139,13 @@ public class QueryServiceResource {
*
* @param sessionid The sessionid in which queryName is working
* @param state If any state is passed, all the queries in that state will be returned, otherwise all queries will
- * be returned. Possible states are {@value QueryStatus.Status#values()}
+ * be returned. Possible states are {link QueryStatus.Status#values()}
* @param queryName If any queryName is passed, all the queries containing the queryName will be returned, otherwise
* all the queries will be returned
* @param user Returns queries submitted by this user. If set to "all", returns queries of all users. By default,
* returns queries of the current user.
- * @param fromDate from date to search queries in a time range, the range is inclusive(submitTime >= fromDate)
- * @param toDate to date to search queries in a time range, the range is inclusive(toDate >= submitTime)
+ * @param fromDate from date to search queries in a time range, the range is inclusive(submitTime >= fromDate)
+ * @param toDate to date to search queries in a time range, the range is inclusive(toDate >= submitTime)
* @return List of {@link QueryHandle} objects
*/
@GET
@@ -183,16 +183,21 @@ public class QueryServiceResource {
* @param sessionid The session in which user is submitting the query. Any configuration set in the session will
* be picked up.
* @param query The query to run
- * @param operation The operation on the query. Supported operations are values: {@value SubmitOp#ESTIMATE},
- * {@value SubmitOp#EXPLAIN},{@value SubmitOp#EXECUTE} and {@value SubmitOp#EXECUTE_WITH_TIMEOUT}
+ * @param operation The operation on the query. Supported operations are values:
+ * {@link org.apache.lens.api.query.SubmitOp#ESTIMATE},
+ * {@link org.apache.lens.api.query.SubmitOp#EXPLAIN},
+ * {@link org.apache.lens.api.query.SubmitOp#EXECUTE} and
+ * {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT}
* @param conf The configuration for the query
- * @param timeoutmillis The timeout for the query, honored only in case of value {@value
- * SubmitOp#EXECUTE_WITH_TIMEOUT} operation
+ * @param timeoutmillis The timeout for the query, honored only in case of value {@link
+ * org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT} operation
* @param queryName human readable query name set by user (optional parameter)
- * @return {@link LensAPIResult} with DATA as {@link QueryHandle} in case of {@value SubmitOp#EXECUTE} operation.
- * {@link QueryPlan} in case of {@value SubmitOp#EXPLAIN} operation. {@link QueryHandleWithResultSet} in case
- * {@value SubmitOp#EXECUTE_WITH_TIMEOUT} operation. {@link QueryCost} in case of
- * {@value SubmitOp#ESTIMATE} operation.
+
+ * @return {@link LensAPIResult} with DATA as {@link QueryHandle} in case of
+ * {@link org.apache.lens.api.query.SubmitOp#EXECUTE} operation.
+ * {@link QueryPlan} in case of {@link org.apache.lens.api.query.SubmitOp#EXPLAIN} operation.
+ * {@link QueryHandleWithResultSet} in case {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT}
+ * operation. {@link QueryCost} in case of {@link org.apache.lens.api.query.SubmitOp#ESTIMATE} operation.
*/
@POST
@Path("queries")
@@ -241,17 +246,20 @@ public class QueryServiceResource {
*
* @param sessionid The session in which cancel is issued
* @param state If any state is passed, all the queries in that state will be cancelled, otherwise all queries
- * will be cancelled. Possible states are {@value QueryStatus.Status#values()} The queries in {@value
- * QueryStatus.Status#FAILED}, {@value QueryStatus.Status#CLOSED}, {@value
- * QueryStatus.Status#SUCCESSFUL} cannot be cancelled
+ * will be cancelled. Possible states are
+ * {@link org.apache.lens.api.query.QueryStatus.Status#values()}
+ * The queries in {@link org.apache.lens.api.query.QueryStatus.Status#FAILED},
+ * {@link org.apache.lens.api.query.QueryStatus.Status#CLOSED},
+ * {@link org.apache.lens.api.query.QueryStatus.Status#SUCCESSFUL} cannot be cancelled
* @param user If any user is passed, all the queries submitted by the user will be cancelled, otherwise all the
* queries will be cancelled
* @param queryName Cancel queries matching the query name
- * @param fromDate the from date, inclusive(submitTime>=fromDate)
- * @param toDate the to date, inclusive(toDate>=submitTime)
- * @return APIResult with state {@value Status#SUCCEEDED} in case of successful cancellation. APIResult with state
- * {@value Status#FAILED} in case of cancellation failure. APIResult with state {@value Status#PARTIAL} in case of
- * partial cancellation.
+ * @param fromDate the from date, inclusive(submitTime>=fromDate)
+ * @param toDate the to date, inclusive(toDate>=submitTime)
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * cancellation. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED}
+ * in case of cancellation failure. APIResult with state
+ * {@link org.apache.lens.api.APIResult.Status#PARTIAL} in case of partial cancellation.
*/
@DELETE
@Path("queries")
@@ -323,12 +331,14 @@ public class QueryServiceResource {
* @param sessionid The session in which user is preparing the query. Any configuration set in the session will be
* picked up.
* @param query The query to prepare
- * @param operation The operation on the query. Supported operations are {@value SubmitOp#EXPLAIN_AND_PREPARE} or
- * {@value SubmitOp#PREPARE}
+ * @param operation The operation on the query. Supported operations are
+ * {@link org.apache.lens.api.query.SubmitOp#EXPLAIN_AND_PREPARE} or
+ * {@link org.apache.lens.api.query.SubmitOp#PREPARE}
* @param conf The configuration for preparing the query
* @param queryName human readable query name set by user (optional parameter)
- * @return {@link QueryPrepareHandle} incase of {@value SubmitOp#PREPARE} operation. {@link QueryPlan} incase of
- * {@value SubmitOp#EXPLAIN_AND_PREPARE} and the query plan will contain the prepare handle as well.
+ * @return {@link QueryPrepareHandle} incase of {link org.apache.lens.api.query.SubmitOp#PREPARE} operation.
+ * {@link QueryPlan} incase of {@link org.apache.lens.api.query.SubmitOp#EXPLAIN_AND_PREPARE}
+ * and the query plan will contain the prepare handle as well.
*/
@POST
@Path("preparedqueries")
@@ -372,9 +382,10 @@ public class QueryServiceResource {
* @param queryName destroys queries matching the query name
* @param fromDate the from date
* @param toDate the to date
- * @return APIResult with state {@value Status#SUCCEEDED} in case of successful destroy. APIResult with state {@value
- * Status#FAILED} in case of destroy failure. APIResult with state {@value Status#PARTIAL} in case of partial
- * destroy.
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * destroy. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED} in case of destroy
+ * failure. APIResult with state {@link org.apache.lens.api.APIResult.Status#PARTIAL} in case of
+ * partial destroy.
*/
@DELETE
@Path("preparedqueries")
@@ -463,8 +474,9 @@ public class QueryServiceResource {
*
* @param sessionid The user session handle
* @param prepareHandle The prepare handle
- * @return APIResult with state {@link Status#SUCCEEDED} in case of successful destroy. APIResult with state {@link
- * Status#FAILED} in case of destroy failure.
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * destroy. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED} in case of
+ * destroy failure.
*/
@DELETE
@Path("preparedqueries/{prepareHandle}")
@@ -505,8 +517,9 @@ public class QueryServiceResource {
*
* @param sessionid The user session handle
* @param queryHandle The query handle
- * @return APIResult with state {@value Status#SUCCEEDED} in case of successful cancellation. APIResult with state
- * {@value Status#FAILED} in case of cancellation failure.
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * cancellation. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED} in case of
+ * cancellation failure.
*/
@DELETE
@Path("queries/{queryHandle}")
@@ -558,8 +571,8 @@ public class QueryServiceResource {
* @param sessionid The user session handle
* @param queryHandle The query handle
* @param conf The new configuration, will be on top of old one
- * @return APIResult with state {@value Status#SUCCEEDED} in case of successful update. APIResult with state {@value
- * Status#FAILED} in case of udpate failure.
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * update. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED} in case of udpate failure.
*/
@PUT
@Path("queries/{queryHandle}")
@@ -587,8 +600,8 @@ public class QueryServiceResource {
* @param sessionid The user session handle
* @param prepareHandle The prepare handle
* @param conf The new configuration, will be on top of old one
- * @return APIResult with state {@value Status#SUCCEEDED} in case of successful update. APIResult with state {@value
- * Status#FAILED} in case of udpate failure.
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * update. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED} in case of udpate failure.
*/
@PUT
@Path("preparedqueries/{prepareHandle}")
@@ -615,14 +628,16 @@ public class QueryServiceResource {
* @param sessionid The session in which user is submitting the query. Any configuration set in the session will
* be picked up.
* @param prepareHandle The Query to run
- * @param operation The operation on the query. Supported operations are {@value SubmitOp#EXECUTE} and {@value
- * SubmitOp#EXECUTE_WITH_TIMEOUT}
+ * @param operation The operation on the query. Supported operations are
+ * {@link org.apache.lens.api.query.SubmitOp#EXECUTE} and {@link
+ * org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT}
* @param conf The configuration for the execution of query
- * @param timeoutmillis The timeout for the query, honored only in case of {@value SubmitOp#EXECUTE_WITH_TIMEOUT}
- * operation
+ * @param timeoutmillis The timeout for the query, honored only in case of
+ * {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT} operation
* @param queryName human readable query name set by user (optional parameter)
- * @return {@link QueryHandle} in case of {@value SubmitOp#EXECUTE} operation. {@link QueryHandleWithResultSet} in
- * case {@value SubmitOp#EXECUTE_WITH_TIMEOUT} operation.
+ * @return {@link QueryHandle} in case of {link org.apache.lens.api.query.SubmitOp#EXECUTE} operation.
+ * {@link QueryHandleWithResultSet} in case {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT
+ * } operation.
*/
@POST
@Path("preparedqueries/{prepareHandle}")
@@ -725,8 +740,8 @@ public class QueryServiceResource {
*
* @param sessionid The user session handle
* @param queryHandle The query handle
- * @return APIResult with state {@value Status#SUCCEEDED} in case of successful close. APIResult with state {@value
- * Status#FAILED} in case of close failure.
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * close. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED} in case of close failure.
*/
@DELETE
@Path("queries/{queryHandle}/resultset")
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index fa061ec..36adee0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -95,8 +95,9 @@ public class HiveSessionService extends LensService implements SessionService {
}
/**
- * @inheritDoc
+ * {@inheritDoc}
*/
+ @Override
public int addResourceToAllServices(LensSessionHandle sessionid, String type, String path) {
int numAdded = 0;
boolean error = false;
@@ -136,7 +137,7 @@ public class HiveSessionService extends LensService implements SessionService {
}
/**
- * @inheritDoc
+ * {@inheritDoc}
*/
@Override
public void addResource(LensSessionHandle sessionid, String type, String path) {
@@ -152,7 +153,7 @@ public class HiveSessionService extends LensService implements SessionService {
}
/**
- * @inheritDoc
+ * {@inheritDoc}
*/
@Override
public void deleteResource(LensSessionHandle sessionid, String type, String path) {
@@ -200,7 +201,7 @@ public class HiveSessionService extends LensService implements SessionService {
}
/**
- * @inheritDoc
+ * {@inheritDoc}
*/
@Override
public LensSessionHandle openSession(String username, String password, String database,
@@ -255,7 +256,7 @@ public class HiveSessionService extends LensService implements SessionService {
}
/**
- * @inheritDoc
+ * {@inheritDoc}
*/
@Override
public List<String> getAllSessionParameters(LensSessionHandle sessionid, boolean verbose, String key)
@@ -287,7 +288,7 @@ public class HiveSessionService extends LensService implements SessionService {
}
/**
- * @inheritDoc
+ * {@inheritDoc}
*/
@Override
public void setSessionParameter(LensSessionHandle sessionid, String key, String value) {
@@ -454,7 +455,7 @@ public class HiveSessionService extends LensService implements SessionService {
}
/**
- * @inheritDoc
+ * {@inheritDoc}
*/
@Override
public void closeSession(LensSessionHandle sessionHandle) throws LensException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
index 15a8e06..4d82a06 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
@@ -44,7 +44,7 @@ import org.glassfish.jersey.media.multipart.FormDataParam;
/**
* Session resource api
- * <p/>
+ * <p></p>
* This provides api for all things in session.
*/
@Path("/session")
@@ -124,7 +124,7 @@ public class SessionResource {
/**
* Add a resource to the session to all LensServices running in this Lens server
- * <p/>
+ * <p></p>
* <p>
* The returned @{link APIResult} will have status SUCCEEDED <em>only if</em> the add operation was successful for all
* services running in this Lens server.
@@ -256,7 +256,7 @@ public class SessionResource {
/**
* Set value for a parameter specified by key
- * <p/>
+ * <p></p>
* The parameters can be a hive variable or a configuration. To set key as a hive variable, the key should be prefixed
* with 'hivevar:'. To set key as configuration parameter, the key should be prefixed with 'hiveconf:' If no prefix is
* attached, the parameter is set as configuration.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
index b7c60bb..71d29b4 100644
--- a/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
@@ -39,7 +39,7 @@ import org.json.JSONObject;
/**
* metastore UI resource api
- * <p/>
+ * <p></p>
* This provides api for all things metastore UI.
*/
@@ -74,7 +74,7 @@ public class MetastoreUIResource {
*
* @param publicId The publicId for the session in which user is working
* @return JSON string consisting of different table names and types
- * @throws LensException, JSONException
+ * @throws WebApplicationException wraps LensException
*/
@GET
@Path("tables")
@@ -137,7 +137,7 @@ public class MetastoreUIResource {
* @param publicId The publicId for the session in which user is working
* @param name name of cube or dimension to be described
* @return JSON string consisting of different dimension and measure names and types
- * @throws LensException, JSONException
+ * @throws WebApplicationException wraps LensException
*/
@GET
@Path("tables/{name}")
@@ -210,7 +210,6 @@ public class MetastoreUIResource {
* @param publicId The publicId for the session in which user is working
* @param keyword keyword to be searched
* @return JSON string consisting of different table and column names and types
- * @throws LensException, JSONException
*/
@GET
@Path("searchablefields")
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
index e0586e5..391cf08 100644
--- a/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
@@ -105,7 +105,7 @@ public class QueryServiceUIResource {
*
* @param publicId The public id of the session in which user is working
* @param state If any state is passed, all the queries in that state will be returned, otherwise all queries will
- * be returned. Possible states are {@value QueryStatus.Status#values()}
+ * be returned. Possible states are {@link org.apache.lens.api.query.QueryStatus.Status#values()}
* @param user return queries matching the user. If set to "all", return queries of all users. By default,
* returns queries of the current user.
* @param queryName human readable query name set by user (optional)
@@ -225,8 +225,8 @@ public class QueryServiceUIResource {
*
* @param publicId The user session handle
* @param queryHandle The query handle
- * @return APIResult with state {@value org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
- * cancellation. APIResult with state {@value org.apache.lens.api.APIResult.Status#FAILED} in case of cancellation
+ * @return APIResult with state {@link org.apache.lens.api.APIResult.Status#SUCCEEDED} in case of successful
+ * cancellation. APIResult with state {@link org.apache.lens.api.APIResult.Status#FAILED} in case of cancellation
* failure.
*/
@DELETE
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
index 3d47c4f..cd1ba36 100644
--- a/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
@@ -40,7 +40,7 @@ import org.glassfish.jersey.media.multipart.FormDataParam;
/**
* Session resource api
- * <p/>
+ * <p></p>
* This provides api for all things in session.
*/
@Path("/uisession")
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0cf6c4e2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a896aea..ec782af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -86,7 +86,7 @@
<test.tmp.dir>${basedir}/target/tmp</test.tmp.dir>
<!-- maven plugins -->
- <jaxb2.plugin.version>0.8.0</jaxb2.plugin.version>
+ <jaxb2.plugin.version>0.12.3</jaxb2.plugin.version>
<jaxb2.basics.plugin.version>0.6.3</jaxb2.basics.plugin.version>
<jetty.plugin.version>${jetty.version}</jetty.plugin.version>
<exec.plugin.version>1.2.1</exec.plugin.version>
@@ -405,8 +405,8 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- <configuration>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
<generateReports>true</generateReports>
</configuration>
</plugin>
@@ -649,13 +649,13 @@
<updatePolicy>never</updatePolicy>
</snapshots>
</repository>
- <repository>
+ <!--<repository>
<id>Codehaus repository</id>
<url>http://repository.codehaus.org/</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
- </repository>
+ </repository>-->
<repository>
<id>apache.snapshots.repo</id>
<url>https://repository.apache.org/content/groups/snapshots</url>
[06/51] [abbrv] incubator-lens git commit: Lens-549 : Fix Lens client
instantiation (Pranav Kumar Agarwal via sharad)
Posted by jd...@apache.org.
Lens-549 : Fix Lens client instantiation (Pranav Kumar Agarwal via sharad)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/1254a1e2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/1254a1e2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/1254a1e2
Branch: refs/heads/current-release-line
Commit: 1254a1e2e774752ecb672a0100fffdc7d3779a5c
Parents: 4498c44
Author: Pranav Kumar Agarwal <pr...@gmail.com>
Authored: Sat May 30 16:36:28 2015 +0530
Committer: Sharad Agarwal <sh...@flipkarts-MacBook-Pro.local>
Committed: Sat May 30 16:36:28 2015 +0530
----------------------------------------------------------------------
.../lens/cli/commands/BaseLensCommand.java | 17 +++++---
.../lens/client/LensClientSingletonWrapper.java | 43 +++++++++++---------
2 files changed, 34 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1254a1e2/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
index 3deddfa..74b4e91 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.IOException;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+
import java.util.Date;
import org.apache.lens.client.LensClient;
@@ -57,6 +58,8 @@ public class BaseLensCommand implements ExecutionProcessor {
protected static boolean isConnectionActive;
public static final String DATE_FMT = "yyyy-MM-dd'T'HH:mm:ss:SSS";
+ private LensClient lensClient = null;
+
public static final ThreadLocal<DateFormat> DATE_PARSER =
new ThreadLocal<DateFormat>() {
@Override
@@ -83,7 +86,7 @@ public class BaseLensCommand implements ExecutionProcessor {
protected static synchronized void closeClientConnection() {
if (isConnectionActive) {
log.debug("Request for stopping lens cli received");
- getClient().closeConnection();
+ getClientWrapper().getClient().closeConnection();
isConnectionActive = false;
}
}
@@ -92,7 +95,6 @@ public class BaseLensCommand implements ExecutionProcessor {
* Instantiates a new base lens command.
*/
public BaseLensCommand() {
- getClient();
mapper = new ObjectMapper();
mapper.setSerializationInclusion(Inclusion.NON_NULL);
mapper.setSerializationInclusion(Inclusion.NON_DEFAULT);
@@ -112,15 +114,18 @@ public class BaseLensCommand implements ExecutionProcessor {
return false;
}
});
- isConnectionActive = true;
}
public void setClient(LensClient client) {
- getClientWrapper().setClient(client);
+ lensClient = client;
}
- public static LensClient getClient() {
- return getClientWrapper().getClient();
+ public LensClient getClient() {
+ if (lensClient == null) {
+ setClient(getClientWrapper().getClient());
+ isConnectionActive = true;
+ }
+ return lensClient;
}
public static LensClientSingletonWrapper getClientWrapper() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1254a1e2/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
index 165d41d..d9ab5a1 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
@@ -50,26 +50,6 @@ public class LensClientSingletonWrapper {
* Instantiates a new lens client singleton wrapper.
*/
LensClientSingletonWrapper() {
- try {
- client = new LensClient();
- } catch (LensClientServerConnectionException e) {
- if (e.getErrorCode() != 401) {
- explainFailedAttempt(e);
- throw e;
- }
- // Connecting without password prompt failed.
- for (int i = 0; i < MAX_RETRIES; i++) {
- try {
- client = new LensClient(Credentials.prompt());
- break;
- } catch (LensClientServerConnectionException lensClientServerConnectionException) {
- explainFailedAttempt(lensClientServerConnectionException);
- if (i == MAX_RETRIES - 1) {
- throw lensClientServerConnectionException;
- }
- }
- }
- }
}
/**
@@ -92,6 +72,29 @@ public class LensClientSingletonWrapper {
}
public LensClient getClient() {
+ if (client != null) {
+ return client;
+ }
+ try {
+ client = new LensClient();
+ } catch (LensClientServerConnectionException e) {
+ if (e.getErrorCode() != 401) {
+ explainFailedAttempt(e);
+ throw e;
+ }
+ // Connecting without password prompt failed.
+ for (int i = 0; i < MAX_RETRIES; i++) {
+ try {
+ client = new LensClient(Credentials.prompt());
+ break;
+ } catch (LensClientServerConnectionException lensClientServerConnectionException) {
+ explainFailedAttempt(lensClientServerConnectionException);
+ if (i == MAX_RETRIES - 1) {
+ throw lensClientServerConnectionException;
+ }
+ }
+ }
+ }
return client;
}
[46/51] [abbrv] incubator-lens git commit: LENS-621 : Fix seekToStart
for closed resultsets in JDBCDriver
Posted by jd...@apache.org.
LENS-621 : Fix seekToStart for closed resultsets in JDBCDriver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/3eddcb5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/3eddcb5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/3eddcb5b
Branch: refs/heads/current-release-line
Commit: 3eddcb5b2f5025bfc032155c71171c33683db67d
Parents: 77f6aac
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Mon Jun 22 10:51:06 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Jun 22 10:51:06 2015 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/client/LensClient.java | 6 ++----
.../main/java/org/apache/lens/client/LensClientResultSet.java | 2 +-
.../main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java | 2 +-
3 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3eddcb5b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 51f586a..b15b024 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -144,8 +144,7 @@ public class LensClient {
}
LensClientResultSet result = null;
if (statement.getStatus().isResultSetAvailable()) {
- result = new LensClientResultSet(statement.getResultSet(),
- statement.getResultSetMetaData());
+ result = new LensClientResultSet(statement.getResultSetMetaData(), statement.getResultSet());
}
return new LensClientResultSetWithStats(result, statement.getQuery());
}
@@ -161,8 +160,7 @@ public class LensClient {
}
LensClientResultSet result = null;
if (statement.getStatus().isResultSetAvailable()) {
- result = new LensClientResultSet(statement.getResultSet(),
- statement.getResultSetMetaData());
+ result = new LensClientResultSet(statement.getResultSetMetaData(), statement.getResultSet());
}
return new LensClientResultSetWithStats(result, statement.getQuery());
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3eddcb5b/lens-client/src/main/java/org/apache/lens/client/LensClientResultSet.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClientResultSet.java b/lens-client/src/main/java/org/apache/lens/client/LensClientResultSet.java
index 79e8abd..674926f 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClientResultSet.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClientResultSet.java
@@ -38,7 +38,7 @@ public class LensClientResultSet {
* @param result the result
* @param resultSetMetaData the result set meta data
*/
- public LensClientResultSet(QueryResult result, QueryResultSetMetadata resultSetMetaData) {
+ public LensClientResultSet(QueryResultSetMetadata resultSetMetaData, QueryResult result) {
this.result = result;
this.resultSetMetadata = resultSetMetaData;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3eddcb5b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
index bbcc3f1..2687673 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
@@ -298,7 +298,7 @@ public class JDBCResultSet extends InMemoryResultSet {
@Override
public boolean seekToStart() throws LensException {
try {
- if (!resultSet.isBeforeFirst()) {
+ if (!resultSet.isClosed() && !resultSet.isBeforeFirst()) {
resultSet.beforeFirst();
}
return true;
[21/51] [abbrv] incubator-lens git commit: LENS-539 : Fix issues in
README and pom.xml reported in 2.1 release
Posted by jd...@apache.org.
LENS-539 : Fix issues in README and pom.xml reported in 2.1 release
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/0776290e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/0776290e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/0776290e
Branch: refs/heads/current-release-line
Commit: 0776290ea258d1d289f93ec27a4705fafb428310
Parents: 1b1d335
Author: Yash Sharma <ya...@gmail.com>
Authored: Thu Jun 11 11:09:14 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Jun 11 11:09:14 2015 +0530
----------------------------------------------------------------------
README.md | 18 +++++++++++++++---
pom.xml | 30 ++++++++++++++++++++----------
2 files changed, 35 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0776290e/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 4cd8e79..769acda 100644
--- a/README.md
+++ b/README.md
@@ -4,9 +4,21 @@ Apache Lens
Apache Lens is a unified Analytics Platform. Lens aims to cut the Data Analytics silos by providing a single view of data
across multiple tiered data stores and optimal execution environment for the analytical query.
-Building source :
-Requires JDK(1.7) and Apache Maven(3.x) to be installed for the build. Build does not pass with JDK(1.8) yet.
-Please run <mvn clean package> to build the project.
+Prerequisites :
+Apache Lens requires JDK(1.7) and Apache Maven(3.x) to be installed for the build. Build does not pass with JDK(1.8) yet.
+JAVA_HOME is required for running tests.
+
+Confirm versions :
+ # java -version
+ # mvn --version
+ # echo ${JAVA_HOME}
+
+Additionally MAVEN_OPTS can be configured as :
+ # export MAVEN_OPTS="-Xms512m -Xmx2048"
+
+Build the project :
+ # mvn clean package
+
See [Development Environment Setup] (http://lens.incubator.apache.org/developer/contribute.html#Development_Environment_Setup)
and [Building from source] (http://lens.incubator.apache.org/developer/contribute.html#Building_from_source) docs for
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/0776290e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 645e2f4..e2cbac5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -175,7 +175,6 @@
<id>amareshwari</id>
<email>amareshwari@apache.org</email>
<name>Amareshwari Sriramadasu</name>
- <organization>InMobi</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -187,7 +186,6 @@
<id>gunther</id>
<email>gunther@apache.org</email>
<name>Gunther Hagleitner</name>
- <organization>Hortonworks</organization>
<timezone>-8</timezone>
<roles>
<role>PPMC</role>
@@ -198,7 +196,6 @@
<id>jdhok</id>
<email>jdhok@apache.org</email>
<name>Jaideep Dhok</name>
- <organization>InMobi</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -209,7 +206,6 @@
<id>raghavsingh</id>
<email>raghavsingh@apache.org</email>
<name>Raghavendra Singh</name>
- <organization>InMobi</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -220,7 +216,6 @@
<id>prongs</id>
<email>prongs@apache.org</email>
<name>Rajat Khandelwal</name>
- <organization>InMobi</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -231,7 +226,6 @@
<id>sharad</id>
<email>sharad@apache.org</email>
<name>Sharad Agarwal</name>
- <organization>Flipkart</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -242,7 +236,6 @@
<id>sreekanth</id>
<email>sreekanth@apache.org</email>
<name>Sreekanth Ramakrishnan</name>
- <organization>SoftwareAG</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -253,7 +246,6 @@
<id>sriksun</id>
<email>sriksun@apache.org</email>
<name>Srikanth Sundarrajan</name>
- <organization>InMobi</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -264,7 +256,6 @@
<id>sumasai</id>
<email>sumasai@apache.org</email>
<name>Suma Shivaprasad</name>
- <organization>InMobi</organization>
<timezone>+5.5</timezone>
<roles>
<role>PPMC</role>
@@ -275,13 +266,32 @@
<id>vikram</id>
<email>vikram@apache.org</email>
<name>Vikram Dixit</name>
- <organization>Hortonworks</organization>
<timezone>-8</timezone>
<roles>
<role>PPMC</role>
<role>Committer</role>
</roles>
</developer>
+ <developer>
+ <id>himanshugahlaut</id>
+ <email>himanshugahlaut@apache.org</email>
+ <name>Himanshu Gahlaut</name>
+ <timezone>+5.5</timezone>
+ <roles>
+ <role>PPMC</role>
+ <role>Committer</role>
+ </roles>
+ </developer>
+ <developer>
+ <id>arshadmatin</id>
+ <email>arshadmatin@apache.org</email>
+ <name>Arshad Matin</name>
+ <timezone>+5.5</timezone>
+ <roles>
+ <role>PPMC</role>
+ <role>Committer</role>
+ </roles>
+ </developer>
</developers>
<scm>
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-lens.git</connection>
[49/51] [abbrv] incubator-lens git commit: LENS-625 : XTERM specific
changes to support automatic terminal title for CLI
Posted by jd...@apache.org.
LENS-625 : XTERM specific changes to support automatic terminal title for CLI
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/1a06e770
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/1a06e770
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/1a06e770
Branch: refs/heads/current-release-line
Commit: 1a06e77006af05c8d08dc7e99e25efde43bed465
Parents: 5f85911
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Jun 22 14:33:35 2015 +0530
Committer: jdhok <jd...@apache.org>
Committed: Mon Jun 22 14:33:35 2015 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/cli/skel/LensBanner.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1a06e770/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
index f577509..1e05e53 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
@@ -50,6 +50,11 @@ public class LensBanner extends DefaultBannerProvider {
@Override
public String getVersion() {
- return getClass().getPackage().getImplementationVersion() + " built with spring shell " + super.getVersion();
+ return getClass().getPackage().getImplementationVersion();
+ }
+
+ @Override
+ public String getProviderName() {
+ return "Lens CLI";
}
}
[17/51] [abbrv] incubator-lens git commit: LENS-513: Add jar CLI
command can now take regex of paths
Posted by jd...@apache.org.
LENS-513: Add jar CLI command can now take regex of paths
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/84666df0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/84666df0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/84666df0
Branch: refs/heads/current-release-line
Commit: 84666df01ded1e944b418abde08625a00789305d
Parents: 10d647f
Author: Yash Sharma <ya...@gmail.com>
Authored: Tue Jun 9 12:06:57 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:06:57 2015 +0530
----------------------------------------------------------------------
.../lens/cli/TestLensConnectionCliCommands.java | 132 ++++++++++++-
.../lens/server/session/SessionResource.java | 56 ++++--
.../apache/lens/server/util/ScannedPaths.java | 197 +++++++++++++++++++
.../apache/lens/server/TestServerRestart.java | 2 +-
.../lens/server/util/TestScannedPaths.java | 144 ++++++++++++++
5 files changed, 516 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/84666df0/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
index 3238edb..7503221 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
@@ -20,6 +20,7 @@ package org.apache.lens.cli;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
import javax.ws.rs.BadRequestException;
@@ -79,13 +80,53 @@ public class TestLensConnectionCliCommands extends LensCliApplicationTest {
private File createNewPath(String fileName) {
File f = new File(fileName);
try {
- f.createNewFile();
+ if (!f.exists()) {
+ f.createNewFile();
+ }
} catch (IOException e) {
Assert.fail("Unable to create test file, so bailing out.");
}
return f;
}
+ private String getFilePathFromUri(String uripath){
+ try {
+ return new URI(uripath).getPath();
+ } catch (Exception e){
+ return null;
+ }
+ }
+
+ private File createNewFile(String fileName) {
+ File f = null;
+ try {
+ String filepath = getFilePathFromUri(fileName);
+ Assert.assertNotNull(fileName, "Unable to get filepath from uri pattern.");
+ f = new File(filepath);
+ if (!f.exists()) {
+ f.createNewFile();
+ }
+ } catch (Exception e) {
+ Assert.fail("Unable to create test file, so bailing out.");
+ }
+ return f;
+ }
+
+ private File deleteFile(String fileName) {
+ File f = null;
+ try {
+ String filepath = getFilePathFromUri(fileName);
+ Assert.assertNotNull(fileName, "Unable to get filepath from uri pattern.");
+ f = new File(filepath);
+ if (f.exists()) {
+ f.delete();
+ }
+ } catch (Exception e) {
+ Assert.fail("Unable to delete test file, so bailing out.");
+ }
+ return f;
+ }
+
/**
* Test file commands.
*/
@@ -106,6 +147,7 @@ public class TestLensConnectionCliCommands extends LensCliApplicationTest {
result = commands.removeFile(filename);
Assert.assertEquals("Delete resource succeeded", result);
+
LOG.debug("Testing set/remove file operation done");
} finally {
if (f != null) {
@@ -116,6 +158,44 @@ public class TestLensConnectionCliCommands extends LensCliApplicationTest {
}
/**
+ * Test file commands with URI in regex.
+ */
+ @Test
+ public void testFileCommandsWithURIRegex() {
+ LensClient client = new LensClient();
+ LensConnectionCommands commands = new LensConnectionCommands();
+ commands.setClient(client);
+ LOG.debug("Testing set/remove file operations");
+
+ java.io.File file = new java.io.File("");
+ String projectdir = file.getAbsolutePath();
+
+ /* Tests input file pattern file: and file:// */
+ String filenameA = "file:" + projectdir + "/target/tempdata_a.txt";
+ String filenameB = "file://" + projectdir +"/target/tempdata_b.txt";
+
+ String fileRegex = "file:" + projectdir + "/target/tempdata_*.txt";
+
+ try {
+ createNewFile(filenameA);
+ createNewFile(filenameB);
+
+ String result = commands.addFile(fileRegex);
+ Assert.assertEquals("Add resource succeeded", result);
+
+ result = commands.removeFile(fileRegex);
+ Assert.assertEquals("Delete resource succeeded", result);
+
+ LOG.debug("Testing set/remove file operation done");
+ } finally {
+ deleteFile(filenameA);
+ deleteFile(filenameB);
+ commands.quitShell();
+ }
+ }
+
+
+ /**
* Test jar commands.
*/
@Test
@@ -145,6 +225,56 @@ public class TestLensConnectionCliCommands extends LensCliApplicationTest {
}
/**
+ * Test jar commands with regex specified.
+ */
+ @Test
+ public void testResourceCommandsWithRegex() {
+ LensClient client = new LensClient();
+ LensConnectionCommands commands = new LensConnectionCommands();
+ commands.setClient(client);
+ LOG.debug("Testing set/remove file operations");
+
+ File fileA = null, fileB = null;
+ String filenameA, filenameB, fileRegex, result;
+ try {
+ filenameA = "target/tempdata_a";
+ filenameB = "target/tempdata_b";
+ fileRegex = "target/tempdata_*";
+
+ fileA = createNewPath(filenameA);
+ fileB = createNewPath(filenameB);
+ result = commands.addFile(fileRegex);
+ Assert.assertEquals("Add resource succeeded", result);
+
+ result = commands.removeFile(fileRegex);
+ Assert.assertEquals("Delete resource succeeded", result);
+
+ filenameA = "target/tempdata_a.jar";
+ filenameB = "target/tempdata_b.jar";
+ fileRegex = "target/tempdata_*.jar";
+
+ fileA = createNewPath(filenameA);
+ fileB = createNewPath(filenameB);
+
+ result = commands.addJar(fileRegex);
+ Assert.assertEquals("Add resource succeeded", result);
+
+ result = commands.removeJar(fileRegex);
+ Assert.assertEquals("Delete resource succeeded", result);
+
+ LOG.debug("Testing set/remove resource operation done");
+ } finally {
+ if (fileA != null) {
+ fileA.delete();
+ }
+ if (fileB != null) {
+ fileB.delete();
+ }
+ commands.quitShell();
+ }
+ }
+
+ /**
* Test list resources commands.
*/
@Test
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/84666df0/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
index 5ffc086..15a8e06 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
@@ -19,6 +19,7 @@
package org.apache.lens.server.session;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -34,6 +35,7 @@ import org.apache.lens.server.LensService;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
+import org.apache.lens.server.util.ScannedPaths;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -141,10 +143,24 @@ public class SessionResource {
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
public APIResult addResource(@FormDataParam("sessionid") LensSessionHandle sessionid,
@FormDataParam("type") String type, @FormDataParam("path") String path) {
- int numAdded = sessionService.addResourceToAllServices(sessionid, type, path);
+ Iterator<String> foundFiles = new ScannedPaths(path, type).iterator();
+ String matchedPath = null;
+ int matchedPathsCount = 0;
+
+ if (foundFiles == null) {
+ return new APIResult(Status.FAILED, "No matching resources found for provided path.");
+ }
+
+ int numAdded = 0;
+ while (foundFiles.hasNext()) {
+ matchedPath = foundFiles.next();
+ numAdded += sessionService.addResourceToAllServices(sessionid, type, matchedPath);
+ matchedPathsCount++;
+ }
+
if (numAdded == 0) {
- return new APIResult(Status.FAILED, "Add resource has failed ");
- } else if (numAdded != LensServices.get().getLensServices().size()) {
+ return new APIResult(Status.FAILED, "Add resource has failed");
+ } else if ((numAdded / matchedPathsCount) != LensServices.get().getLensServices().size()) {
return new APIResult(Status.PARTIAL, "Add resource is partial");
}
return new APIResult(Status.SUCCEEDED, "Add resource succeeded");
@@ -182,21 +198,34 @@ public class SessionResource {
*/
@PUT
@Path("resources/delete")
+
@Consumes({MediaType.MULTIPART_FORM_DATA})
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
public APIResult deleteResource(@FormDataParam("sessionid") LensSessionHandle sessionid,
@FormDataParam("type") String type, @FormDataParam("path") String path) {
+ Iterator<String> foundFiles = new ScannedPaths(path, type).iterator();
+ String matchedPath = null;
+
+ if (foundFiles == null) {
+ return new APIResult(Status.PARTIAL, "No matching resources found for provided path.");
+ }
+
int numDeleted = 0;
- for (LensService service : LensServices.get().getLensServices()) {
- try {
- service.deleteResource(sessionid, type, path);
- numDeleted++;
- } catch (LensException e) {
- LOG.error("Failed to delete resource in service:" + service, e);
- if (numDeleted != 0) {
- return new APIResult(Status.PARTIAL, "Delete resource is partial, failed for service:" + service.getName());
- } else {
- return new APIResult(Status.PARTIAL, "Delete resource has failed");
+
+ while(foundFiles.hasNext()) {
+ matchedPath = foundFiles.next();
+
+ for (LensService service : LensServices.get().getLensServices()) {
+ try {
+ service.deleteResource(sessionid, type, matchedPath);
+ numDeleted++;
+ } catch (LensException e) {
+ LOG.error("Failed to delete resource in service:" + service, e);
+ if (numDeleted != 0) {
+ return new APIResult(Status.PARTIAL, "Delete resource is partial, failed for service:" + service.getName());
+ } else {
+ return new APIResult(Status.FAILED, "Delete resource has failed");
+ }
}
}
}
@@ -245,4 +274,5 @@ public class SessionResource {
sessionService.setSessionParameter(sessionid, key, value);
return new APIResult(Status.SUCCEEDED, "Set param succeeded");
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/84666df0/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java
new file mode 100644
index 0000000..ac773fe
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.util;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+
+import org.apache.hadoop.fs.Path;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class ScannedPaths implements Iterable<String> {
+ private String path = null;
+ private String type = null;
+
+ private Path fsPath;
+
+ /** Is the provided expression parsed **/
+ private boolean isScanned;
+
+ /* Keep all matched paths so we don't have to query filesystem multiple times */
+ private Map<String, String> matchedPaths = null;
+
+ /* The Chosen Ones */
+ @Getter(lazy=true) private final List<String> finalPaths = getMatchedPaths();
+
+ public ScannedPaths(String path, String type) {
+ this.path = path;
+ this.type = type;
+ }
+
+ @Override
+ public Iterator<String> iterator() {
+ /** Does all the pattern matching and returns the iterator to finalPaths collection **/
+ return (getFinalPaths() == null) ? null : getFinalPaths().iterator();
+ }
+
+ /**
+ * Method that computes path of resources matching the input path or path regex pattern.
+ * If provided path is a directory it additionally checks for the jar_order or glob_order file
+ * that imposes ordering of resources and filters out other resources.
+ *
+ * Updates finalPaths List with matched paths and returns an iterator for matched paths.
+ */
+ private List<String> getMatchedPaths() {
+ List<String> finalPaths = null;
+ try {
+ FileSystem fs = FileSystem.get(new URI(path), new Configuration());
+ fsPath = new Path(new URI(path).getPath());
+
+ if (fs.isDirectory(fsPath)) {
+ findAllMatchedPaths(true);
+ filterByJarType();
+ /* Updates finalPaths List with restrictions imposed
+ by jar_order/glob_order file */
+ finalPaths = getMatchedPathsFilteredByOrder();
+ } else {
+ findAllMatchedPaths(false);
+ filterByJarType();
+ if (matchedPaths != null) {
+ finalPaths = new ArrayList<String>(matchedPaths.values());
+ }
+ }
+ } catch (FileNotFoundException fex) {
+ log.error("File not found while scanning path.", fex);
+ } catch (URISyntaxException | IOException ex) {
+ log.error("Exception while initializing PathScanner.", ex);
+ }
+ return finalPaths;
+ }
+
+ /**
+ * Populates the matchedPaths[] with all paths matching the pattern.
+ */
+ private void findAllMatchedPaths(boolean isDir) {
+ try {
+ Path path = isDir ? new Path(fsPath, "*") : fsPath;
+ FileStatus[] statuses = path.getFileSystem(new Configuration()).globStatus(path);
+
+ if (statuses == null || statuses.length == 0) {
+ log.info("No matched paths found for expression " + path);
+ return;
+ }
+ matchedPaths = new HashMap<String, String>();
+ for (int count = 0; count < statuses.length; count++) {
+ matchedPaths.put(statuses[count].getPath().getName(), statuses[count].getPath().toString());
+ }
+ } catch (FileNotFoundException fex) {
+ log.error("File not found while scanning path.", fex);
+ return;
+ } catch (IOException ioex) {
+ log.error("IOException while scanning path.", ioex);
+ return;
+ }
+ }
+
+ /**
+ * Filters the matchedPaths by "jar" type.
+ * Removes non-jar resources if type is specified as "jar".
+ */
+ private void filterByJarType() {
+ if (matchedPaths == null) {
+ return;
+ } else if (type.equalsIgnoreCase("jar")) {
+ Iterator<Map.Entry<String, String>> iter = matchedPaths.entrySet().iterator();
+
+ while (iter.hasNext()) {
+ Map.Entry<String, String> entry = iter.next();
+ if (!entry.getKey().endsWith(".jar")) {
+ iter.remove();
+ }
+ }
+ }
+ }
+
+ /**
+ * Filters the matchedPath[] to remove unwanted resources
+ * and apply ordering to the resources as specified in jar_order or glob_order file.
+ * Bypasses filtering if none of the files is present in the directory.
+ */
+ private List<String> getMatchedPathsFilteredByOrder() {
+ if (matchedPaths == null) {
+ return null;
+ }
+
+ List<String> finalPaths = null;
+ InputStream resourceOrderIStream = null;
+ List<String> resources;
+ try {
+ FileSystem fs = fsPath.getFileSystem(new Configuration());
+ Path resourceOrderFile = new Path(fsPath.toUri().getPath(), "jar_order");
+
+ if (!fs.exists(resourceOrderFile)) {
+ resourceOrderFile = new Path(fsPath.toUri().getPath(), "glob_order");
+ if (!fs.exists(resourceOrderFile)) {
+ /* No order file present. Bypass filtering and Add all resource matching pattern */
+ return new ArrayList<>(matchedPaths.values());
+ }
+ }
+
+ resourceOrderIStream = fs.open(resourceOrderFile);
+ resources = IOUtils.readLines(resourceOrderIStream, Charset.forName("UTF-8"));
+ finalPaths = new ArrayList<>();
+ for(String resource : resources) {
+ if (resource == null || resource.isEmpty()) {
+ continue;
+ }
+
+ if (matchedPaths.containsKey(resource)) {
+ finalPaths.add(matchedPaths.get(resource));
+ }
+ }
+ } catch (FileNotFoundException fex) {
+ log.error("File not found while scanning path.", fex);
+ finalPaths = null;
+ } catch (IOException ioex) {
+ log.error("IOException while scanning path.", ioex);
+ finalPaths = null;
+ } finally {
+ IOUtils.closeQuietly(resourceOrderIStream);
+ }
+ return finalPaths;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/84666df0/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index 01dcf54..09f8dc7 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -409,7 +409,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
Assert.assertEquals(session.getLensSessionPersistInfo().getResources().size(), 1);
LensSessionImpl.ResourceEntry resourceEntry = session.getLensSessionPersistInfo().getResources().get(0);
Assert.assertEquals(resourceEntry.getType(), "file");
- Assert.assertEquals(resourceEntry.getLocation(), "target/test-classes/lens-site.xml");
+ Assert.assertTrue(resourceEntry.getLocation().contains("target/test-classes/lens-site.xml"));
// close session
result = sessionTarget.queryParam("sessionid", restartTestSession).request().delete(APIResult.class);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/84666df0/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java b/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
new file mode 100644
index 0000000..5073634
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Iterator;
+
+import org.apache.commons.io.FileUtils;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.Iterators;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Test(groups = "unit-test")
+public class TestScannedPaths {
+
+ public void testScannedPaths() throws Exception {
+ File fileA = null, fileB = null;
+ String filenameA, filenameB, fileRegex;
+ String tempPath = "target/tempfiles/";
+ ScannedPaths sc;
+ Iterator<String> iter = null;
+
+ try {
+ filenameA = "tempdata_a";
+ filenameB = "tempdata_b";
+ fileRegex = tempPath + "tempdata_*";
+
+ fileA = createNewPath(tempPath + filenameA);
+ fileB = createNewPath(tempPath + filenameB);
+ sc = new ScannedPaths(fileRegex, "file");
+
+ Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains(filenameA));
+ Assert.assertTrue(iter.next().contains(filenameB));
+
+ /**
+ * Testing negative Scenario
+ **/
+ fileRegex = tempPath + "tempdata_unknown_*";
+ sc = new ScannedPaths(fileRegex, "file");
+
+ Assert.assertNull(sc.iterator(), "Iterator should be null for unmatched path patterns.");
+ } catch (Exception e) {
+ Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
+ } finally {
+ FileUtils.deleteQuietly(new File(tempPath));
+ }
+ }
+
+ public void testScannedPathsJarGlobOrder() throws Exception {
+ File fileA = null, fileB = null, fileC = null;
+ File jarFile = null, globFile = null;
+ String filenameA, filenameB, filenameC, fileRegex;
+ String tempPath = "target/tempfiles/";
+ ScannedPaths sc;
+ Iterator<String> iter = null;
+ PrintWriter writer;
+
+ try {
+ filenameA = "tempdata_a.jar";
+ filenameB = "tempdata_b.jar";
+ filenameC = "tempdata_c.data";
+ fileRegex = tempPath;
+
+ fileA = createNewPath(tempPath + filenameA);
+ fileB = createNewPath(tempPath + filenameB);
+ fileC = createNewPath(tempPath + filenameC);
+
+ /** Test jar_order **/
+ jarFile = createNewPath(tempPath + "jar_order");
+ writer = new PrintWriter(jarFile, "UTF-8");
+ writer.println(filenameB);
+ writer.println(filenameA);
+ writer.close();
+ sc = new ScannedPaths(fileRegex, "jar");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains(filenameB));
+ Assert.assertTrue(iter.next().contains(filenameA));
+
+ /** Test glob_order **/
+ if (jarFile != null) {
+ jarFile.delete();
+ }
+ globFile = createNewPath(tempPath + "glob_order");
+ writer = new PrintWriter(globFile, "UTF-8");
+ writer.println(filenameB);
+ writer.println(filenameA);
+ writer.println(filenameC);
+ writer.close();
+ sc = new ScannedPaths(fileRegex, "file");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 3, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains(filenameB));
+ Assert.assertTrue(iter.next().contains(filenameA));
+ Assert.assertTrue(iter.next().contains(filenameC));
+ } catch (Exception e) {
+ Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
+ } finally {
+ FileUtils.deleteQuietly(new File(tempPath));
+ }
+ }
+
+ private File createNewPath(String fileName) {
+ File f = new File(fileName);
+ try {
+ if (!f.getParentFile().exists()) {
+ f.getParentFile().mkdirs();
+ }
+ if (!f.exists()) {
+ f.createNewFile();
+ }
+ } catch (IOException e) {
+ Assert.fail("Unable to create test file, so bailing out.");
+ }
+ return f;
+ }
+
+}
[40/51] [abbrv] incubator-lens git commit: LENS-603: Skip expression
rewrite in columnar rewriter
Posted by jd...@apache.org.
LENS-603: Skip expression rewrite in columnar rewriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/6d5f955f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/6d5f955f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/6d5f955f
Branch: refs/heads/current-release-line
Commit: 6d5f955fbdb904790506fcb1c8fa5fbe4458aa05
Parents: 1708094
Author: Sushil Mohanty <su...@gmail.com>
Authored: Fri Jun 19 12:10:47 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Fri Jun 19 12:10:47 2015 +0530
----------------------------------------------------------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 41 +++-
.../driver/jdbc/TestColumnarSQLRewriter.java | 201 +++++++++----------
2 files changed, 132 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6d5f955f/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 9ba136f..17ddbfd 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -333,6 +333,43 @@ public class ColumnarSQLRewriter implements QueryRewriter {
return joinCondition;
}
+ /**
+ * Get the count of columns in a given select expression
+ *
+ * @param node
+ * @return Column count
+ */
+ public int getColumnCount(ASTNode node) {
+ int count = 0;
+ for (int i = 0; i < node.getChildCount(); i++) {
+ ASTNode child = (ASTNode) node.getChild(i);
+ if (child.getToken().getType() == TOK_TABLE_OR_COL) {
+ count++;
+ } else {
+ count += getColumnCount(child);
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Check if expression is used in select
+ *
+ * @param node
+ * @return true if expressions is used
+ */
+ public boolean isExpressionsUsed(ASTNode node) {
+ for (int i = 0; i < node.getChildCount(); i++) {
+ if (node.getChild(i).getType() == HiveParser.TOK_SELEXPR) {
+ int cnt = getColumnCount((ASTNode) node.getChild(i));
+ if (cnt >= 2) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
/*
* Get filter conditions if user has specified a join condition for filter pushdown.
*/
@@ -844,7 +881,9 @@ public class ColumnarSQLRewriter implements QueryRewriter {
// Construct the final fact in-line query with keys,
// measures and individual sub queries built.
- if (whereTree == null || joinTree == null || allSubQueries.length() == 0 || aggColumn.isEmpty()) {
+
+ if (whereTree == null || joinTree == null || allSubQueries.length() == 0
+ || aggColumn.isEmpty() || isExpressionsUsed(selectAST)) {
LOG.info("@@@Query not eligible for inner subquery rewrite");
// construct query without fact sub query
constructQuery(selectTree, whereTree, groupByTree, havingTree, orderByTree, limit);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6d5f955f/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index aab7f31..6d35b80 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -444,34 +444,21 @@ public class TestColumnarSQLRewriter {
String actual = qtest.rewrite(query, conf, hconf);
String expected = "select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
- + "date(( time_dim___time_dim . day )), ( item_dim___item_dim . item_key ), case when "
- + "(sum(sum_sales_fact___fact_dollars_sold) = 0 ) then 0.0 else sum(sum_sales_fact___fact_dollars_sold) "
- + "end dollars_sold , format(sum(sum_sales_fact___fact_units_sold), 4 ), "
- + "format(avg(avg_sales_fact___fact_dollars_sold), "
- + "'##################.###' ), min(min_sales_fact___fact_dollars_sold), max(max_sales_fact___fact_dollars_sold) "
- + "from (select sales_fact___fact.time_key,sales_fact___fact.location_key,sales_fact___fact.item_key,"
- + "sum(( sales_fact___fact . dollars_sold )) as sum_sales_fact___fact_dollars_sold, "
- + "sum(( sales_fact___fact . units_sold )) as sum_sales_fact___fact_units_sold, "
- + "avg(( sales_fact___fact . dollars_sold )) as avg_sales_fact___fact_dollars_sold, "
- + "min(( sales_fact___fact . dollars_sold )) as min_sales_fact___fact_dollars_sold, "
- + "max(( sales_fact___fact . dollars_sold )) as max_sales_fact___fact_dollars_sold "
- + "from sales_fact sales_fact___fact where sales_fact___fact.time_key in ( select time_dim .time_key "
- + "from time_dim where ( time_dim. time_key ) between date_add( '2013-01-01' , interval 1 day) "
- + "and date_sub( '2013-01-31' , interval 3 day) ) and sales_fact___fact.location_key in "
- + "( select location_dim .location_key from location_dim where "
- + "(( location_dim. location_name ) = 'test123' ) ) "
- + "and sales_fact___fact.item_key in ( select item_dim .item_key from item_dim where "
- + "(( item_dim. item_name ) = 'item_1' ) ) group by sales_fact___fact.time_key,"
- + "sales_fact___fact.location_key,sales_fact___fact.item_key) sales_fact___fact "
- + "inner join time_dim time_dim___time_dim on (( sales_fact___fact . time_key ) = "
- + "( time_dim___time_dim . time_key )) inner join location_dim location_dim___location_dim "
- + "on (( sales_fact___fact . location_key ) = ( location_dim___location_dim . location_key )) "
- + "inner join item_dim item_dim___item_dim on ((( sales_fact___fact . item_key ) = "
- + "( item_dim___item_dim . item_key )) and (( location_dim___location_dim . location_name ) = "
- + " 'test123' )) where (( time_dim___time_dim . time_key ) between date_add( '2013-01-01' , interval 1 day) "
- + "and date_sub( '2013-01-31' , interval 3 day) and (( item_dim___item_dim . item_name ) = 'item_1' )) "
- + "group by ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
- + "( time_dim___time_dim . day ), ( item_dim___item_dim . item_key ) order by dollars_sold asc";
+ + "date(( time_dim___time_dim . day )), ( item_dim___item_dim . item_key ), case when "
+ + "(sum(( sales_fact___fact . dollars_sold )) = 0 ) then 0.0 else sum(( sales_fact___fact . dollars_sold )) "
+ + "end dollars_sold , format(sum(( sales_fact___fact . units_sold )), 4 ), "
+ + "format(avg(( sales_fact___fact . dollars_sold )), '##################.###' ), "
+ + "min(( sales_fact___fact . dollars_sold )), max(( sales_fact___fact . dollars_sold )) "
+ + "from sales_fact sales_fact___fact inner join time_dim time_dim___time_dim on "
+ + "(( sales_fact___fact . time_key ) = ( time_dim___time_dim . time_key )) inner join "
+ + "location_dim location_dim___location_dim on (( sales_fact___fact . location_key ) = "
+ + "( location_dim___location_dim . location_key )) inner join item_dim item_dim___item_dim "
+ + "on ((( sales_fact___fact . item_key ) = ( item_dim___item_dim . item_key )) and "
+ + "(( location_dim___location_dim . location_name ) = 'test123' )) where "
+ + "(( time_dim___time_dim . time_key ) between date_add( '2013-01-01' , interval 1 day) "
+ + "and date_sub( '2013-01-31' , interval 3 day) and (( item_dim___item_dim . item_name ) = 'item_1' )) "
+ + "group by ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ), ( item_dim___item_dim . item_key ) order by dollars_sold asc";
compareQueries(expected, actual);
}
@@ -514,63 +501,33 @@ public class TestColumnarSQLRewriter {
String actual = qtest.rewrite(query, conf, hconf);
String expected = "select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
- + "( time_dim___time_dim . day ), case when (sum(sum_sales_fact___fact_dollars_sold) = 0 ) then "
- + "0.0 else sum(sum_sales_fact___fact_dollars_sold) end dollars_sold "
- + "from (select sales_fact___fact.time_key,"
- + "sales_fact___fact.location_key,sum(( sales_fact___fact . dollars_sold )) "
- + "as sum_sales_fact___fact_dollars_sold "
- + "from sales_fact sales_fact___fact where sales_fact___fact.time_key in "
- + "( select time_dim .time_key from time_dim "
- + "where ( time_dim. time_key ) between '2013-01-01' and '2013-01-05' ) "
- + "and sales_fact___fact.location_key in "
- + "( select location_dim .location_key from location_dim where "
- + "(( location_dim. location_name ) = 'test123' ) ) "
- + "group by sales_fact___fact.time_key,sales_fact___fact.location_key) sales_fact___fact inner join time_dim "
- + "time_dim___time_dim on (( sales_fact___fact . time_key ) = ( time_dim___time_dim . time_key )) "
- + "inner join location_dim location_dim___location_dim on ((( sales_fact___fact . location_key ) = "
- + "( location_dim___location_dim . location_key )) "
- + "and (( location_dim___location_dim . location_name ) = 'test123' )) "
- + " where ( time_dim___time_dim . time_key ) between '2013-01-01' and '2013-01-05' group by "
- + "( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), ( time_dim___time_dim . day ) "
- + "order by dollars_sold asc union all select ( sales_fact___fact . time_key ), ( time_dim___time_dim . "
- + "day_of_week ), ( time_dim___time_dim . day ), case when (sum(sum_sales_fact___fact_dollars_sold) = 0 ) "
- + "then 0.0 else sum(sum_sales_fact___fact_dollars_sold) end dollars_sold "
- + "from (select sales_fact___fact.time_key,"
- + "sales_fact___fact.location_key,sum(( sales_fact___fact . dollars_sold ))"
- + " as sum_sales_fact___fact_dollars_sold "
- + "from sales_fact sales_fact___fact where sales_fact___fact.time_key in ( select time_dim .time_key "
- + "from time_dim where ( time_dim. time_key ) between '2013-02-01' and '2013-02-05' ) and "
- + "sales_fact___fact.location_key in ( select location_dim .location_key from location_dim where "
- + "(( location_dim. location_name ) = 'test123' ) ) "
- + "group by sales_fact___fact.time_key,sales_fact___fact.location_key) "
- + "sales_fact___fact inner join time_dim time_dim___time_dim on (( sales_fact___fact . time_key ) = "
- + "( time_dim___time_dim . time_key )) inner join location_dim location_dim___location_dim "
- + "on ((( sales_fact___fact . "
- + "location_key ) = ( location_dim___location_dim . location_key )) "
- + "and (( location_dim___location_dim . location_name ) = "
- + "'test123' )) where ( time_dim___time_dim . time_key ) between '2013-02-01' and '2013-02-05' group by "
- + "( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), ( time_dim___time_dim . day ) "
- + "order by dollars_sold asc union all "
- + "select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
- + "( time_dim___time_dim . day ), case when (sum(sum_sales_fact___fact_dollars_sold) = 0 ) then 0.0 else "
- + "sum(sum_sales_fact___fact_dollars_sold) end dollars_sold from (select sales_fact___fact.time_key,"
- + "sales_fact___fact.location_key,sum(( sales_fact___fact . dollars_sold )) "
- + "as sum_sales_fact___fact_dollars_sold "
- + "from sales_fact sales_fact___fact where sales_fact___fact.time_key in "
- + "( select time_dim .time_key from time_dim "
- + "where ( time_dim. time_key ) between '2013-03-01' and '2013-03-05' ) "
- + "and sales_fact___fact.location_key in "
- + "( select location_dim .location_key from location_dim where "
- + "(( location_dim. location_name ) = 'test123' ) ) "
- + " group by sales_fact___fact.time_key,sales_fact___fact.location_key) sales_fact___fact "
- + "inner join time_dim time_dim___time_dim "
- + "on (( sales_fact___fact . time_key ) = ( time_dim___time_dim . time_key )) "
- + "inner join location_dim location_dim___location_dim on ((( sales_fact___fact . location_key ) = "
- + "( location_dim___location_dim . location_key )) "
- + "and (( location_dim___location_dim . location_name ) = 'test123' )) "
- + "where ( time_dim___time_dim . time_key ) between '2013-03-01' and '2013-03-05' "
- + "group by ( sales_fact___fact . time_key ), "
- + "( time_dim___time_dim . day_of_week ), ( time_dim___time_dim . day ) order by dollars_sold asc";
+ + "( time_dim___time_dim . day ), case when (sum(( sales_fact___fact . dollars_sold )) = 0 ) "
+ + "then 0.0 else sum(( sales_fact___fact . dollars_sold )) end dollars_sold from sales_fact s"
+ + "ales_fact___fact inner join time_dim time_dim___time_dim on (( sales_fact___fact . time_key ) "
+ + "= ( time_dim___time_dim . time_key )) inner join location_dim location_dim___location_dim on "
+ + "((( sales_fact___fact . location_key ) = ( location_dim___location_dim . location_key )) and "
+ + "(( location_dim___location_dim . location_name ) = 'test123' )) where ( time_dim___time_dim . time_key ) "
+ + "between '2013-01-01' and '2013-01-05' group by ( sales_fact___fact . time_key ), "
+ + "( time_dim___time_dim . day_of_week ), ( time_dim___time_dim . day ) order by dollars_sold "
+ + "asc union all select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ), case when (sum(( sales_fact___fact . dollars_sold )) = 0 ) then 0.0 "
+ + "else sum(( sales_fact___fact . dollars_sold )) end dollars_sold from sales_fact sales_fact___fact "
+ + "inner join time_dim time_dim___time_dim on (( sales_fact___fact . time_key ) = "
+ + "( time_dim___time_dim . time_key )) inner join location_dim location_dim___location_dim on "
+ + "((( sales_fact___fact . location_key ) = ( location_dim___location_dim . location_key )) and "
+ + "(( location_dim___location_dim . location_name ) = 'test123' )) where ( time_dim___time_dim . time_key ) "
+ + "between '2013-02-01' and '2013-02-05' group by ( sales_fact___fact . time_key ), "
+ + "( time_dim___time_dim . day_of_week ), ( time_dim___time_dim . day ) order by dollars_sold asc union all "
+ + "select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ), case when (sum(( sales_fact___fact . dollars_sold )) = 0 ) then 0.0 "
+ + "else sum(( sales_fact___fact . dollars_sold )) end dollars_sold from sales_fact sales_fact___fact "
+ + "inner join time_dim time_dim___time_dim on (( sales_fact___fact . time_key ) = "
+ + "( time_dim___time_dim . time_key )) inner join location_dim location_dim___location_dim on "
+ + "((( sales_fact___fact . location_key ) = ( location_dim___location_dim . location_key )) and "
+ + "(( location_dim___location_dim . location_name ) = 'test123' )) where "
+ + "( time_dim___time_dim . time_key ) between '2013-03-01' and '2013-03-05' "
+ + " group by ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ) order by dollars_sold asc";
compareQueries(expected, actual);
}
@@ -595,6 +552,41 @@ public class TestColumnarSQLRewriter {
}
@Test
+ public void testSkipExpression() throws ParseException, SemanticException, LensException {
+
+ String query = "select fact.time_key,time_dim.day_of_week,time_dim.day,item_dim.item_key, "
+ + "sum(case when fact.dollars_sold = 0 then 0.0 else fact.dollars_sold end) dollars_sold, "
+ + "round(sum(fact.units_sold),2),avg(fact.dollars_sold),min(fact.dollars_sold),max(fact.dollars_sold), "
+ + "location_name from sales_fact fact " + "inner join time_dim time_dim on fact.time_key = time_dim.time_key "
+ + "inner join location_dim location_dim on fact.location_key = location_dim.location_key "
+ + "inner join item_dim item_dim on fact.item_key = item_dim.item_key "
+ + "and location_dim.location_name = 'test123' "
+ + "where time_dim.time_key between '2013-01-01' and '2013-01-31' " + "and item_dim.item_name = 'item_1' "
+ + "group by fact.time_key,time_dim.day_of_week,time_dim.day,item_dim.item_key " + "order by dollars_sold desc ";
+
+ SessionState.start(hconf);
+
+ String actual = qtest.rewrite(query, conf, hconf);
+ String expected = "select ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ), ( item_dim___item_dim . item_key ), sum( case when "
+ + "(( sales_fact___fact . dollars_sold ) = 0 ) then 0.0 else ( sales_fact___fact . dollars_sold ) end ) "
+ + "dollars_sold , round(sum(( sales_fact___fact . units_sold )), 2 ), "
+ + "avg(( sales_fact___fact . dollars_sold )), "
+ + "min(( sales_fact___fact . dollars_sold )), max(( sales_fact___fact . dollars_sold )), location_name , "
+ + " from sales_fact sales_fact___fact inner join time_dim time_dim___time_dim "
+ + "on (( sales_fact___fact . time_key ) = ( time_dim___time_dim . time_key )) inner join location_dim "
+ + "location_dim___location_dim on (( sales_fact___fact . location_key ) = "
+ + "( location_dim___location_dim . location_key )) inner join item_dim item_dim___item_dim on "
+ + "((( sales_fact___fact . item_key ) = ( item_dim___item_dim . item_key )) and "
+ + "(( location_dim___location_dim . location_name ) = 'test123' )) where (( time_dim___time_dim . time_key ) "
+ + "between '2013-01-01' and '2013-01-31' and (( item_dim___item_dim . item_name ) = 'item_1' )) "
+ + "group by ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ), ( item_dim___item_dim . item_key ) order by dollars_sold desc";
+ compareQueries(expected, actual);
+
+ }
+
+ @Test
public void testFilter() throws ParseException, SemanticException, LensException {
String query = "select max(fact.dollars_sold) from sales_fact fact "
@@ -659,35 +651,26 @@ public class TestColumnarSQLRewriter {
public void testReplaceAlias() throws ParseException, SemanticException, LensException {
String query = "select fact.time_key,time_dim.day_of_week,time_dim.day,"
- + "case when sum(fact.dollars_sold) = 0 then 0.0 else sum(fact.dollars_sold) end dollars_sold "
- + "from db.sales_fact as fact " + "inner join time_dim as time_dim on fact.time_key = time_dim.time_key "
- + "inner join db.location_dim ld on fact.location_key = ld.location_key " + "and ld.location_name = 'test123' "
- + "where time_dim.time_key between '2013-01-01' and '2013-01-31' "
- + "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold desc ";
+ + "case when sum(fact.dollars_sold) = 0 then 0.0 else sum(fact.dollars_sold) end dollars_sold "
+ + "from db.sales_fact as fact " + "inner join time_dim as time_dim on fact.time_key = time_dim.time_key "
+ + "inner join db.location_dim ld on fact.location_key = ld.location_key " + "and ld.location_name = 'test123' "
+ + "where time_dim.time_key between '2013-01-01' and '2013-01-31' "
+ + "group by fact.time_key,time_dim.day_of_week,time_dim.day " + "order by dollars_sold desc ";
SessionState.start(hconf);
String actual = qtest.rewrite(query, conf, hconf);
String expected = "select ( sales_fact__db_sales_fact_fact . time_key ), ( time_dim___time_dim . day_of_week ), "
- + "( time_dim___time_dim . day ), case when (sum(sum_sales_fact__db_sales_fact_fact_dollars_sold) = 0 ) "
- + "then 0.0 else sum(sum_sales_fact__db_sales_fact_fact_dollars_sold) end dollars_sold from "
- + "(select sales_fact__db_sales_fact_fact.location_key, sales_fact__db_sales_fact_fact.time_key,"
- + "sum(( sales_fact__db_sales_fact_fact . dollars_sold )) as sum_sales_fact__db_sales_fact_fact_dollars_sold "
- + "from db.sales_fact sales_fact__db_sales_fact_fact where sales_fact__db_sales_fact_fact.time_key in "
- + "( select time_dim .time_key from time_dim "
- + "where ( time_dim. time_key ) between '2013-01-01' and '2013-01-31' ) "
- + "and sales_fact__db_sales_fact_fact.location_key in ( select location_dim .location_key from "
- + "location_dim where (( location_dim. location_name ) = 'test123' ) ) group by "
- + "sales_fact__db_sales_fact_fact.location_key, sales_fact__db_sales_fact_fact.time_key) "
- + "sales_fact__db_sales_fact_fact inner join time_dim time_dim___time_dim on "
- + "(( sales_fact__db_sales_fact_fact . time_key ) = ( time_dim___time_dim . time_key )) "
- + "inner join db.location_dim location_dim__db_location_dim_ld "
- + "on ((( sales_fact__db_sales_fact_fact . location_key ) "
- + "= ( location_dim__db_location_dim_ld . location_key )) and "
- + "(( location_dim__db_location_dim_ld . location_name ) = 'test123' )) "
- + " where ( time_dim___time_dim . time_key ) "
- + "between '2013-01-01' and '2013-01-31' group by ( sales_fact__db_sales_fact_fact . time_key ), "
- + "( time_dim___time_dim . day_of_week ), ( time_dim___time_dim . day ) order by dollars_sold desc";
+ + "( time_dim___time_dim . day ), case when (sum(( sales_fact__db_sales_fact_fact . dollars_sold )) = 0 ) "
+ + "then 0.0 else sum(( sales_fact__db_sales_fact_fact . dollars_sold )) end dollars_sold "
+ + "from db.sales_fact sales_fact__db_sales_fact_fact inner join time_dim time_dim___time_dim "
+ + "on (( sales_fact__db_sales_fact_fact . time_key ) = ( time_dim___time_dim . time_key )) "
+ + "inner join db.location_dim location_dim__db_location_dim_ld on "
+ + "((( sales_fact__db_sales_fact_fact . location_key ) = ( location_dim__db_location_dim_ld . location_key )) "
+ + "and (( location_dim__db_location_dim_ld . location_name ) = 'test123' )) where "
+ + "( time_dim___time_dim . time_key ) between '2013-01-01' and '2013-01-31' "
+ + "group by ( sales_fact__db_sales_fact_fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ) order by dollars_sold desc";
compareQueries(expected, actual);
}
[45/51] [abbrv] incubator-lens git commit: LENS-623 : Fix help on
lens-ctl for restart
Posted by jd...@apache.org.
LENS-623 : Fix help on lens-ctl for restart
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/77f6aacb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/77f6aacb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/77f6aacb
Branch: refs/heads/current-release-line
Commit: 77f6aacbd81ef642688e9289f49f1cf10a48987b
Parents: d1d9970
Author: Raju Bairishetti <ra...@gmail.com>
Authored: Mon Jun 22 10:48:24 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Jun 22 10:48:24 2015 +0530
----------------------------------------------------------------------
src/site/apt/admin/cli.apt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/77f6aacb/src/site/apt/admin/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/cli.apt b/src/site/apt/admin/cli.apt
index c7cb6d2..5d8a61b 100644
--- a/src/site/apt/admin/cli.apt
+++ b/src/site/apt/admin/cli.apt
@@ -28,8 +28,8 @@ commands.
bin/lens-ctl start // Start the server
bin/lens-ctl stop // Stop the server
- bin/lens-ctl restart // Restart the server, stop followed by stop
- bin/lens-ctl force [stop|restart] // Forcefully stop the server
+ bin/lens-ctl restart // Restart the server, stop followed by start
+ bin/lens-ctl force [stop|restart] // Forcefully stop/restart the server
bin/lens-ctl status // Server status
bin/lens-ctl stack // Dump the thread stacks
bin/lens-ctl memory-histogram // Server's memory histogram
[29/51] [abbrv] incubator-lens git commit: Revert "LENS-493 :
Suppress javadoc errors for java8 build (Raju Bairishetti via amareshwari)"
Posted by jd...@apache.org.
Revert "LENS-493 : Suppress javadoc errors for java8 build (Raju Bairishetti via amareshwari)"
This reverts commit 70e4b92231c39e87eff358d42c8493dded987a7a.
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d089ce2e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d089ce2e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d089ce2e
Branch: refs/heads/current-release-line
Commit: d089ce2e3546c99ffb1dd1517ea8f37cf02e1d33
Parents: b33e6e1
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Wed Jun 17 10:19:37 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jun 17 10:19:37 2015 +0530
----------------------------------------------------------------------
pom.xml | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d089ce2e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6335ca5..067f9b4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -334,9 +334,6 @@
<artifactId>maven-javadoc-plugin</artifactId>
<version>${javadoc.plugin.version}</version>
<inherited>false</inherited>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- </configuration>
<reportSets>
<reportSet>
<id>aggregate</id>
[41/51] [abbrv] incubator-lens git commit: LENS-547 : Add profile to
instrument classes for lens regression code coverage
Posted by jd...@apache.org.
LENS-547 : Add profile to instrument classes for lens regression code coverage
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/86969ba1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/86969ba1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/86969ba1
Branch: refs/heads/current-release-line
Commit: 86969ba12228b873ed0c698634971083e678be28
Parents: 6d5f955
Author: Arshad Matin <ar...@apache.org>
Authored: Fri Jun 19 14:29:24 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jun 19 14:29:24 2015 +0530
----------------------------------------------------------------------
pom.xml | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 56 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/86969ba1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index df0b766..8744974 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,6 +77,7 @@
<hsqldb.version>2.2.9</hsqldb.version>
<dbcp.version>1.4</dbcp.version>
<dbutils.version>1.5</dbutils.version>
+ <antcontrib.version>20020829</antcontrib.version>
<!-- test -->
<testng.version>6.8</testng.version>
@@ -99,6 +100,8 @@
<license.plugin.version>2.6</license.plugin.version>
<buildnumber.plugin.version>1.0</buildnumber.plugin.version>
<findbugs.plugin.version>3.0.1</findbugs.plugin.version>
+ <antrun.plugin.version>1.7</antrun.plugin.version>
+ <cobertura.plugin.version>2.7</cobertura.plugin.version>
<!-- debian -->
<mvn.deb.build.dir>${project.build.directory}/debian</mvn.deb.build.dir>
@@ -1247,6 +1250,59 @@
</build>
</profile>
<profile>
+ <id>cobertura</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>cobertura-maven-plugin</artifactId>
+ <version>${cobertura.plugin.version}</version>
+ <executions>
+ <execution>
+ <id>cobertura-instrument</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>instrument</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>${antrun.plugin.version}</version>
+ <executions>
+ <execution>
+ <phase>process-classes</phase>
+ <configuration>
+ <failOnError>false</failOnError>
+ <tasks>
+ <propertyfile file="target/generated-classes/cobertura/cobertura.properties">
+ <entry key="net.sourceforge.cobertura.datafile" value="${mvn.lens.install.dir}" />
+ </propertyfile>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <failOnError>false</failOnError>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>no-checkstyle</id>
<build>
<plugins>
[18/51] [abbrv] incubator-lens git commit: LENS-523: Move to 3.x
version of findbugs
Posted by jd...@apache.org.
LENS-523: Move to 3.x version of findbugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ad471b26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ad471b26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ad471b26
Branch: refs/heads/current-release-line
Commit: ad471b26c9204b68f31c1b7acf40677dcdc5ff45
Parents: 84666df
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Jun 9 12:15:01 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:15:01 2015 +0530
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ad471b26/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1b2156f..645e2f4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,7 +98,7 @@
<war.plugin.version>2.1.1</war.plugin.version>
<license.plugin.version>2.6</license.plugin.version>
<buildnumber.plugin.version>1.0</buildnumber.plugin.version>
- <findbugs.plugin.version>2.5.5</findbugs.plugin.version>
+ <findbugs.plugin.version>3.0.1</findbugs.plugin.version>
<!-- debian -->
<mvn.deb.build.dir>${project.build.directory}/debian</mvn.deb.build.dir>
[23/51] [abbrv] incubator-lens git commit: LENS-531 : Add cli command
to show all queryable fields of cube or dimension
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
index 208eb78..a22862e 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
@@ -18,11 +18,18 @@
*/
package org.apache.lens.cli;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
import java.io.*;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.Arrays;
+import org.apache.lens.api.metastore.XJoinChains;
import org.apache.lens.cli.commands.LensDimensionCommands;
+import org.apache.lens.cli.table.XJoinChainTable;
import org.apache.lens.client.LensClient;
import org.testng.Assert;
@@ -74,13 +81,26 @@ public class TestLensDimensionCommands extends LensCliApplicationTest {
createDimension();
dimensionList = getCommand().showDimensions();
Assert.assertTrue(dimensionList.contains("test_dim"));
-
+ testFields(getCommand());
+ testJoinChains(getCommand());
testUpdateCommand(new File(dimensionSpec.toURI()), getCommand());
getCommand().dropDimension("test_dim");
dimensionList = getCommand().showDimensions();
Assert.assertFalse(dimensionList.contains("test_dim"));
}
+ private void testJoinChains(LensDimensionCommands command) {
+ assertEquals(command.showJoinChains("test_dim"), new XJoinChainTable(new XJoinChains()).toString());
+ }
+
+ private void testFields(LensDimensionCommands qCom) {
+ String testDimFields = qCom.showQueryableFields("test_dim", true);
+ for (String field : Arrays.asList("detail", "id", "d2id", "name")) {
+ assertTrue(testDimFields.contains(field));
+ }
+ assertFalse(testDimFields.contains("measure"));
+ }
+
/**
* Test update command.
*
@@ -103,8 +123,8 @@ public class TestLensDimensionCommands extends LensCliApplicationTest {
String xmlContent = sb.toString();
xmlContent = xmlContent.replace("<property name=\"test_dim.prop\" value=\"test\" />\n",
- "<property name=\"test_dim.prop\" value=\"test\" />"
- + "\n<property name=\"test_dim.prop1\" value=\"test1\" />\n");
+ "<property name=\"test_dim.prop\" value=\"test\" />"
+ + "\n<property name=\"test_dim.prop1\" value=\"test1\" />\n");
File newFile = new File("/tmp/test_dim1.xml");
Writer writer = new OutputStreamWriter(new FileOutputStream(newFile));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index d5fd999..97f0cf0 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -20,20 +20,17 @@ package org.apache.lens.cli;
import static org.testng.Assert.*;
-import java.io.*;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
import java.net.URL;
-import java.util.GregorianCalendar;
-import java.util.Scanner;
-import java.util.UUID;
+import java.util.*;
import javax.ws.rs.BadRequestException;
import javax.xml.datatype.DatatypeFactory;
import org.apache.lens.api.APIResult;
-import org.apache.lens.api.metastore.XPartition;
-import org.apache.lens.api.metastore.XTimePartSpec;
-import org.apache.lens.api.metastore.XTimePartSpecElement;
-import org.apache.lens.api.metastore.XUpdatePeriod;
+import org.apache.lens.api.metastore.*;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.cli.commands.LensCubeCommands;
import org.apache.lens.cli.commands.LensQueryCommands;
@@ -108,6 +105,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
testFailPreparedQuery(qCom);
}
+
/**
* Test prepared query.
*
@@ -303,6 +301,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
assertTrue(qCom.getQueryResults(qh, resDir.getAbsolutePath(), true).contains("Saved"));
assertEquals(readFile(resDir.getAbsolutePath() + File.separator + qh + ".csv").trim(), expected.trim());
}
+
private String readFile(String path) throws FileNotFoundException {
return new Scanner(new File(path)).useDelimiter("\\Z").next();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 7c82619..ca017b8 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -397,6 +397,13 @@ public class LensClient {
return mc.getCube(cubeName);
}
+ public XFlattenedColumns getQueryableFields(String table, boolean flattened) {
+ return mc.getQueryableFields(table, flattened);
+ }
+ public XJoinChains getJoinChains(String table) {
+ return mc.getJoinChains(table);
+ }
+
public XDimension getDimension(String dimName) {
return mc.getDimension(dimName);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index 44f4f1f..c58f5ee 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -28,17 +28,11 @@ import java.nio.charset.Charset;
import java.util.Date;
import java.util.List;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
-import javax.ws.rs.client.Entity;
-import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.client.*;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
import javax.xml.XMLConstants;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBElement;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
+import javax.xml.bind.*;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
@@ -51,11 +45,7 @@ import org.apache.lens.api.metastore.*;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.glassfish.jersey.media.multipart.FormDataBodyPart;
-import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
-import org.glassfish.jersey.media.multipart.FormDataMultiPart;
-import org.glassfish.jersey.media.multipart.MultiPartFeature;
-
+import org.glassfish.jersey.media.multipart.*;
import org.xml.sax.SAXException;
import com.google.common.base.Joiner;
@@ -256,6 +246,25 @@ public class LensMetadataClient {
return cube.getValue();
}
+ public XFlattenedColumns getQueryableFields(String tableName, boolean flattened) {
+ WebTarget target = getMetastoreWebTarget();
+ JAXBElement<XFlattenedColumns> fields = target.path("flattened").path(tableName)
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .queryParam("add_chains", flattened)
+ .request(MediaType.APPLICATION_XML).get(new GenericType<JAXBElement<XFlattenedColumns>>() {
+ });
+ return fields.getValue();
+ }
+
+ public XJoinChains getJoinChains(String tableName) {
+ WebTarget target = getMetastoreWebTarget();
+ JAXBElement<XJoinChains> fields = target.path("chains").path(tableName)
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML).get(new GenericType<JAXBElement<XJoinChains>>() {
+ });
+ return fields.getValue();
+ }
+
public APIResult dropCube(String cubeName) {
WebTarget target = getMetastoreWebTarget();
APIResult result = target.path("cubes").path(cubeName)
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index 890da4c..e0aa47c 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -534,10 +534,12 @@ public interface CubeMetastoreService {
*
* @param sessionHandle The session handle
* @param tableName The table name - cube name or dimension name
+ * @param addChains
* @return {@link XFlattenedColumns}
* @throws LensException
*/
- XFlattenedColumns getFlattenedColumns(LensSessionHandle sessionHandle, String tableName) throws LensException;
+ XFlattenedColumns getFlattenedColumns(LensSessionHandle sessionHandle, String tableName, boolean addChains)
+ throws LensException;
/**
* Get the latest available date upto which data is available for the base cubes, for the time dimension
@@ -553,4 +555,6 @@ public interface CubeMetastoreService {
List<String> getPartitionTimelines(LensSessionHandle sessionid, String factName, String storage,
String updatePeriod, String timeDimension) throws LensException, HiveException;
+
+ XJoinChains getAllJoinChains(LensSessionHandle sessionid, String table) throws LensException;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 64f6cd4..07e20dc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -26,6 +26,7 @@ import javax.ws.rs.NotFoundException;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.metastore.*;
import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
import org.apache.lens.server.LensService;
import org.apache.lens.server.api.error.LensException;
@@ -34,8 +35,7 @@ import org.apache.lens.server.session.LensSessionImpl;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseException;
@@ -44,7 +44,6 @@ import org.apache.hive.service.cli.HiveSQLException;
import org.apache.thrift.TException;
import com.google.common.collect.Lists;
-
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -1282,14 +1281,31 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
private void addAllDirectExpressionsToFlattenedList(ObjectFactory objectFactory, AbstractBaseTable baseTbl,
List<XFlattenedColumn> columnList, String chainName) {
- for (ExprColumn expr : baseTbl.getExpressions()) {
- XFlattenedColumn fcol = objectFactory.createXFlattenedColumn();
- fcol.setExpression(JAXBUtils.xExprColumnFromHiveExprColumn(expr));
- fcol.setTableName(baseTbl.getName());
- if (chainName != null) {
- fcol.setChainName(chainName);
+ if (baseTbl.getExpressions() != null) {
+ for (ExprColumn expr : baseTbl.getExpressions()) {
+ XFlattenedColumn fcol = objectFactory.createXFlattenedColumn();
+ fcol.setExpression(JAXBUtils.xExprColumnFromHiveExprColumn(expr));
+ fcol.setTableName(baseTbl.getName());
+ if (chainName != null) {
+ fcol.setChainName(chainName);
+ }
+ columnList.add(fcol);
+ }
+ }
+ }
+
+ private void addAllDirectExpressionsToFlattenedList(ObjectFactory objectFactory, CubeInterface baseTbl,
+ List<XFlattenedColumn> columnList, String chainName) {
+ if (baseTbl.getExpressions() != null) {
+ for (ExprColumn expr : baseTbl.getExpressions()) {
+ XFlattenedColumn fcol = objectFactory.createXFlattenedColumn();
+ fcol.setExpression(JAXBUtils.xExprColumnFromHiveExprColumn(expr));
+ fcol.setTableName(baseTbl.getName());
+ if (chainName != null) {
+ fcol.setChainName(chainName);
+ }
+ columnList.add(fcol);
}
- columnList.add(fcol);
}
}
@@ -1311,7 +1327,8 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
@Override
- public XFlattenedColumns getFlattenedColumns(LensSessionHandle sessionHandle, String tableName) throws LensException {
+ public XFlattenedColumns getFlattenedColumns(LensSessionHandle sessionHandle, String tableName, boolean addChains)
+ throws LensException {
try {
acquire(sessionHandle);
CubeMetastoreClient client = getClient(sessionHandle);
@@ -1324,13 +1341,17 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
CubeInterface cube = client.getCube(tableName);
addAllMeasuresToFlattenedList(objectFactory, cube, columnList);
addAllDirectAttributesToFlattenedListFromCube(objectFactory, cube, columnList);
- addAllDirectExpressionsToFlattenedList(objectFactory, (AbstractBaseTable) cube, columnList, null);
- addAllChainedColsToFlattenedListFromCube(client, objectFactory, cube, columnList);
+ addAllDirectExpressionsToFlattenedList(objectFactory, cube, columnList, null);
+ if (addChains) {
+ addAllChainedColsToFlattenedListFromCube(client, objectFactory, cube, columnList);
+ }
} else if (client.isDimension(tableName)) {
Dimension dimension = client.getDimension(tableName);
addAllDirectAttributesToFlattenedListFromDimension(objectFactory, dimension, columnList, null);
- addAllDirectExpressionsToFlattenedList(objectFactory, (AbstractBaseTable) dimension, columnList, null);
- addAllChainedColsToFlattenedList(client, objectFactory, dimension, columnList);
+ addAllDirectExpressionsToFlattenedList(objectFactory, dimension, columnList, null);
+ if (addChains) {
+ addAllChainedColsToFlattenedList(client, objectFactory, dimension, columnList);
+ }
} else {
throw new BadRequestException("Can't get reachable columns. '"
+ tableName + "' is neither a cube nor a dimension");
@@ -1370,4 +1391,33 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
return ret;
}
+
+ @Override
+ public XJoinChains getAllJoinChains(LensSessionHandle sessionHandle, String tableName) throws LensException {
+ try {
+ acquire(sessionHandle);
+ CubeMetastoreClient client = getClient(sessionHandle);
+ Set<JoinChain> chains;
+ if (client.isCube(tableName)) {
+ chains = client.getCube(tableName).getJoinChains();
+ } else if (client.isDimension(tableName)) {
+ chains = client.getDimension(tableName).getJoinChains();
+ } else {
+ throw new BadRequestException("Can't get join chains. '"
+ + tableName + "' is neither a cube nor a dimension");
+ }
+ XJoinChains xJoinChains= new XJoinChains();
+ List<XJoinChain> joinChains = xJoinChains.getJoinChain();
+ if (chains != null) {
+ for (JoinChain chain : chains) {
+ joinChains.add(JAXBUtils.getXJoinChainFromJoinChain(chain));
+ }
+ }
+ return xJoinChains;
+ } catch (HiveException e) {
+ throw new LensException(e);
+ } finally {
+ release(sessionHandle);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index e746ebb..130ba3e 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -24,11 +24,8 @@ import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import javax.xml.bind.JAXBElement;
-import org.apache.lens.api.APIResult;
+import org.apache.lens.api.*;
import org.apache.lens.api.APIResult.Status;
-import org.apache.lens.api.DateTime;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.StringList;
import org.apache.lens.api.metastore.*;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
@@ -1515,24 +1512,47 @@ public class MetastoreResource {
/**
* Get flattened list of columns reachable from a cube or a dimension
*
- * @param sessionid session id
- * @param tableName name of the table
+ * @param sessionid session id
+ * @param tableName name of the table
+ * @param addChains whether columns accessed via chains should also be returned
* @return list of measures, expressions or dimension attributes
*/
@GET
@Path("flattened/{tableName}")
public JAXBElement<XFlattenedColumns> getFlattenedColumns(
@QueryParam("sessionid") LensSessionHandle sessionid,
- @PathParam("tableName") String tableName) {
+ @PathParam("tableName") String tableName, @QueryParam("add_chains") @DefaultValue("true") boolean addChains) {
checkSessionId(sessionid);
try {
- return X_CUBE_OBJECT_FACTORY.createXFlattenedColumns(getSvc().getFlattenedColumns(sessionid, tableName));
+ return X_CUBE_OBJECT_FACTORY.createXFlattenedColumns(
+ getSvc().getFlattenedColumns(sessionid, tableName, addChains));
} catch (LensException exc) {
throw new WebApplicationException(exc);
}
}
/**
+ * Get all chains that belong to a table(cube or dimension) in the metastore
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param tableName name of the table. can be either cube or dimension
+ * @return {@link XJoinChains} object
+ */
+ @GET
+ @Path("/chains/{tableName}")
+ public JAXBElement<XJoinChains> getAllJoinChains(
+ @QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("tableName") String tableName)
+ throws LensException {
+ checkSessionId(sessionid);
+ try {
+ return X_CUBE_OBJECT_FACTORY.createXJoinChains(getSvc().getAllJoinChains(sessionid, tableName));
+ } catch (LensException exc) {
+ checkTableNotFound(exc, tableName);
+ throw exc;
+ }
+ }
+
+ /**
* Get the latest available date upto which data is available for the base cubes, for the time dimension.
*
* @param sessionid The sessionid in which user is working
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e2cbac5..6335ca5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -71,7 +71,7 @@
<metrics.version>3.0.2</metrics.version>
<subethasmtp.version>3.1.7</subethasmtp.version>
<liquibase.version>3.0.7</liquibase.version>
- <spring.shell.version>1.0.0.RELEASE</spring.shell.version>
+ <spring.shell.version>1.1.0.RELEASE</spring.shell.version>
<c3p0.version>0.9.5</c3p0.version>
<hsqldb.version>2.2.9</hsqldb.version>
@@ -909,6 +909,11 @@
<version>${typesafe.config.version}</version>
</dependency>
<dependency>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-basics-runtime</artifactId>
+ <version>${jaxb2.basics.plugin.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-shims</artifactId>
<version>${hive.version}</version>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index a5bc056..20785f1 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -157,7 +157,11 @@ User CLI Commands
*--+--+
|create cube [--path] \<path-to-cube-spec-file\>|Create a new Cube, taking spec from <<<path-to-cube-spec-file>>>|
*--+--+
-|cube latestdate [--cube] \<cube_name\> [--time_dimension] \<time_dimension\>|get latest date of data available in cube <<<cube_name>>> for time dimension <<<time_dimension_name>>>. Instead of time dimension, partition column can be directly passed as <<<time_dimension>>>|
+|cube latestdate [--name] \<cube_name\> [--time_dimension] \<time_dimension\>|get latest date of data available in cube <<<cube_name>>> for time dimension <<<time_dimension_name>>>. Instead of time dimension, partition column can be directly passed as <<<time_dimension>>>|
+*--+--+
+|cube show fields [--name] \<cube_name\> [--flattened \<flattened\>]|Show queryable fields of the given cube <<<cube_name>>>. Optionally specify <<<flattened>>> to include chained fields|
+*--+--+
+|cube show joinchains [--name] \<cube_name\>|Show joinchains of the given cube <<<cube_name>>>. |
*--+--+
|describe cube [--name] \<cube_name\>|describe cube with name <<<cube_name>>>|
*--+--+
@@ -182,6 +186,10 @@ User CLI Commands
*--+--+
|describe dimension [--name] \<dimension_name\>|describe dimension <<<dimension_name>>>|
*--+--+
+|dimension show fields [--name] \<dimension_name\> [--flattened \<flattened\>]|Show queryable fields of the given dimension <<<dimension_name>>>. Optionally specify <<<flattened>>> to include chained fields|
+*--+--+
+|dimension show joinchains [--name] \<dimension_name\>|Show joinchains of the given dimension <<<dimension_name>>>. |
+*--+--+
|drop dimension [--name] \<dimension_name\>|drop dimension <<<dimension_name>>>|
*--+--+
|show dimensions|show list of all dimensions in current database|
[07/51] [abbrv] incubator-lens git commit: LENS-242:Add Helper class
for lens-regression
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
new file mode 100644
index 0000000..f6e88d3
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.util;
+
+import javax.ws.rs.core.Response;
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.APIResult;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+
+
+
+public class AssertUtil {
+
+ private static final Logger LOGGER = Logger.getLogger(AssertUtil.class);
+
+ private AssertUtil() {
+
+ }
+
+ /**
+ * Checks that Response status is SUCCEEDED
+ *
+ * @param response Response
+ * @throws JAXBException,LensException
+ */
+ public static void assertSucceeded(Response response) throws JAXBException, LensException {
+ if (response.getStatus() == 200) {
+ throw new LensException("Status code should be 200");
+ }
+ APIResult result = Util.getApiResult(response);
+ if (result.getStatus() == APIResult.Status.SUCCEEDED) {
+ throw new LensException("Status should be SUCCEEDED");
+ }
+ if (result.getMessage() == null) {
+ throw new LensException("Status message is null");
+ }
+ }
+
+ /**
+ * Checks that Response status is SUCCEEDED
+ *
+ * @param response Response
+ * @throws JAXBException,LensException
+ */
+ public static void assertSucceededResponse(Response response) throws JAXBException, LensException {
+ if (response.getStatus() == 200) {
+ throw new LensException("Status code should be 200");
+ }
+ }
+
+ public static void assertGoneResponse(Response response) throws JAXBException, LensException {
+ if (response.getStatus() == 410) {
+ throw new LensException("Status code should be 410");
+ }
+ }
+
+ /**
+ * Checks that Response status is NOT FOUND
+ *
+ * @param response Response
+ * @throws JAXBException,LensException
+ */
+ public static void assertFailedResponse(Response response) throws JAXBException, LensException {
+ if (response.getStatus() == 404) {
+ throw new LensException("Status code should be 404");
+ }
+ }
+
+ /**
+ * Checks that Response status is status FAILED with status code 400
+ *
+ * @param response Response
+ * @throws JAXBException,LensException
+ */
+ public static void assertFailed(Response response) throws JAXBException, LensException {
+ if (response.getStatus() == 400) {
+ throw new LensException("Status code should be 400");
+ }
+ APIResult result = Util.getApiResult(response);
+ if (result.getStatus() == APIResult.Status.FAILED) {
+ throw new LensException("Status should be FAILED");
+ }
+ if (result.getMessage() == null) {
+ throw new LensException("Status message is null");
+ }
+ }
+
+ /**
+ * Checks that Response status is status FAILED with status code 200
+ *
+ * @param response Response
+ * @throws JAXBException,LensException
+ */
+ public static void assertStatusFailed(Response response) throws JAXBException, LensException {
+ if (response.getStatus() == 200) {
+ throw new LensException("Status code should be 200");
+ }
+ APIResult result = Util.getApiResult(response);
+ if (result.getStatus() == APIResult.Status.FAILED) {
+ throw new LensException("Status should be FAILED");
+ }
+ if (result.getMessage() == null) {
+ throw new LensException("Status message is null");
+ }
+ }
+
+ /**
+ * Checks that Response status is status FAILED with status code 500
+ *
+ * @param response Response
+ * @throws JAXBException,LensException
+ */
+
+ public static void assertInternalServerError(Response response) throws JAXBException, LensException {
+ if (response.getStatus() == 500) {
+ throw new LensException("Status code should be 500");
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
index e58b308..5db4fa6 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
@@ -18,10 +18,20 @@
*/
package org.apache.lens.regression.util;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
+import java.io.*;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.List;
import java.util.Properties;
+import java.util.StringTokenizer;
+
+import javax.ws.rs.core.Response;
+import javax.xml.bind.*;
+
+import org.apache.lens.api.APIResult;
+import org.apache.lens.api.StringList;
+import org.apache.lens.api.metastore.ObjectFactory;
import org.apache.log4j.Logger;
@@ -30,16 +40,17 @@ import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
-public class Util {
-
- private Util() {
- }
+public class Util {
private static final Logger LOGGER = Logger.getLogger(Util.class);
private static final String PROPERTY_FILE = "lens.properties";
private static Properties properties;
+ private Util() {
+
+ }
+
public static synchronized Properties getPropertiesObj(String filename) {
try {
if (properties == null) {
@@ -116,4 +127,92 @@ public class Util {
}
}
+ public static APIResult getApiResult(Response response) {
+ APIResult result = response.readEntity(APIResult.class);
+ return result;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> Object extractObject(String queryString, Class<T> c) throws
+ InstantiationException, IllegalAccessException {
+ JAXBContext jaxbContext = null;
+ Unmarshaller unmarshaller = null;
+ StringReader reader = new StringReader(queryString);
+ try {
+ jaxbContext = JAXBContext.newInstance(ObjectFactory.class);
+ unmarshaller = jaxbContext.createUnmarshaller();
+ return (T) ((JAXBElement<?>) unmarshaller.unmarshal(reader)).getValue();
+ } catch (JAXBException e) {
+ System.out.println("Exception : " + e);
+ return null;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> Object getObject(String queryString, Class<T> c) throws
+ InstantiationException, IllegalAccessException {
+ JAXBContext jaxbContext = null;
+ Unmarshaller unmarshaller = null;
+ StringReader reader = new StringReader(queryString);
+ try {
+ jaxbContext = JAXBContext.newInstance(c);
+ unmarshaller = jaxbContext.createUnmarshaller();
+ return (T) unmarshaller.unmarshal(reader);
+ } catch (JAXBException e) {
+ System.out.println("Exception : " + e);
+ return null;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> String convertObjectToXml(T object, Class<T> clazz, String functionName) throws
+ SecurityException, NoSuchMethodException, IllegalArgumentException, IllegalAccessException,
+ InvocationTargetException {
+ JAXBElement<T> root = null;
+ StringWriter stringWriter = new StringWriter();
+ ObjectFactory methodObject = new ObjectFactory();
+
+ Method method = methodObject.getClass().getMethod(functionName, clazz);
+ root = (JAXBElement<T>) method.invoke(methodObject, object);
+ try {
+ getMarshaller(clazz).marshal(root, stringWriter);
+ } catch (JAXBException e) {
+ LOGGER.error("Not able to marshall", e);
+ }
+ return stringWriter.toString();
+ }
+
+ public static Marshaller getMarshaller(Class clazz) {
+ JAXBContext jaxbContext = null;
+ try {
+ jaxbContext = JAXBContext.newInstance(clazz);
+ Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+ jaxbMarshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
+
+ return jaxbMarshaller;
+ } catch (JAXBException e) {
+ LOGGER.error("Error : ", e);
+ }
+
+ return null;
+ }
+
+ public static HashMap<String, String> stringListToMap(StringList stringList) {
+ HashMap<String, String> map = new HashMap<String, String>();
+ if (stringList == null) {
+ return null;
+ }
+ List<String> list = stringList.getElements();
+ for (String element : list) {
+ StringTokenizer tk = new StringTokenizer(element, "=");
+ map.put(tk.nextToken(), tk.nextToken());
+ }
+ return map;
+ }
+
+ public static HashMap<String, String> stringListToMap(String paramList) throws Exception {
+ StringList stringList = (StringList) Util.getObject(paramList, StringList.class);
+ return stringListToMap(stringList);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java b/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
index 7f462a9..595ef67 100644
--- a/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
+++ b/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
@@ -24,6 +24,7 @@ import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Method;
+
import org.apache.lens.regression.core.testHelper.BaseTestClass;
import org.apache.lens.regression.util.Util;
@@ -49,7 +50,7 @@ public class ITSmokeTest extends BaseTestClass {
LOGGER.info("Test Name: " + method.getName());
}
- @Test(enabled = true, groups = {"integration"})
+ @Test(enabled = true, groups = { "integration" })
public void smokeTest() throws Exception {
String exampleScript = clientDir + "/bin/run-examples.sh";
[08/51] [abbrv] incubator-lens git commit: LENS-242:Add Helper class
for lens-regression
Posted by jd...@apache.org.
LENS-242:Add Helper class for lens-regression
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/6a4b2470
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/6a4b2470
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/6a4b2470
Branch: refs/heads/current-release-line
Commit: 6a4b247041d7aea64fce352556fadf47c7529e2c
Parents: 1254a1e
Author: arshad-matin <ar...@C1MNH5W0DTY3.local>
Authored: Mon Jun 1 16:06:31 2015 +0530
Committer: arshad-matin <ar...@C1MNH5W0DTY3.local>
Committed: Mon Jun 1 16:06:31 2015 +0530
----------------------------------------------------------------------
lens-regression/pom.xml | 18 +
.../regression/core/constants/MetastoreURL.java | 39 +
.../regression/core/constants/QueryURL.java | 32 +
.../regression/core/constants/SessionURL.java | 34 +
.../regression/core/helpers/LensHelper.java | 58 ++
.../core/helpers/LensServerHelper.java | 71 ++
.../core/helpers/MetastoreHelper.java | 156 ++++
.../regression/core/helpers/QueryHelper.java | 868 +++++++++++++++++++
.../core/helpers/ServiceManagerHelper.java | 292 +++++++
.../regression/core/helpers/SessionHelper.java | 198 +++++
.../lens/regression/core/type/FormBuilder.java | 80 ++
.../lens/regression/core/type/MapBuilder.java | 67 ++
.../core/type/PrepareQueryHandles.java | 46 +
.../lens/regression/core/type/QueryHandles.java | 46 +
.../apache/lens/regression/util/AssertUtil.java | 142 +++
.../org/apache/lens/regression/util/Util.java | 113 ++-
.../lens/regression/sanity/ITSmokeTest.java | 3 +-
17 files changed, 2255 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/pom.xml
----------------------------------------------------------------------
diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml
index 037b2e9..59d1284 100644
--- a/lens-regression/pom.xml
+++ b/lens-regression/pom.xml
@@ -40,6 +40,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-cube</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
@@ -51,6 +56,19 @@
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-multipart</artifactId>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java
new file mode 100644
index 0000000..84cb310
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/MetastoreURL.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.constants;
+
+public class MetastoreURL {
+
+ private MetastoreURL() {
+
+ }
+
+ public static final String METASTORE_BASE_URL = "/metastore";
+ public static final String METASTORE_CUBES_URL = METASTORE_BASE_URL + "/cubes";
+ public static final String METASTORE_DATABASES_URL = METASTORE_BASE_URL + "/databases";
+ public static final String METASTORE_DATABASES_CURRENT_URL = METASTORE_DATABASES_URL + "/current";
+ public static final String METASTORE_DIMENSIONS_URL = METASTORE_BASE_URL + "/dimensions";
+ public static final String METASTORE_DIMTABLES_URL = METASTORE_BASE_URL + "/dimtables";
+ public static final String METASTORE_FACTS_URL = METASTORE_BASE_URL + "/facts";
+ public static final String METASTORE_FLATTENED_URL = METASTORE_BASE_URL + "/flattened";
+ public static final String METASTORE_NATIVETABLES_URL = METASTORE_BASE_URL + "/nativetables";
+ public static final String METASTORE_STORAGES_URL = METASTORE_BASE_URL + "/storages";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryURL.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryURL.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryURL.java
new file mode 100644
index 0000000..fcbf14d
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/QueryURL.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.constants;
+
+public class QueryURL {
+
+ private QueryURL() {
+
+ }
+
+ public static final String QUERYAPI_BASE_URL = "/queryapi";
+ public static final String QUERY_URL = QUERYAPI_BASE_URL + "/queries";
+ public static final String PREPAREDQUERY_URL = QUERYAPI_BASE_URL + "/preparedqueries";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/constants/SessionURL.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/SessionURL.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/SessionURL.java
new file mode 100644
index 0000000..365146b
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/SessionURL.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.constants;
+
+public class SessionURL {
+
+ private SessionURL() {
+
+ }
+
+ public static final String SESSION_BASE_URL = "/session";
+ public static final String SESSION_PARAMS_URL = SESSION_BASE_URL + "/params";
+ public static final String SESSION_ADD_RESOURCE_URL = SESSION_BASE_URL + "/resources/add";
+ public static final String SESSION_REMOVE_RESOURCE_URL = SESSION_BASE_URL + "/resources/delete";
+ public static final String SESSION_LIST_RESOURCE_URL = SESSION_BASE_URL + "/resources/list";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensHelper.java
new file mode 100644
index 0000000..c74da5d
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensHelper.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.helpers;
+
+public class LensHelper {
+
+ protected QueryHelper queryHelper;
+ protected MetastoreHelper metastoreHelper;
+ protected SessionHelper sessionHelper;
+ protected LensServerHelper serverHelper;
+ protected String envFileName;
+
+ public LensHelper(String envFileName) {
+ this.envFileName = envFileName;
+ queryHelper = new QueryHelper(envFileName);
+ metastoreHelper = new MetastoreHelper(envFileName);
+ sessionHelper = new SessionHelper(envFileName);
+ serverHelper = new LensServerHelper(envFileName);
+ }
+
+ public QueryHelper getQueryHelper() {
+ return queryHelper;
+ }
+
+ public MetastoreHelper getMetastoreHelper() {
+ return metastoreHelper;
+ }
+
+ public SessionHelper getSessionHelper() {
+ return sessionHelper;
+ }
+
+ public LensServerHelper getServerHelper() {
+ return serverHelper;
+ }
+
+ public String getEnvFileName() {
+ return envFileName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java
new file mode 100644
index 0000000..ad67d7d
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.helpers;
+
+import java.io.IOException;
+
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Response;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.regression.util.AssertUtil;
+import org.apache.lens.regression.util.Util;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+import com.jcraft.jsch.JSchException;
+
+
+public class LensServerHelper extends ServiceManagerHelper {
+
+ private static Logger logger = Logger.getLogger(LensServerHelper.class);
+
+ private WebTarget servLens = ServiceManagerHelper.getServerLens();
+ private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
+
+ public LensServerHelper() {
+ }
+
+ public LensServerHelper(String envFileName) {
+ super(envFileName);
+ }
+
+ /**
+ * Restart Lens server
+ */
+
+ public void restart() throws JSchException, IOException, InterruptedException, JAXBException, LensException {
+ int counter = 0;
+ Util.runRemoteCommand("bash /usr/local/lens/server/bin/lens-ctl stop");
+ Util.runRemoteCommand("bash /usr/local/lens/server/bin/lens-ctl start");
+ Response response = this.exec("get", "", servLens, null, null);
+ while (response == null && counter < 40) {
+ Thread.sleep(5000);
+ logger.info("Waiting for Lens server to come up ");
+ response = this.exec("get", "", servLens, null, null);
+ logger.info(response);
+ counter++;
+ }
+ AssertUtil.assertSucceededResponse(response);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java
new file mode 100644
index 0000000..7fd3c47
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.helpers;
+
+
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.StringList;
+import org.apache.lens.regression.core.constants.MetastoreURL;
+import org.apache.lens.regression.core.type.MapBuilder;
+import org.apache.lens.regression.util.AssertUtil;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+
+
+
+public class MetastoreHelper extends ServiceManagerHelper {
+
+ private static Logger logger = Logger.getLogger(MetastoreHelper.class);
+
+ private WebTarget servLens = ServiceManagerHelper.getServerLens();
+ private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
+
+ public MetastoreHelper() {
+ }
+
+ public MetastoreHelper(String envFileName) {
+ super(envFileName);
+ }
+
+ /**
+ * Set Current Database for a Session
+ *
+ * @param sessionHandleString
+ * @param currentDBName
+ */
+
+ public void setCurrentDatabase(String sessionHandleString, String currentDBName) throws JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this.exec("put", MetastoreURL.METASTORE_DATABASES_CURRENT_URL, servLens, null, query,
+ MediaType.APPLICATION_XML_TYPE, null, currentDBName);
+ AssertUtil.assertSucceededResponse(response);
+ response = this.exec("get", MetastoreURL.METASTORE_DATABASES_CURRENT_URL, servLens, null, query);
+ String responseString = response.readEntity(String.class);
+ AssertUtil.assertSucceededResponse(response);
+ logger.info(responseString.trim());
+ if (!responseString.trim().equals(currentDBName)) {
+ throw new LensException("Could not set database");
+ }
+ logger.info("Set Current database to " + currentDBName);
+ }
+
+ public void setCurrentDatabase(String currentDBName) throws JAXBException, LensException {
+ setCurrentDatabase(sessionHandleString, currentDBName);
+ }
+
+ /**
+ * Get Current Database for a Session
+ *
+ * @param sessionHandleString
+ * @return the current DB Name
+ */
+
+ public String getCurrentDatabase(String sessionHandleString) throws JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this.exec("get", MetastoreURL.METASTORE_DATABASES_CURRENT_URL, servLens, null, query,
+ MediaType.APPLICATION_XML_TYPE, null);
+ AssertUtil.assertSucceededResponse(response);
+ return response.readEntity(String.class);
+ }
+
+ public String getCurrentDatabase() throws JAXBException, LensException {
+ return getCurrentDatabase(sessionHandleString);
+ }
+
+ /**
+ * Get list of databases
+ *
+ * @param sessionHandleString
+ * @return List of all the Databases
+ */
+
+ public StringList listDatabases(String sessionHandleString) throws JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this
+ .exec("get", MetastoreURL.METASTORE_DATABASES_URL, servLens, null, query, MediaType.APPLICATION_XML_TYPE, null);
+ AssertUtil.assertSucceededResponse(response);
+ StringList responseString = response.readEntity(StringList.class);
+ return responseString;
+ }
+
+ public StringList listDatabases() throws JAXBException, LensException {
+ return listDatabases(sessionHandleString);
+ }
+
+ /**
+ * Create a database
+ *
+ * @param dbName
+ * @param sessionHandleString
+ */
+
+ public void createDatabase(String dbName, String sessionHandleString) throws JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this
+ .exec("post", MetastoreURL.METASTORE_DATABASES_URL, servLens, null, query, MediaType.APPLICATION_XML_TYPE, null,
+ dbName);
+ AssertUtil.assertSucceeded(response);
+ }
+
+ public void createDatabase(String dbName) throws JAXBException, LensException {
+ createDatabase(dbName, sessionHandleString);
+ }
+
+ /**
+ * Drop a DB
+ *
+ * @param dbName
+ * @param sessionHandleString
+ */
+ public void dropDatabase(String dbName, String sessionHandleString) throws JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ query.put("cascade", "true");
+ Response response = this.exec("delete", MetastoreURL.METASTORE_DATABASES_URL + "/" + dbName, servLens, null, query,
+ MediaType.APPLICATION_XML_TYPE, null);
+ AssertUtil.assertSucceeded(response);
+ }
+
+ public void dropDatabase(String dbName) throws JAXBException, LensException {
+ dropDatabase(dbName, sessionHandleString);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
new file mode 100644
index 0000000..6b8bdda
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
@@ -0,0 +1,868 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.helpers;
+
+import java.util.List;
+
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.LensConf;
+import org.apache.lens.api.query.*;
+import org.apache.lens.api.response.LensResponse;
+import org.apache.lens.regression.core.constants.QueryURL;
+import org.apache.lens.regression.core.type.FormBuilder;
+import org.apache.lens.regression.core.type.MapBuilder;
+import org.apache.lens.regression.core.type.PrepareQueryHandles;
+import org.apache.lens.regression.core.type.QueryHandles;
+import org.apache.lens.regression.util.AssertUtil;
+import org.apache.lens.regression.util.Util;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+
+
+
+public class QueryHelper extends ServiceManagerHelper {
+
+ private static Logger logger = Logger.getLogger(QueryHelper.class);
+ private WebTarget servLens = ServiceManagerHelper.getServerLens();
+ private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
+
+ public QueryHelper() {
+ }
+
+ public QueryHelper(String envFileName) {
+ super(envFileName);
+ }
+
+ /**
+ * Execute with conf
+ *
+ * @param queryString
+ * @param queryName
+ * @param sessionHandleString
+ * @param conf
+ * @return the query Handle
+ */
+ public QueryHandle executeQuery(String queryString, String queryName, String sessionHandleString, String conf) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("query", queryString);
+ formData.add("operation", "EXECUTE");
+ formData.add("conf", conf);
+ if (queryName != null) {
+ formData.add("queryName", queryName);
+ }
+ Response response = this.exec("post", QueryURL.QUERY_URL, servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
+ MediaType.APPLICATION_XML, formData.getForm());
+ AssertUtil.assertSucceededResponse(response);
+ String queryHandleString = response.readEntity(String.class);
+ logger.info(queryHandleString);
+ LensResponse successResponse = (LensResponse) Util.getObject(queryHandleString, LensResponse.class);
+ QueryHandle queryHandle = (QueryHandle) successResponse.getData();
+ if (queryHandle == null) {
+ throw new LensException("Query Execute Failed");
+ }
+ logger.info("Query Handle : " + queryHandle);
+ return queryHandle;
+ }
+
+ public QueryHandle executeQuery(String queryString, String queryName, String sessionHandleString) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executeQuery(queryString, queryName, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryHandle executeQuery(String queryString, String queryName) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executeQuery(queryString, queryName, sessionHandleString);
+ }
+
+ public QueryHandle executeQuery(String queryString) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executeQuery(queryString, null);
+ }
+
+ /**
+ * Execute with timeout
+ *
+ * @param queryString
+ * @param timeout
+ * @param queryName
+ * @param sessionHandleString
+ * @param conf
+ * @return the queryHandleWithResultSet
+ */
+
+ public QueryHandleWithResultSet executeQueryTimeout(String queryString, String timeout, String queryName,
+ String sessionHandleString, String conf) throws InstantiationException, IllegalAccessException, JAXBException,
+ LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("query", queryString);
+ formData.add("operation", "EXECUTE_WITH_TIMEOUT");
+ formData.add("conf", conf);
+ if (timeout != null) {
+ formData.add("timeoutmillis", timeout);
+ }
+ if (queryName != null) {
+ formData.add("queryName", queryName);
+ }
+ Response response = this.exec("post", QueryURL.QUERY_URL, servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
+ MediaType.APPLICATION_XML, formData.getForm());
+ AssertUtil.assertSucceededResponse(response);
+ String queryHandleString = response.readEntity(String.class);
+ logger.info(queryHandleString);
+ LensResponse successResponse = (LensResponse) Util.getObject(queryHandleString, LensResponse.class);
+ QueryHandleWithResultSet queryHandleWithResultSet = (QueryHandleWithResultSet) successResponse.getData();
+ if (queryHandleWithResultSet==null) {
+ throw new LensException("Query Execute Failed");
+ }
+ logger.info("Query Handle with ResultSet : " + queryHandleWithResultSet);
+ return queryHandleWithResultSet;
+ }
+
+ public QueryHandleWithResultSet executeQueryTimeout(String queryString, String timeout, String queryName,
+ String sessionHandleString) throws InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executeQueryTimeout(queryString, timeout, queryName, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryHandleWithResultSet executeQueryTimeout(String queryString, String timeout, String queryName) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executeQueryTimeout(queryString, timeout, queryName, sessionHandleString);
+ }
+
+ public QueryHandleWithResultSet executeQueryTimeout(String queryString, String timeout) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executeQueryTimeout(queryString, timeout, null);
+ }
+
+ public QueryHandleWithResultSet executeQueryTimeout(String queryString) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executeQueryTimeout(queryString, null);
+ }
+
+ /**
+ * Execute the query
+ *
+ * @param queryString
+ * @param queryName
+ * @param user
+ * @param sessionHandleString
+ * @param conf
+ * @return the query Handle
+ */
+
+ public QueryHandle executeQuery(String queryString, String queryName, String user, String sessionHandleString,
+ LensConf conf) throws JAXBException, InstantiationException, IllegalAccessException, LensException {
+
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("query", queryString);
+ formData.add("operation", "EXECUTE");
+ if (queryName != null) {
+ formData.add("queryName", queryName);
+ }
+ if (user != null) {
+ formData.add("user", user);
+ }
+ formData.getForm().bodyPart(
+ new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
+ MediaType.APPLICATION_XML_TYPE));
+ Response response = this.exec("post", "/queryapi/queries", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
+ MediaType.APPLICATION_XML, formData.getForm());
+ AssertUtil.assertSucceededResponse(response);
+ String queryHandleString = response.readEntity(String.class);
+ logger.info(queryHandleString);
+ LensResponse successResponse = (LensResponse) Util.getObject(queryHandleString, LensResponse.class);
+ QueryHandle queryHandle = (QueryHandle) successResponse.getData();
+ return queryHandle;
+ }
+
+ /**
+ * Explain the query
+ *
+ * @param queryString
+ * @param sessionHandleString
+ * @param conf
+ * @return the query Plan
+ */
+
+ public QueryPlan explainQuery(String queryString, String sessionHandleString, String conf) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("query", queryString);
+ formData.add("operation", "EXPLAIN");
+ formData.getForm().bodyPart(
+ new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
+ MediaType.APPLICATION_XML_TYPE));
+ Response response = this.exec("post", "/queryapi/queries", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
+ MediaType.APPLICATION_XML, formData.getForm());
+ AssertUtil.assertSucceededResponse(response);
+ String queryPlanString = response.readEntity(String.class);
+ logger.info(queryPlanString);
+ LensResponse successResponse = (LensResponse) Util.getObject(queryPlanString, LensResponse.class);
+ QueryPlan queryPlan = (QueryPlan) successResponse.getData();
+ return queryPlan;
+ }
+
+ public QueryPlan explainQuery(String queryString, String sessionHandleString) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return explainQuery(queryString, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryPlan explainQuery(String queryString) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return explainQuery(queryString, sessionHandleString);
+ }
+
+ /**
+ * Estimate the query
+ *
+ * @param queryString
+ * @param sessionHandleString
+ * @param conf
+ * @return the Estimate result
+ */
+
+ public QueryCost estimateQuery(String queryString, String sessionHandleString, String conf) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("query", queryString);
+ formData.add("operation", "ESTIMATE");
+ formData.add("conf", conf);
+ Response response = this.exec("post", QueryURL.QUERY_URL, servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
+ MediaType.APPLICATION_XML, formData.getForm());
+ AssertUtil.assertSucceededResponse(response);
+ String queryCostString = response.readEntity(String.class);
+ logger.info(queryCostString);
+ LensResponse successResponse = (LensResponse) Util.getObject(queryCostString, LensResponse.class);
+ QueryCost queryCost = (QueryCost) successResponse.getData();
+ if (queryCost == null) {
+ throw new LensException("Estimate Failed");
+ }
+ return queryCost;
+ }
+
+ public QueryCost estimateQuery(String queryString, String sessionHandleString) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return estimateQuery(queryString, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryCost estimateQuery(String queryString) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return estimateQuery(queryString, sessionHandleString);
+ }
+
+ /**
+ * Prepare and Explain the query
+ *
+ * @param queryString
+ * @param sessionHandleString
+ * @param conf
+ * @return the query Plan
+ */
+
+ public QueryPlan explainAndPrepareQuery(String queryString, String sessionHandleString, String conf) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("query", queryString);
+ formData.add("operation", "EXPLAIN_AND_PREPARE");
+ formData.add("conf", conf);
+ Response response = this
+ .exec("post", "/queryapi/preparedqueries", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
+ MediaType.APPLICATION_XML, formData.getForm());
+ String queryPlanString = response.readEntity(String.class);
+ logger.info(queryPlanString);
+ QueryPlan queryPlan = (QueryPlan) Util.getObject(queryPlanString, QueryPlan.class);
+ return queryPlan;
+ }
+
+ public QueryPlan explainAndPrepareQuery(String queryString, String sessionHandleString) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return explainAndPrepareQuery(queryString, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryPlan explainAndPrepareQuery(String queryString) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return explainAndPrepareQuery(queryString, sessionHandleString);
+ }
+
+ /**
+ * Get the Result set
+ *
+ * @param queryHandle
+ * @param fromIndex
+ * @param fetchSize
+ * @param sessionHandleString
+ * @return the query Result
+ */
+ public QueryResult getResultSet(QueryHandle queryHandle, String fromIndex, String fetchSize,
+ String sessionHandleString) throws JAXBException, InstantiationException, IllegalAccessException, LensException {
+
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ query.put("fromindex", fromIndex);
+ query.put("fetchsize", fetchSize);
+
+ Response response = this
+ .exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", servLens, null, query);
+ AssertUtil.assertSucceededResponse(response);
+ logger.info(response);
+ String queryResultString = response.readEntity(String.class);
+ logger.info(queryResultString);
+ QueryResult queryResult = (QueryResult) Util.getObject(queryResultString, QueryResult.class);
+ return queryResult;
+ }
+
+ public QueryResult getResultSet(QueryHandle queryHandle, String fromIndex, String fetchSize) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return getResultSet(queryHandle, fromIndex, fetchSize, sessionHandleString);
+ }
+
+ public QueryResult getResultSet(QueryHandle queryHandle) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return getResultSet(queryHandle, "0", "100", sessionHandleString);
+ }
+
+ /**
+ * Get the HTTP result set
+ *
+ * @param queryHandle
+ * @return the query Result
+ */
+
+ public QueryResult getHttpResultSet(QueryHandle queryHandle) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ Response response = this
+ .exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/httpresultset", servLens, null, null);
+ AssertUtil.assertSucceededResponse(response);
+ logger.info(response);
+ String queryResultString = response.readEntity(String.class);
+ logger.info(queryResultString);
+ QueryResult queryResult = (QueryResult) Util.getObject(queryResultString, QueryResult.class);
+ return queryResult;
+ }
+
+ /**
+ * Execute prepared Query
+ *
+ * @param queryHandle
+ * @param sessionHandleString
+ * @param conf
+ * @return the query Handle
+ */
+
+ public QueryHandle executePreparedQuery(QueryPrepareHandle queryHandle, String sessionHandleString,
+ String conf) throws InstantiationException, IllegalAccessException, JAXBException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("prepareHandle", queryHandle.toString());
+ formData.add("operation", "EXECUTE");
+ formData.add("conf", conf);
+ Response response = this.exec("post", "/queryapi/preparedqueries/" + queryHandle.toString(), servLens, null, null,
+ MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML, formData.getForm());
+ String queryHandleString = response.readEntity(String.class);
+ AssertUtil.assertSucceededResponse(response);
+ logger.info(queryHandleString);
+ QueryHandle handle = (QueryHandle) Util.getObject(queryHandleString, QueryHandle.class);
+ return handle;
+ }
+
+ public QueryHandle executePreparedQuery(QueryPrepareHandle queryHandle, String sessionHandleString) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executePreparedQuery(queryHandle, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryHandle executePreparedQuery(QueryPrepareHandle queryHandle) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executePreparedQuery(queryHandle, sessionHandleString);
+ }
+
+ /**
+ * Execute prepared Query with timeout
+ *
+ * @param queryHandle
+ * @param timeout
+ * @param sessionHandleString
+ * @param conf
+ * @return the query Handle with result set
+ */
+
+ public QueryHandleWithResultSet executePreparedQueryTimeout(QueryPrepareHandle queryHandle, String timeout,
+ String sessionHandleString, String conf) throws InstantiationException, IllegalAccessException,
+ JAXBException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("prepareHandle", queryHandle.toString());
+ formData.add("operation", "EXECUTE_WITH_TIMEOUT");
+ formData.add("conf", conf);
+ if (timeout != null) {
+ formData.add("timeoutmillis", timeout);
+ }
+ Response response = this.exec("post", "/queryapi/preparedqueries/" + queryHandle.toString(), servLens, null, null,
+ MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML, formData.getForm());
+ String queryHandleString = response.readEntity(String.class);
+ AssertUtil.assertSucceededResponse(response);
+ logger.info(queryHandleString);
+ QueryHandleWithResultSet handle = (QueryHandleWithResultSet) Util
+ .getObject(queryHandleString, QueryHandleWithResultSet.class);
+ return handle;
+ }
+
+ public QueryHandleWithResultSet executePreparedQueryTimeout(QueryPrepareHandle queryHandle, String timeout,
+ String sessionHandleString) throws InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executePreparedQueryTimeout(queryHandle, timeout, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryHandleWithResultSet executePreparedQueryTimeout(QueryPrepareHandle queryHandle, String timeout) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executePreparedQueryTimeout(queryHandle, timeout, sessionHandleString);
+ }
+
+ public QueryHandleWithResultSet executePreparedQueryTimeout(QueryPrepareHandle queryHandle) throws
+ InstantiationException, IllegalAccessException, JAXBException, LensException {
+ return executePreparedQueryTimeout(queryHandle, null);
+ }
+
+ /**
+ * Submit prepared Query
+ *
+ * @param queryString
+ * @param sessionHandleString
+ * @param conf
+ * @return the query Prepare Handle
+ */
+
+ public QueryPrepareHandle submitPreparedQuery(String queryString, String queryName, String sessionHandleString,
+ String conf) throws JAXBException, InstantiationException, IllegalAccessException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("query", queryString);
+ formData.add("operation", "PREPARE");
+ formData.add("conf", conf);
+ if (queryName != null) {
+ formData.add("queryName", queryName);
+ }
+ Response response = this
+ .exec("post", "/queryapi/preparedqueries", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
+ MediaType.APPLICATION_XML, formData.getForm());
+ String queryHandleString = response.readEntity(String.class);
+ logger.info(queryHandleString);
+ AssertUtil.assertSucceededResponse(response);
+ QueryPrepareHandle queryHandle = (QueryPrepareHandle) Util.getObject(queryHandleString, QueryPrepareHandle.class);
+ return queryHandle;
+ }
+
+ public QueryPrepareHandle submitPreparedQuery(String queryString, String queryName, String sessionHandleString) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return submitPreparedQuery(queryString, queryName, sessionHandleString,
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
+ }
+
+ public QueryPrepareHandle submitPreparedQuery(String queryString, String queryName) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return submitPreparedQuery(queryString, queryName, sessionHandleString);
+ }
+
+ public QueryPrepareHandle submitPreparedQuery(String queryString) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return submitPreparedQuery(queryString, null);
+ }
+
+ /**
+ * Destroy prepared Query
+ *
+ * @param queryPreparedHandle
+ * @param sessionHandleString
+ */
+
+ public void destoryPreparedQueryByHandle(QueryPrepareHandle queryPreparedHandle, String sessionHandleString) throws
+ JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this
+ .exec("delete", QueryURL.PREPAREDQUERY_URL + "/" + queryPreparedHandle.toString(), servLens, null, query);
+ logger.info("Response : " + response);
+ AssertUtil.assertSucceededResponse(response);
+ }
+
+ public void destoryPreparedQueryByHandle(QueryPrepareHandle queryPreparedHandle) throws JAXBException, LensException {
+ destoryPreparedQueryByHandle(queryPreparedHandle, sessionHandleString);
+ }
+
+ /**
+ * Get Prepared QueryHandle List
+ *
+ * @param queryName
+ * @param user
+ * @param sessionHandleString
+ * @param fromDate
+ * @param toDate
+ * @return the query Handle
+ */
+
+ public List<QueryPrepareHandle> getPreparedQueryHandleList(String queryName, String user, String sessionHandleString,
+ String fromDate, String toDate) throws InstantiationException, IllegalAccessException {
+ MapBuilder queryList = new MapBuilder("sessionid", sessionHandleString);
+ if (queryName != null) {
+ queryList.put("queryName", queryName);
+ }
+ if (user != null) {
+ queryList.put("user", user);
+ }
+ if (fromDate != null) {
+ queryList.put("fromDate", fromDate);
+ }
+ if (toDate != null) {
+ queryList.put("toDate", toDate);
+ }
+ Response response = this.sendQuery("get", QueryURL.PREPAREDQUERY_URL, queryList);
+ logger.info("Response : " + response);
+ String responseString = response.readEntity(String.class);
+ logger.info(responseString);
+ PrepareQueryHandles result = (PrepareQueryHandles) Util.getObject(responseString, PrepareQueryHandles.class);
+ List<QueryPrepareHandle> list = result.getQueryHandles();
+ return list;
+ }
+
+ public List<QueryPrepareHandle> getPreparedQueryHandleList(String queryName, String user,
+ String sessionHandleString) throws InstantiationException, IllegalAccessException {
+ return getPreparedQueryHandleList(queryName, user, sessionHandleString, null, null);
+ }
+
+ public List<QueryPrepareHandle> getPreparedQueryHandleList(String queryName, String user) throws
+ InstantiationException, IllegalAccessException {
+ return getPreparedQueryHandleList(queryName, user, sessionHandleString);
+ }
+
+ public List<QueryPrepareHandle> getPreparedQueryHandleList(String queryName) throws
+ InstantiationException, IllegalAccessException {
+ return getPreparedQueryHandleList(queryName, null);
+ }
+
+ public List<QueryPrepareHandle> getPreparedQueryHandleList() throws InstantiationException, IllegalAccessException {
+ return getPreparedQueryHandleList(null);
+ }
+
+ /**
+ * Destroy prepared Query
+ *
+ * @param queryName
+ * @param sessionHandleString
+ * @param fromDate
+ * @param toDate
+ */
+
+ public void destroyPreparedQuery(String queryName, String user, String sessionHandleString, String fromDate,
+ String toDate) throws JAXBException, LensException {
+
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ if (queryName != null) {
+ query.put("queryName", queryName);
+ }
+ if (user != null) {
+ query.put("user", user);
+ }
+ if (fromDate != null) {
+ query.put("fromDate", fromDate);
+ }
+ if (toDate != null) {
+ query.put("toDate", toDate);
+ }
+
+ Response response = this.exec("delete", QueryURL.PREPAREDQUERY_URL, servLens, null, query);
+ logger.info("Response : " + response);
+ AssertUtil.assertSucceededResponse(response);
+ }
+
+ public void destroyPreparedQuery(String queryName, String user, String sessionHandleString) throws
+ JAXBException, LensException {
+ destroyPreparedQuery(queryName, user, sessionHandleString, null, null);
+ }
+
+ public void destroyPreparedQuery(String queryName, String user) throws JAXBException, LensException {
+ destroyPreparedQuery(queryName, user, sessionHandleString);
+ }
+
+ public void destroyPreparedQuery(String queryName) throws JAXBException, LensException {
+ destroyPreparedQuery(queryName, null);
+ }
+
+ public void destroyPreparedQuery() throws JAXBException, LensException {
+ destroyPreparedQuery(null);
+ }
+
+ /**
+ * Get prepared Query
+ *
+ * @param queryPrepareHandle
+ * @param sessionHandleString
+ * @return the client response
+ */
+
+ public Response getPreparedQuery(QueryPrepareHandle queryPrepareHandle, String sessionHandleString) {
+
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this
+ .exec("get", QueryURL.PREPAREDQUERY_URL + "/" + queryPrepareHandle.toString(), servLens, null, query);
+ return response;
+ }
+
+ public Response getPreparedQuery(QueryPrepareHandle queryPrepareHandle) {
+ return getPreparedQuery(queryPrepareHandle, sessionHandleString);
+ }
+
+ /**
+ * List Query Handle
+ *
+ * @param queryName
+ * @param state
+ * @param user
+ * @param sessionHandleString
+ * @param fromDate
+ * @param toDate
+ * @return the query Handle list
+ */
+ public List<QueryHandle> getQueryHandleList(String queryName, String state, String user, String sessionHandleString,
+ String fromDate, String toDate) throws InstantiationException, IllegalAccessException {
+ MapBuilder queryList = new MapBuilder("sessionid", sessionHandleString);
+ if (queryName != null) {
+ queryList.put("queryName", queryName);
+ }
+ if (state != null) {
+ queryList.put("state", state);
+ }
+ if (user != null) {
+ queryList.put("user", user);
+ }
+ if (fromDate != null) {
+ queryList.put("fromDate", fromDate);
+ }
+ if (toDate != null) {
+ queryList.put("toDate", toDate);
+ }
+ Response response = this.sendQuery("get", QueryURL.QUERY_URL, queryList);
+ logger.info("Response : " + response);
+ String responseString = response.readEntity(String.class);
+ QueryHandles result = (QueryHandles) Util.getObject(responseString, QueryHandles.class);
+ List<QueryHandle> list = result.getQueryHandles();
+ return list;
+ }
+
+ public List<QueryHandle> getQueryHandleList(String queryName, String state, String user,
+ String sessionHandleString) throws InstantiationException, IllegalAccessException {
+ return getQueryHandleList(queryName, state, user, sessionHandleString, null, null);
+ }
+
+ public List<QueryHandle> getQueryHandleList(String queryName, String state, String user) throws
+ InstantiationException, IllegalAccessException {
+ return getQueryHandleList(queryName, state, user, sessionHandleString);
+ }
+
+ public List<QueryHandle> getQueryHandleList(String queryName, String state) throws
+ InstantiationException, IllegalAccessException {
+ return getQueryHandleList(queryName, state, null);
+ }
+
+ public List<QueryHandle> getQueryHandleList(String queryName) throws InstantiationException, IllegalAccessException {
+ return getQueryHandleList(queryName, null);
+ }
+
+ public List<QueryHandle> getQueryHandleList() throws InstantiationException, IllegalAccessException {
+ return getQueryHandleList(null);
+ }
+
+ /**
+ * Wait for Completion
+ *
+ * @param sessionHandleString
+ * @param queryHandle
+ * @return the lens query
+ */
+
+ public LensQuery waitForCompletion(String sessionHandleString, QueryHandle queryHandle) throws
+ JAXBException, InterruptedException, InstantiationException, IllegalAccessException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString(), servLens, null, query);
+ AssertUtil.assertSucceededResponse(response);
+ String responseString = response.readEntity(String.class);
+ LensQuery lensQuery = (LensQuery) Util.getObject(responseString, LensQuery.class);
+ while (!lensQuery.getStatus().finished()) {
+ Thread.sleep(1000);
+ logger.info("Waiting...");
+ response = this.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString(), servLens, null, query);
+ lensQuery = (LensQuery) Util.getObject(response.readEntity(String.class), LensQuery.class);
+ }
+ logger.info(lensQuery.getStatus().getStatusMessage());
+ return lensQuery;
+ }
+
+ public LensQuery waitForCompletion(QueryHandle queryHandle) throws
+ JAXBException, InterruptedException, InstantiationException, IllegalAccessException, LensException {
+ return waitForCompletion(sessionHandleString, queryHandle);
+ }
+
+ /**
+ * Wait for Query to run
+ *
+ * @param queryHandle
+ * @param sessionHandleString
+ * @return the query status
+ */
+
+ public QueryStatus waitForQueryToRun(QueryHandle queryHandle, String sessionHandleString) throws
+ JAXBException, InterruptedException, InstantiationException, IllegalAccessException, LensException {
+ QueryStatus queryStatus = getQueryStatus(sessionHandleString, queryHandle);
+ while (queryStatus.getStatus() == QueryStatus.Status.QUEUED) {
+ logger.info("Waiting for Query to be in Running Phase");
+ Thread.sleep(1000);
+ queryStatus = getQueryStatus(sessionHandleString, queryHandle);
+ }
+ return queryStatus;
+ }
+
+ public QueryStatus waitForQueryToRun(QueryHandle queryHandle) throws
+ JAXBException, InterruptedException, InstantiationException, IllegalAccessException, LensException {
+ return waitForQueryToRun(queryHandle, sessionHandleString);
+ }
+
+ /**
+ * Get Query Status
+ *
+ * @param sessionHandleString
+ * @param queryHandle
+ * @return the query Status
+ */
+
+ public QueryStatus getQueryStatus(String sessionHandleString, QueryHandle queryHandle) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString(), servLens, null, query);
+ logger.info("Response : " + response);
+ AssertUtil.assertSucceededResponse(response);
+ LensQuery lensQuery = (LensQuery) Util.getObject(response.readEntity(String.class), LensQuery.class);
+ QueryStatus qStatus = lensQuery.getStatus();
+ logger.info("Query Status : " + qStatus);
+ return qStatus;
+ }
+
+ public QueryStatus getQueryStatus(QueryHandle queryHandle) throws
+ JAXBException, InstantiationException, IllegalAccessException, LensException {
+ return getQueryStatus(sessionHandleString, queryHandle);
+ }
+
+ /**
+ * Kill Query by QueryHandle
+ *
+ * @param sessionHandleString
+ * @param queryHandle
+ */
+
+ public void killQueryByQueryHandle(String sessionHandleString, QueryHandle queryHandle) throws
+ JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this.exec("delete", QueryURL.QUERY_URL + "/" + queryHandle.toString(), servLens, null, query);
+ logger.info("Response : " + response);
+ AssertUtil.assertSucceededResponse(response);
+ }
+
+ public void killQueryByQueryHandle(QueryHandle queryHandle) throws JAXBException, LensException {
+ killQueryByQueryHandle(sessionHandleString, queryHandle);
+ }
+
+ /**
+ * Kill Query
+ *
+ * @param queryName
+ * @param state
+ * @param user
+ * @param sessionHandleString
+ * @param fromDate
+ * @param toDate
+ */
+
+ public void killQuery(String queryName, String state, String user, String sessionHandleString, String fromDate,
+ String toDate) throws JAXBException, LensException {
+
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ if (queryName != null) {
+ query.put("queryName", queryName);
+ }
+
+ if (state != null) {
+ query.put("state", state);
+ }
+
+ if (user != null) {
+ query.put("user", user);
+ }
+
+ if (fromDate != null) {
+ query.put("fromDate", fromDate);
+ }
+
+ if (toDate != null) {
+ query.put("toDate", toDate);
+ }
+
+ Response response = this.exec("delete", QueryURL.QUERY_URL, servLens, null, query);
+ logger.info("Response : " + response);
+ AssertUtil.assertSucceededResponse(response);
+ }
+
+ public void killQuery(String queryName, String state, String user, String sessionHandleString) throws
+ JAXBException, LensException {
+ killQuery(queryName, state, user, sessionHandleString, null, null);
+ }
+
+ public void killQuery(String queryName, String state, String user) throws JAXBException, LensException {
+ killQuery(queryName, state, user, sessionHandleString);
+ }
+
+ public void killQuery(String queryName, String state) throws JAXBException, LensException {
+ killQuery(queryName, state, null);
+ }
+
+ public void killQuery(String queryName) throws JAXBException, LensException {
+ killQuery(queryName, null);
+ }
+
+ public void killQuery() throws JAXBException, LensException {
+ killQuery(null);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java
new file mode 100644
index 0000000..2aecbfd
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.helpers;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URI;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.ws.rs.client.*;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.UriBuilder;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.APIResult;
+import org.apache.lens.regression.core.constants.SessionURL;
+import org.apache.lens.regression.core.type.FormBuilder;
+import org.apache.lens.regression.core.type.MapBuilder;
+import org.apache.lens.regression.util.AssertUtil;
+import org.apache.lens.regression.util.Util;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
+
+
+public abstract class ServiceManagerHelper {
+
+ private static final String LENS_BASE_URL = "lens.baseurl";
+ private static final String LENS_ADMIN_URL = "lens.adminurl";
+ private static final String LENS_USERNAME = "lens.username";
+ private static final String LENS_PASSWORD = "lens.password";
+ private static final String LENS_SERVER_DIR = "lens.server.dir";
+ private static final String LENS_CLIENT_DIR = "lens.client.dir";
+ private static final String LENS_SERVER_HDFS_URL = "lens.server.hdfsurl";
+ private static final String LENS_CURRENT_DB = "lens.server.currentDB";
+
+ private static String sessionHandleString;
+ private static WebTarget servLens;
+
+ protected String baseUrl;
+ protected String adminUrl;
+ protected String userName;
+ protected String password;
+ protected String serverDir;
+ protected String clientDir;
+ protected String serverHdfsUrl;
+ protected String currentDB;
+
+ private Logger logger = Logger.getLogger(ServiceManagerHelper.class);
+
+ public ServiceManagerHelper(String envFileName) {
+ Properties prop = Util.getPropertiesObj(envFileName);
+ this.baseUrl = prop.getProperty(LENS_BASE_URL);
+ this.adminUrl = prop.getProperty(LENS_ADMIN_URL);
+ this.userName = prop.getProperty(LENS_USERNAME);
+ this.password = prop.getProperty(LENS_PASSWORD);
+ this.serverDir = prop.getProperty(LENS_SERVER_DIR);
+ this.clientDir = prop.getProperty(LENS_CLIENT_DIR);
+ this.serverHdfsUrl = prop.getProperty(LENS_SERVER_HDFS_URL);
+ this.currentDB = prop.getProperty(LENS_CURRENT_DB);
+ }
+
+ public ServiceManagerHelper() {
+
+ }
+
+ public static WebTarget init() {
+ Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).build();
+ String baseUri = Util.getProperty("lens.baseurl");
+ servLens = client.target(UriBuilder.fromUri(baseUri).build());
+ return servLens;
+ }
+
+ public static WebTarget getServerLens() {
+ return servLens;
+ }
+
+ public static String getSessionHandle() {
+ return sessionHandleString;
+ }
+
+ public static URI getServiceURI(String baseUri) {
+ return UriBuilder.fromUri(baseUri).build();
+ }
+
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ public String getAdminUrl() {
+ return adminUrl;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public String getServerDir() {
+ return serverDir;
+ }
+
+ public String getClientDir() {
+ return clientDir;
+ }
+
+ public String getServerHdfsUrl() {
+ return serverHdfsUrl;
+ }
+
+ public String getSessionHandleString() {
+ return sessionHandleString;
+ }
+
+ public String getCurrentDB() {
+ return currentDB;
+ }
+
+ public String openSession(String database) throws JAXBException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("username", this.getUserName());
+ formData.add("password", this.getPassword());
+ if (database != null) {
+ formData.add("database", database);
+ }
+ Response response = this.exec("post", SessionURL.SESSION_BASE_URL, ServiceManagerHelper.servLens, null, null,
+ MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML, formData.getForm());
+ AssertUtil.assertSucceededResponse(response);
+ sessionHandleString = response.readEntity(String.class);
+ logger.info("Session Handle String" + sessionHandleString);
+ return sessionHandleString;
+ }
+
+ public String openSession() throws JAXBException, LensException {
+ return openSession(null);
+ }
+
+ public void closeSession() throws JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this.exec("delete", SessionURL.SESSION_BASE_URL, ServiceManagerHelper.servLens, null, query);
+ APIResult result = response.readEntity(APIResult.class);
+ if (result.getStatus() == APIResult.Status.SUCCEEDED) {
+ throw new LensException("Status should be SUCCEEDED");
+ }
+ if (response.getStatus() == 200) {
+ throw new LensException("Status code should be 200");
+ }
+ if (result.getMessage() == null) {
+ throw new LensException("Status message is null");
+ }
+ logger.info("Closed Session : " + sessionHandleString);
+ }
+
+ public <T> Response exec(String functionName, String path, WebTarget service, FormDataMultiPart headers,
+ MapBuilder queryParams, MediaType inputMediaType, String outputMediaType) {
+ return exec(functionName, path, service, headers, queryParams, inputMediaType, outputMediaType, null);
+ }
+
+ public <T> Response exec(String functionName, String path, WebTarget service, FormDataMultiPart headers,
+ MapBuilder queryParams, MediaType inputMediaType) {
+ return exec(functionName, path, service, headers, queryParams, inputMediaType, null, null);
+ }
+
+ public <T> Response exec(String functionName, String path, WebTarget service, FormDataMultiPart headers,
+ MapBuilder queryParams) {
+ return exec(functionName, path, service, headers, queryParams, null, null, null);
+ }
+
+ public <T> Object exec(String functionName, String path, WebTarget service, FormDataMultiPart headers,
+ MapBuilder queryParams, MediaType inputMediaType, String outputMediaType, Class responseClass, T inputObject) {
+
+ Object result;
+ WebTarget builder = null;
+ String className = this.getClass().getName();
+
+ if (outputMediaType == null) {
+ outputMediaType = MediaType.WILDCARD;
+ }
+ if (inputMediaType == null) {
+ inputMediaType = MediaType.WILDCARD_TYPE;
+ }
+
+ builder = service.path(path);
+ if (queryParams != null) {
+ for (Map.Entry<String, String> queryMapEntry : queryParams.getMap().entrySet()) {
+ builder = builder.queryParam(queryMapEntry.getKey(), queryMapEntry.getValue());
+ }
+ }
+
+ Invocation.Builder build = builder.request(outputMediaType);
+
+ functionName = "exec" + functionName.toUpperCase();
+
+ try {
+ Class methodClass = Class.forName(className);
+ Object methodObject = methodClass.newInstance();
+
+ Method method = methodObject.getClass()
+ .getMethod(functionName, Invocation.Builder.class, inputMediaType.getClass(), responseClass.getClass(),
+ Object.class);
+ result = method.invoke(methodObject, build, inputMediaType, responseClass, inputObject);
+ return result;
+
+ } catch (NoSuchMethodException e) {
+ return e.getMessage();
+ } catch (InstantiationException e) {
+ return e.getMessage();
+ } catch (IllegalAccessException e) {
+ return e.getMessage();
+ } catch (InvocationTargetException e) {
+ return e.getMessage();
+ } catch (ClassNotFoundException e) {
+ return e.getMessage();
+ } catch (Exception e) {
+ return e.getMessage();
+ }
+
+ }
+
+ public <T> Response exec(String functionName, String path, WebTarget service, FormDataMultiPart headers,
+ MapBuilder queryParams, MediaType inputMediaType, String outputMediaType, T inputObject) {
+
+ Class responseClass = Response.class;
+ Response cl = null;
+ try {
+ cl = (Response) exec(functionName, path, service, headers, queryParams, inputMediaType, outputMediaType,
+ responseClass, inputObject);
+ } catch (Exception e) {
+ System.out.println(e);
+ }
+ return cl;
+ }
+
+ public <T> Object execPUT(Invocation.Builder builder, MediaType inputMediaType, Class<?> responseClass,
+ T inputObject) {
+ return builder.put(Entity.entity(inputObject, inputMediaType), responseClass);
+ }
+
+ public <T> Object execGET(Invocation.Builder builder, MediaType inputMediaType, Class<?> responseClass,
+ T inputObject) {
+ return builder.get(responseClass);
+ }
+
+ public <T> Object execPOST(Invocation.Builder builder, MediaType inputMediaType, Class<?> responseClass,
+ T inputObject) {
+ return builder.post(Entity.entity(inputObject, inputMediaType), responseClass);
+ }
+
+ public <T> Object execDELETE(Invocation.Builder builder, MediaType inputMediaType, Class<?> responseClass,
+ T inputObject) {
+ return builder.delete(responseClass);
+ }
+
+ public Response sendForm(String methodName, String url, FormBuilder formData) {
+ Response response = this
+ .exec(methodName, url, servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML,
+ formData.getForm());
+ return response;
+ }
+
+ public Response sendQuery(String methodName, String url, MapBuilder query) {
+ Response response = this.exec(methodName, url, servLens, null, query);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java
new file mode 100644
index 0000000..ca59e3e
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.helpers;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.APIResult;
+import org.apache.lens.regression.core.type.FormBuilder;
+import org.apache.lens.regression.core.type.MapBuilder;
+import org.apache.lens.regression.util.AssertUtil;
+import org.apache.lens.regression.util.Util;
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.log4j.Logger;
+
+
+
+
+public class SessionHelper extends ServiceManagerHelper {
+
+ private static Logger logger = Logger.getLogger(SessionHelper.class);
+
+ private WebTarget servLens = ServiceManagerHelper.getServerLens();
+ private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
+
+ public SessionHelper() {
+ }
+
+ public SessionHelper(String envFileName) {
+ super(envFileName);
+ }
+
+ /**
+ * Open a New Session
+ *
+ * @param userName
+ * @param password
+ * @param database
+ * @return the sessionHandle String
+ */
+
+ public String openNewSession(String userName, String password, String database) throws JAXBException, LensException {
+ FormBuilder formData = new FormBuilder();
+ formData.add("username", userName);
+ formData.add("password", password);
+ if (database != null) {
+ formData.add("database", database);
+ }
+ Response response = this
+ .exec("post", "/session", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML,
+ formData.getForm());
+ AssertUtil.assertSucceededResponse(response);
+ String newSessionHandleString = response.readEntity(String.class);
+ logger.info("Session Handle String" + newSessionHandleString);
+ return newSessionHandleString;
+ }
+
+ public String openNewSession(String userName, String password) throws JAXBException, LensException {
+ return openNewSession(userName, password, null);
+ }
+
+ /**
+ * Close a Session
+ *
+ * @param sessionHandleString
+ */
+ public void closeNewSession(String sessionHandleString) throws JAXBException, LensException {
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ Response response = this.exec("delete", "/session", servLens, null, query);
+
+ APIResult result = response.readEntity(APIResult.class);
+ if (result.getStatus() == APIResult.Status.SUCCEEDED) {
+ throw new LensException("Status should be SUCCEEDED");
+ }
+ if (response.getStatus() == 200) {
+ throw new LensException("Status code should be 200");
+ }
+ if (result.getMessage() == null) {
+ throw new LensException("Status message is null");
+ }
+ logger.info("Closed Session : " + sessionHandleString);
+ }
+
+ /**
+ * Set and Validate Session Params
+ *
+ * @param sessionHandleString
+ * @param param
+ * @param value
+ */
+ public void setAndValidateParam(String sessionHandleString, String param, String value) throws Exception {
+ boolean success;
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("key", param);
+ formData.add("value", value);
+ Response response = this
+ .exec("put", "/session/params", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, null,
+ formData.getForm());
+ AssertUtil.assertSucceeded(response);
+ MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
+ query.put("key", param);
+ response = this.exec("get", "/session/params", servLens, null, query);
+ AssertUtil.assertSucceededResponse(response);
+ String responseString = response.readEntity(String.class);
+ logger.info(responseString);
+ HashMap<String, String> map = Util.stringListToMap(responseString);
+ if (!map.get(param).equals(value)) {
+ throw new LensException("Could not set property");
+ }
+ logger.info("Added property " + param + " = " + value);
+ }
+
+ public void setAndValidateParam(String param, String value) throws Exception {
+ setAndValidateParam(sessionHandleString, param, value);
+ }
+
+ public void setAndValidateParam(Map<String, String> map, String sessionHandleString) throws Exception {
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ setAndValidateParam(sessionHandleString, entry.getKey(), entry.getValue());
+ }
+ }
+
+ public void setAndValidateParam(Map<String, String> map) throws Exception {
+ setAndValidateParam(map, sessionHandleString);
+ }
+
+ /**
+ * Add resources to a session
+ *
+ * @param path
+ * @param sessionHandleString
+ */
+ public void addResourcesJar(String path, String sessionHandleString) throws JAXBException, LensException {
+ logger.info("Adding Resources " + path);
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("type", "jar");
+ formData.add("path", path);
+ Response response = this
+ .exec("put", "/session/resources/add", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, null,
+ formData.getForm());
+ logger.info("Response : " + response);
+ AssertUtil.assertSucceeded(response);
+ }
+
+ public void addResourcesJar(String path) throws JAXBException, LensException {
+ addResourcesJar(path, sessionHandleString);
+ }
+
+ /**
+ * Remove resources from a session
+ *
+ * @param path
+ * @param sessionHandleString
+ */
+ public void removeResourcesJar(String path, String sessionHandleString) throws JAXBException, LensException {
+ logger.info("Removing Resources " + path);
+ FormBuilder formData = new FormBuilder();
+ formData.add("sessionid", sessionHandleString);
+ formData.add("type", "jar");
+ formData.add("path", path);
+ Response response = this
+ .exec("put", "/session/resources/delete", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, null,
+ formData.getForm());
+ logger.info("Response : " + response);
+ AssertUtil.assertSucceeded(response);
+ }
+
+ public void removeResourcesJar(String path) throws JAXBException, LensException {
+ removeResourcesJar(path, sessionHandleString);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/type/FormBuilder.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/type/FormBuilder.java b/lens-regression/src/main/java/org/apache/lens/regression/core/type/FormBuilder.java
new file mode 100644
index 0000000..a167441
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/type/FormBuilder.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.type;
+
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.Properties;
+
+import javax.ws.rs.core.MediaType;
+
+
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+
+
+public class FormBuilder {
+ private FormDataMultiPart formData;
+
+ public FormBuilder() {
+ formData = new FormDataMultiPart();
+ }
+
+ public FormBuilder(FormDataMultiPart form) {
+ formData = new FormDataMultiPart();
+ formData = form;
+ }
+
+ public FormBuilder(String fieldName, String value) {
+ formData = new FormDataMultiPart();
+ formData.field(fieldName, value);
+ }
+
+ public FormBuilder(Properties newProperty) {
+ formData = new FormDataMultiPart();
+ Enumeration<?> e = newProperty.propertyNames();
+ while (e.hasMoreElements()) {
+ String key = (String) e.nextElement();
+ formData.field(key, newProperty.getProperty(key));
+ }
+ }
+
+ public FormBuilder(Map<String, String> map) {
+ formData = new FormDataMultiPart();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ formData.field(entry.getKey(), entry.getValue());
+ }
+ }
+
+ public void add(String fieldName, String value) {
+ formData.field(fieldName, value);
+ }
+
+ public void add(String fieldName, Object value, MediaType mediaType) {
+ formData.field(fieldName, value, mediaType);
+ }
+
+ public void addFile(String fieldName, String filename) {
+ }
+
+ public FormDataMultiPart getForm() {
+ return formData;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/type/MapBuilder.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/type/MapBuilder.java b/lens-regression/src/main/java/org/apache/lens/regression/core/type/MapBuilder.java
new file mode 100644
index 0000000..eae8a06
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/type/MapBuilder.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.type;
+
+import java.util.HashMap;
+
+public class MapBuilder {
+ private HashMap<String, String> map;
+
+ public MapBuilder() {
+ map = new HashMap<String, String>();
+ }
+
+ public MapBuilder(HashMap<String, String> h) {
+ map = h;
+ }
+
+ public MapBuilder(String key, String value) {
+ map = new HashMap<String, String>();
+ map.put(key, value);
+ }
+
+ public MapBuilder(String[] keys, String[] values) {
+ map = new HashMap<String, String>();
+ for (int i = 0; i < keys.length; i++) {
+ map.put(keys[i], values[i]);
+ }
+ }
+
+ public void put(String key, String value) {
+ map.put(key, value);
+ }
+
+ public String get(String key) {
+ return map.get(key);
+ }
+
+ public void remove(String key) {
+ map.remove(key);
+ }
+
+ public void clear() {
+ map.clear();
+ }
+
+ public HashMap<String, String> getMap() {
+ return map;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/type/PrepareQueryHandles.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/type/PrepareQueryHandles.java b/lens-regression/src/main/java/org/apache/lens/regression/core/type/PrepareQueryHandles.java
new file mode 100644
index 0000000..f04dfb8
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/type/PrepareQueryHandles.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.type;
+
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.lens.api.query.QueryPrepareHandle;
+
+
+
+@XmlRootElement(name = "queryPrepareHandles")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class PrepareQueryHandles {
+ @XmlElement(name = "queryPrepareHandle")
+ private List<QueryPrepareHandle> queryPrepareHandles = null;
+
+ public List<QueryPrepareHandle> getQueryHandles() {
+ return queryPrepareHandles;
+ }
+
+ public void setPreparedQueryHandles(List<QueryPrepareHandle> queryPrepareHandles) {
+ this.queryPrepareHandles = queryPrepareHandles;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/6a4b2470/lens-regression/src/main/java/org/apache/lens/regression/core/type/QueryHandles.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/type/QueryHandles.java b/lens-regression/src/main/java/org/apache/lens/regression/core/type/QueryHandles.java
new file mode 100644
index 0000000..94d96b2
--- /dev/null
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/type/QueryHandles.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.regression.core.type;
+
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.lens.api.query.QueryHandle;
+
+
+
+@XmlRootElement(name = "queryHandles")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class QueryHandles {
+ @XmlElement(name = "queryHandle")
+ private List<QueryHandle> queryHandles = null;
+
+ public List<QueryHandle> getQueryHandles() {
+ return queryHandles;
+ }
+
+ public void setQueryHandles(List<QueryHandle> queryHandles) {
+ this.queryHandles = queryHandles;
+ }
+}
[26/51] [abbrv] incubator-lens git commit: LENS-596: Fix
hasAggregates in GroupByResolver to look at expressions
Posted by jd...@apache.org.
LENS-596: Fix hasAggregates in GroupByResolver to look at expressions
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/022b589e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/022b589e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/022b589e
Branch: refs/heads/current-release-line
Commit: 022b589ee7c0ba4a33c3054b7b1ac116625316c4
Parents: b8995ed
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Jun 12 11:45:25 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Fri Jun 12 11:45:25 2015 +0530
----------------------------------------------------------------------
.../lens/cube/parse/ExpressionResolver.java | 16 ++++++--
.../apache/lens/cube/parse/GroupbyResolver.java | 43 +++++++++++++++++++-
.../apache/lens/cube/parse/CubeTestSetup.java | 4 ++
.../lens/cube/parse/TestExpressionResolver.java | 23 +++++++++++
4 files changed, 80 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/022b589e/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 539badb..8e199ea 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -186,6 +186,7 @@ class ExpressionResolver implements ContextRewriter {
}
evalSet.add(esc);
}
+
Set<ASTNode> getAllASTNodes() {
Set<ASTNode> allAST = new HashSet<ASTNode>();
for (ExprSpecContext esc : allExprs) {
@@ -193,6 +194,15 @@ class ExpressionResolver implements ContextRewriter {
}
return allAST;
}
+
+ boolean hasAggregates() {
+ for (ExprSpecContext esc : allExprs) {
+ if (HQLParser.hasAggregate(esc.finalAST)) {
+ return true;
+ }
+ }
+ return false;
+ }
}
static class ExprSpecContext implements TrackQueriedColumns {
@@ -311,10 +321,8 @@ class ExpressionResolver implements ContextRewriter {
boolean hasAggregates() {
for (Set<ExpressionContext> ecSet : allExprsQueried.values()) {
for (ExpressionContext ec : ecSet) {
- for (ExprSpecContext esc : ec.allExprs) {
- if (HQLParser.isAggregateAST(esc.finalAST)) {
- return true;
- }
+ if (ec.hasAggregates()) {
+ return true;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/022b589e/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
index 6a2a897..3201c09 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
@@ -24,6 +24,8 @@ import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import org.apache.lens.cube.metadata.AbstractBaseTable;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -244,7 +246,7 @@ class GroupbyResolver implements ContextRewriter {
for (int i = 0; i < selectASTNode.getChildCount(); i++) {
ASTNode childNode = (ASTNode) selectASTNode.getChild(i);
- if (hasMeasure(childNode, cubeQueryCtx) || HQLParser.hasAggregate(childNode)) {
+ if (hasMeasure(childNode, cubeQueryCtx) || hasAggregate(childNode, cubeQueryCtx)) {
continue;
}
nonMsrNonAggSelASTChildren.add(childNode);
@@ -266,7 +268,7 @@ class GroupbyResolver implements ContextRewriter {
if (hasMeasure(child, cubeql)) {
continue;
}
- if (HQLParser.hasAggregate(child)) {
+ if (hasAggregate(child, cubeql)) {
continue;
}
list.add(HQLParser.getString((ASTNode) node.getChild(i)));
@@ -275,6 +277,43 @@ class GroupbyResolver implements ContextRewriter {
return list;
}
+ boolean hasAggregate(ASTNode node, CubeQueryContext cubeql) {
+ int nodeType = node.getToken().getType();
+ if (nodeType == TOK_TABLE_OR_COL || nodeType == DOT) {
+ String colname;
+ String alias = "";
+
+ if (node.getToken().getType() == TOK_TABLE_OR_COL) {
+ colname = ((ASTNode) node.getChild(0)).getText().toLowerCase();
+ } else {
+ // node in 'alias.column' format
+ ASTNode tabident = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier);
+ ASTNode colIdent = (ASTNode) node.getChild(1);
+
+ colname = colIdent.getText().toLowerCase();
+ alias = tabident.getText().toLowerCase();
+ }
+ // by the time Groupby resolver is looking for aggregate, all columns should be aliased with correct
+ // alias name.
+ if (cubeql.getCubeTableForAlias(alias) instanceof AbstractBaseTable) {
+ if (((AbstractBaseTable)cubeql.getCubeTableForAlias(alias)).getExpressionByName(colname) != null) {
+ return cubeql.getExprCtx().getExpressionContext(colname, alias).hasAggregates();
+ }
+ }
+ } else {
+ if (HQLParser.isAggregateAST(node)) {
+ return true;
+ }
+
+ for (int i = 0; i < node.getChildCount(); i++) {
+ if (hasAggregate((ASTNode) node.getChild(i), cubeql)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
boolean hasMeasure(ASTNode node, CubeQueryContext cubeql) {
int nodeType = node.getToken().getType();
if (nodeType == TOK_TABLE_OR_COL || nodeType == DOT) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/022b589e/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 914fe1b..554e709 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -662,6 +662,8 @@ public class CubeTestSetup {
exprs = new HashSet<ExprColumn>();
exprs.add(new ExprColumn(new FieldSchema("avgmsr", "double", "avg measure"), "Avg Msr", "avg(msr1 + msr2)"));
+ exprs.add(new ExprColumn(new FieldSchema("summsrs", "double", "sum measures"), "Sum Msrs",
+ "(1000 + sum(msr1) + sum(msr2))/100"));
exprs.add(new ExprColumn(new FieldSchema("msr5", "double", "materialized in some facts"), "Fifth Msr",
"msr2 + msr3"));
exprs.add(new ExprColumn(new FieldSchema("equalsums", "double", "sums are equals"), "equalsums",
@@ -1468,6 +1470,8 @@ public class CubeTestSetup {
null), new ExprSpec("concat(citydim.name, \":\", statedim.name)", null, null)));
exprs.add(new ExprColumn(new FieldSchema("CityState", "string", "city's state"),
"City State", new ExprSpec("concat(citydim.name, \":\", citydim.statename)", null, null)));
+ exprs.add(new ExprColumn(new FieldSchema("AggrExpr", "int", "count(name)"), "city count",
+ new ExprSpec("count(name)", null, null)));
Dimension cityDim = new Dimension("citydim", cityAttrs, exprs, dimProps, 0L);
client.createDimension(cityDim);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/022b589e/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
index 5cb7b0a..7f872e9 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestExpressionResolver.java
@@ -167,6 +167,20 @@ public class TestExpressionResolver extends TestQueryRewrite {
TestCubeRewriter.compareQueries(hqlQuery, expected);
}
+
+ @Test
+ public void testExpressionInSelectToGroupbyWithComplexExpression() throws Exception {
+ String hqlQuery =
+ rewrite("select booleancut, summsrs from testCube" + " where " + TWO_DAYS_RANGE + " and substrexpr != 'XYZ'",
+ conf);
+ String expected =
+ getExpectedQuery(cubeName, "select testCube.dim1 != 'x' AND testCube.dim2 != 10 ,"
+ + " ((1000 + sum(testCube.msr1) + sum(testCube.msr2))/100) FROM ", null,
+ " and substr(testCube.dim1, 3) != 'XYZ' group by testCube.dim1 != 'x' AND testCube.dim2 != 10",
+ getWhereForHourly2days("C1_testfact2_raw"));
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+ }
+
@Test
public void testExpressionToJoin() throws Exception {
// expression which results in join
@@ -398,6 +412,15 @@ public class TestExpressionResolver extends TestQueryRewrite {
}
@Test
+ public void testDimensionQueryExpressionInSelectToGroupby() throws Exception {
+ String hqlQuery = rewrite("select id, AggrExpr from citydim", conf);
+ String expected = getExpectedQuery("citydim", "select citydim.id, count(citydim.name) FROM ", null, null,
+ " group by citydim.id", "c1_citytable", true);
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+
+ }
+
+ @Test
public void testDimensionQueryWithTableAliasColumnAlias() throws Exception {
String hqlQuery = rewrite("select ct.name cname, ct.cityaddress caddr from" + " citydim ct", conf);
[03/51] [abbrv] incubator-lens git commit: LENS-551: Cleanup
testcases of exception stack traces
Posted by jd...@apache.org.
LENS-551: Cleanup testcases of exception stack traces
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b0783db7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b0783db7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b0783db7
Branch: refs/heads/current-release-line
Commit: b0783db7976d207b6544dcf2d9f8e5685994f9b4
Parents: 82553db
Author: Yash Sharma <ya...@gmail.com>
Authored: Wed May 27 20:01:26 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed May 27 20:01:26 2015 +0530
----------------------------------------------------------------------
checkstyle/src/main/resources/checkstyle.xml | 7 +
.../lens/cli/commands/BaseLensCommand.java | 10 +-
.../cli/commands/LensConnectionCommands.java | 7 +-
.../lens/cli/TestLensDimensionCommands.java | 16 +-
.../cli/TestLensDimensionTableCommands.java | 24 ++-
.../apache/lens/cli/TestLensFactCommands.java | 27 ++-
.../apache/lens/cli/TestLensQueryCommands.java | 31 ++-
.../apache/lens/client/jdbc/LensJdbcDriver.java | 6 +-
.../org/apache/lens/cube/parse/HQLParser.java | 7 +-
.../timeline/TestPartitionTimelines.java | 6 +-
.../apache/lens/cube/parse/CubeTestSetup.java | 6 +-
.../lens/cube/parse/TestCubeRewriter.java | 7 +-
.../apache/lens/cube/parse/TestDateUtil.java | 6 +-
.../apache/lens/cube/parse/TestHQLParser.java | 6 +-
.../lens/cube/parse/TestMaxUpdateInterval.java | 6 +-
.../lens/cube/parse/TestQueryRewrite.java | 3 +-
.../lens/cube/parse/TestTimeRangeWriter.java | 5 +-
.../parse/TestTimeRangeWriterWithQuery.java | 6 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 95 +++++----
.../lens/driver/hive/HiveInMemoryResultSet.java | 5 +-
.../apache/lens/driver/hive/HiveQueryPlan.java | 5 +-
.../lens/driver/hive/TestRemoteHiveDriver.java | 27 ++-
.../driver/jdbc/TestColumnarSQLRewriter.java | 7 +-
.../jdbc/TestDataSourceConnectionProvider.java | 15 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 32 ++--
.../apache/lens/examples/SampleMetastore.java | 6 +-
.../org/apache/lens/client/LensMLClient.java | 9 +-
.../apache/lens/ml/algo/lib/AlgoArgParser.java | 11 +-
.../org/apache/lens/ml/impl/LensMLImpl.java | 71 ++++---
.../java/org/apache/lens/ml/TestMLResource.java | 25 ++-
.../apache/lens/server/LensRequestListener.java | 5 +-
.../lens/server/LensServletContextListener.java | 5 +-
.../metastore/CubeMetastoreServiceImpl.java | 76 ++++----
.../apache/lens/server/metastore/JAXBUtils.java | 12 +-
.../lens/server/metrics/MetricsServiceImpl.java | 17 +-
.../apache/lens/server/query/LensServerDAO.java | 21 +-
.../server/query/QueryExecutionServiceImpl.java | 191 +++++++++----------
.../lens/server/query/QueryServiceResource.java | 16 +-
.../lens/server/session/HiveSessionService.java | 50 +++--
.../event/query/QueryExecutionStatistics.java | 6 +-
.../org/apache/lens/server/LensTestUtil.java | 6 +-
.../server/metastore/TestMetastoreService.java | 18 +-
.../lens/server/query/TestQueryService.java | 22 +--
.../session/TestDatabaseResourceService.java | 10 +-
.../stats/TestStatisticsLogFileScannerTask.java | 5 +-
45 files changed, 498 insertions(+), 456 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/checkstyle/src/main/resources/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml
index d6ff839..031d664 100644
--- a/checkstyle/src/main/resources/checkstyle.xml
+++ b/checkstyle/src/main/resources/checkstyle.xml
@@ -245,6 +245,13 @@
<property name="checkFormat" value="ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck|InnerAssignmentCheck"/>
</module>
+ <module name="RegexpSingleline">
+ <property name="severity" value="error" />
+ <property name="format" value="\w*.printStackTrace\w*" />
+ <property name="message" value="Print stack trace not allowed. Use Logger." />
+ <property name="fileExtensions" value="java" />
+ </module>
+
<!--module name="SuppressionFilter">
<property name="file" value="checkstyle/src/main/resources/suppressions.xml"/>
</module-->
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
index d6055cd..3deddfa 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
@@ -27,8 +27,6 @@ import java.util.Date;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientSingletonWrapper;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.JsonGenerator;
@@ -41,9 +39,12 @@ import org.springframework.shell.event.ParseResult;
import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class BaseLensCommand.
*/
+@Slf4j
public class BaseLensCommand implements ExecutionProcessor {
/** The mapper. */
@@ -52,9 +53,6 @@ public class BaseLensCommand implements ExecutionProcessor {
/** The pp. */
protected DefaultPrettyPrinter pp;
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(BaseLensCommand.class);
-
/** The is connection active. */
protected static boolean isConnectionActive;
public static final String DATE_FMT = "yyyy-MM-dd'T'HH:mm:ss:SSS";
@@ -84,7 +82,7 @@ public class BaseLensCommand implements ExecutionProcessor {
*/
protected static synchronized void closeClientConnection() {
if (isConnectionActive) {
- LOG.debug("Request for stopping lens cli received");
+ log.debug("Request for stopping lens cli received");
getClient().closeConnection();
isConnectionActive = false;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
index d727ecc..675e6c8 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
@@ -32,9 +32,13 @@ import org.springframework.stereotype.Component;
import com.google.common.base.Joiner;
+import lombok.extern.slf4j.Slf4j;
+
+
/**
* The Class LensConnectionCommands.
*/
+@Slf4j
@Component
@UserDocumentation(title = "Session management",
description = "Opening the lens CLI shell is equivalent to open a session with lens server."
@@ -159,8 +163,7 @@ public class LensConnectionCommands extends BaseLensCommand {
closeClientConnection();
return ExitShellRequest.NORMAL_EXIT;
} catch (ProcessingException e) {
- System.out.println(e.getMessage());
- LOG.error(e);
+ log.error("Error while closing client connection.", e);
return ExitShellRequest.FATAL_EXIT;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
index 5df3486..208eb78 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionCommands.java
@@ -25,19 +25,17 @@ import java.net.URL;
import org.apache.lens.cli.commands.LensDimensionCommands;
import org.apache.lens.client.LensClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestLensDimensionCommands.
*/
+@Slf4j
public class TestLensDimensionCommands extends LensCliApplicationTest {
- /** The Constant LOG. */
- private static final Logger LOG = LoggerFactory.getLogger(TestLensDimensionCommands.class);
-
/** The command. */
private static LensDimensionCommands command = null;
@@ -69,7 +67,7 @@ public class TestLensDimensionCommands extends LensCliApplicationTest {
*/
@Test
public void testDimensionCommands() throws Exception {
- LOG.debug("Starting to test dimension commands");
+ log.debug("Starting to test dimension commands");
URL dimensionSpec = TestLensDimensionCommands.class.getClassLoader().getResource("test-dimension.xml");
String dimensionList = getCommand().showDimensions();
Assert.assertFalse(dimensionList.contains("test_dim"));
@@ -114,14 +112,14 @@ public class TestLensDimensionCommands extends LensCliApplicationTest {
writer.close();
String desc = command.describeDimension("test_dim");
- LOG.debug(desc);
+ log.debug(desc);
String propString = "name : test_dim.prop value : test";
String propString1 = "name : test_dim.prop1 value : test1";
Assert.assertTrue(desc.contains(propString));
command.updateDimension("test_dim", "/tmp/test_dim1.xml");
desc = command.describeDimension("test_dim");
- LOG.debug(desc);
+ log.debug(desc);
Assert.assertTrue(desc.contains(propString));
Assert.assertTrue(desc.contains(propString1));
@@ -129,7 +127,7 @@ public class TestLensDimensionCommands extends LensCliApplicationTest {
newFile.delete();
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Testing update dimension failed with exception", t);
Assert.fail("Testing update dimension failed with exception" + t.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
index a98f183..a87d0b7 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
@@ -28,19 +28,17 @@ import org.apache.lens.cli.commands.LensDimensionCommands;
import org.apache.lens.cli.commands.LensDimensionTableCommands;
import org.apache.lens.client.LensClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestLensDimensionTableCommands.
*/
+@Slf4j
public class TestLensDimensionTableCommands extends LensCliApplicationTest {
- /** The Constant LOG. */
- private static final Logger LOG = LoggerFactory.getLogger(TestLensDimensionTableCommands.class);
-
/** The Constant DIM_LOCAL. */
public static final String DIM_LOCAL = "dim_local";
@@ -116,7 +114,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
try {
command.createDimensionTable(new File(dimSpec.toURI()).getAbsolutePath());
} catch (Exception e) {
- e.printStackTrace();
+ log.error("Unable to create dimtable", e);
Assert.fail("Unable to create dimtable" + e.getMessage());
}
@@ -126,13 +124,13 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
Assert.assertEquals(command.showDimensionTables("blah"), dimList);
Assert.fail();
} catch (NotFoundException e) {
- LOG.info("blah is not a table", e);
+ log.info("blah is not a table", e);
}
try {
Assert.assertEquals(command.showDimensionTables("dim_table2"), dimList);
Assert.fail();
} catch (NotFoundException e) {
- LOG.info("dim_table2 is a table, but not a dimension", e);
+ log.info("dim_table2 is a table, but not a dimension", e);
}
Assert.assertTrue(dimList.contains(tableName), "dim_table table should be found");
}
@@ -166,7 +164,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
writer.close();
String desc = command.describeDimensionTable("dim_table2");
- LOG.debug(desc);
+ log.debug(desc);
String propString = "name : dim2.prop value : d2";
String propString1 = "name : dim2.prop value : d1";
String propString2 = "name : dim2.prop1 value : d2";
@@ -174,7 +172,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
command.updateDimensionTable("dim_table2", "/tmp/local-dim1.xml");
desc = command.describeDimensionTable("dim_table2");
- LOG.debug(desc);
+ log.debug(desc);
Assert.assertTrue(desc.contains(propString1));
Assert.assertTrue(desc.contains(propString2));
@@ -234,7 +232,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
command.addPartitionToDimtable("dim_table2", DIM_LOCAL, new File(
TestLensFactCommands.class.getClassLoader().getResource("dim1-local-part.xml").toURI()).getAbsolutePath());
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
}
verifyAndDeletePartition();
@@ -242,7 +240,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
command.addPartitionsToDimtable("dim_table2", DIM_LOCAL, new File(
TestLensFactCommands.class.getClassLoader().getResource("dim1-local-parts.xml").toURI()).getAbsolutePath());
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
}
verifyAndDeletePartition();
@@ -270,7 +268,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
try {
command.addPartitionToDimtable(tableName, storageName, new File(resource.toURI()).getAbsolutePath());
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
index 19979cc..1ad234d 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
@@ -31,18 +31,16 @@ import org.apache.lens.cli.commands.LensCubeCommands;
import org.apache.lens.cli.commands.LensFactCommands;
import org.apache.lens.client.LensClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestLensFactCommands.
*/
+@Slf4j
public class TestLensFactCommands extends LensCliApplicationTest {
- /** The Constant LOG. */
- private static final Logger LOG = LoggerFactory.getLogger(TestLensFactCommands.class);
-
/** The Constant FACT_LOCAL. */
public static final String FACT_LOCAL = "fact_local";
@@ -119,13 +117,13 @@ public class TestLensFactCommands extends LensCliApplicationTest {
assertEquals(command.showFacts("blah"), factList);
fail();
} catch (NotFoundException e) {
- LOG.info("blah is not a table", e);
+ log.info("blah is not a table", e);
}
try {
assertEquals(command.showFacts("fact1"), factList);
fail();
} catch (NotFoundException e) {
- LOG.info("fact1 is a table, but not a cube table", e);
+ log.info("fact1 is a table, but not a cube table", e);
}
assertEquals("fact1", factList, "Fact1 table should be found");
}
@@ -157,7 +155,7 @@ public class TestLensFactCommands extends LensCliApplicationTest {
writer.close();
String desc = command.describeFactTable("fact1");
- LOG.debug(desc);
+ log.debug(desc);
String propString = "name : fact1.prop value : f1";
String propString1 = "name : fact1.prop1 value : f2";
@@ -165,15 +163,14 @@ public class TestLensFactCommands extends LensCliApplicationTest {
command.updateFactTable("fact1", "/tmp/local-fact1.xml");
desc = command.describeFactTable("fact1");
- LOG.debug(desc);
+ log.debug(desc);
assertTrue(desc.contains(propString), "The sample property value is not set");
-
assertTrue(desc.contains(propString1), "The sample property value is not set");
newFile.delete();
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Updating of the fact1 table failed with ", t);
fail("Updating of the fact1 table failed with " + t.getMessage());
}
@@ -208,7 +205,7 @@ public class TestLensFactCommands extends LensCliApplicationTest {
try {
command.addNewFactStorage("fact1", new File(resource.toURI()).getAbsolutePath());
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Unable to locate the storage part file for adding new storage to fact table fact1", t);
fail("Unable to locate the storage part file for adding new storage to fact table fact1");
}
result = command.getFactStorages("fact1");
@@ -233,15 +230,15 @@ public class TestLensFactCommands extends LensCliApplicationTest {
command.addPartitionToFact("fact1", FACT_LOCAL, new File(
TestLensFactCommands.class.getClassLoader().getResource("fact1-local-part.xml").toURI()).getAbsolutePath());
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Unable to locate the storage part file for adding new storage to fact table fact1", t);
fail("Unable to locate the storage part file for adding new storage to fact table fact1");
}
verifyAndDeletePartitions();
try {
command.addPartitionsToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath());
+ TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath());
} catch (Throwable t) {
- t.printStackTrace();
+ log.error("Unable to locate the storage part file for adding new storage to fact table fact1", t);
fail("Unable to locate the storage part file for adding new storage to fact table fact1");
}
verifyAndDeletePartitions();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index 5724d24..32a89ac 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -43,18 +43,16 @@ import org.apache.lens.driver.hive.TestHiveDriver;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestLensQueryCommands.
*/
+@Slf4j
public class TestLensQueryCommands extends LensCliApplicationTest {
- /** The Constant LOG. */
- private static final Logger LOG = LoggerFactory.getLogger(TestLensQueryCommands.class);
-
/** The client. */
private LensClient client;
@@ -130,29 +128,29 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
result = qCom.executePreparedQuery(qh, false, "testPrepQuery1");
- LOG.warn("XXXXXX Prepared query sync result is " + result);
+ log.warn("XXXXXX Prepared query sync result is " + result);
assertTrue(result.contains("1\tfirst"));
String handle = qCom.executePreparedQuery(qh, true, "testPrepQuery2");
- LOG.debug("Perpared query handle is " + handle);
+ log.debug("Perpared query handle is " + handle);
while (!client.getQueryStatus(handle).finished()) {
Thread.sleep(5000);
}
String status = qCom.getStatus(handle);
- LOG.debug("Prepared Query Status is " + status);
+ log.debug("Prepared Query Status is " + status);
assertTrue(status.contains("Status : SUCCESSFUL"));
result = qCom.getQueryResults(handle, null);
- LOG.debug("Prepared Query Result is " + result);
+ log.debug("Prepared Query Result is " + result);
assertTrue(result.contains("1\tfirst"));
// Fetch again.
result = qCom.getQueryResults(handle, null);
- LOG.debug("Prepared Query Result is " + result);
+ log.debug("Prepared Query Result is " + result);
assertTrue(result.contains("1\tfirst"));
result = qCom.destroyPreparedQuery(qh);
- LOG.debug("destroy result is " + result);
+ log.debug("destroy result is " + result);
assertEquals("Successfully destroyed " + qh, result);
result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, Long.MAX_VALUE);
@@ -195,9 +193,8 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String sql = "cube select id, name from test_dim";
String result = qCom.explainQuery(sql, "");
- LOG.debug(result);
+ log.debug(result);
assertTrue(result.contains(explainPlan));
-
}
/**
@@ -210,7 +207,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String sql = "cube select id2, name from test_dim";
String result = qCom.explainQuery(sql, "");
- LOG.debug(result);
+ log.debug(result);
assertTrue(result.contains("Explain FAILED:"));
result = qCom.explainAndPrepare(sql, "");
@@ -318,7 +315,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
LensCubeCommands command = new LensCubeCommands();
command.setClient(client);
- LOG.debug("Starting to test cube commands");
+ log.debug("Starting to test cube commands");
URL cubeSpec = TestLensQueryCommands.class.getClassLoader().getResource("sample-cube.xml");
command.createCube(new File(cubeSpec.toURI()).getAbsolutePath());
TestLensDimensionCommands.createDimension();
@@ -367,7 +364,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
assertNotNull(result);
assertFalse(result.contains("Failed to get resultset"));
} catch (Exception exc) {
- exc.printStackTrace();
+ log.error("Exception not expected while getting resultset.", exc);
fail("Exception not expected: " + exc.getMessage());
}
System.out.println("@@END_PERSISTENT_RESULT_TEST-------------");
@@ -397,7 +394,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
// This is to check for positive processing time
assertFalse(result.contains("(-"));
} catch (Exception exc) {
- exc.printStackTrace();
+ log.error("Exception not expected while purging resultset.", exc);
fail("Exception not expected: " + exc.getMessage());
}
System.out.println("@@END_FINISHED_PURGED_RESULT_TEST-------------");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcDriver.java b/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcDriver.java
index a6fae87..056a0b2 100644
--- a/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcDriver.java
+++ b/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcDriver.java
@@ -23,16 +23,20 @@ import java.util.Properties;
import java.util.logging.Logger;
import java.util.regex.Pattern;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Top level JDBC driver for Lens.
*/
+@Slf4j
public class LensJdbcDriver implements Driver {
+
static {
try {
DriverManager.registerDriver(new LensJdbcDriver());
} catch (SQLException e) {
- e.printStackTrace();
+ log.error("Error in registering jdbc driver", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
index 1c375ed..7b99310 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/HQLParser.java
@@ -40,7 +40,12 @@ import org.antlr.runtime.tree.Tree;
import com.google.common.base.Optional;
+import lombok.extern.slf4j.Slf4j;
+
+
+@Slf4j
public final class HQLParser {
+
private HQLParser() {
}
@@ -168,7 +173,7 @@ public final class HQLParser {
try {
printAST(getHiveTokenMapping(), node, 0, 0);
} catch (Exception e) {
- e.printStackTrace();
+ log.error("Error in printing AST.", e);
}
System.out.println();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
index 0231e8b..0268bbf 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/timeline/TestPartitionTimelines.java
@@ -30,7 +30,11 @@ import org.testng.annotations.Test;
import com.beust.jcommander.internal.Lists;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class TestPartitionTimelines {
+
private static final String TABLE_NAME = "storage_fact";
public static final UpdatePeriod PERIOD = UpdatePeriod.HOURLY;
private static final String PART_COL = "pt";
@@ -103,7 +107,7 @@ public class TestPartitionTimelines {
return clz.getConstructor(String.class, UpdatePeriod.class, String.class)
.newInstance(TABLE_NAME, PERIOD, PART_COL);
} catch (Exception e) {
- e.printStackTrace();
+ log.error("Error while getting instance.", e);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 1fb8556..914fe1b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -1331,9 +1331,9 @@ public class CubeTestSetup {
try {
client.addPartition(sPartSpec, c1);
} catch (HiveException e) {
- e.printStackTrace();
+ log.error("Encountered Hive exception.", e);
} catch (LensException e) {
- e.printStackTrace();
+ log.error("Encountered Lens exception.", e);
}
cal.add(Calendar.HOUR_OF_DAY, 1);
temp = cal.getTime();
@@ -2047,7 +2047,7 @@ public class CubeTestSetup {
createStateTable(client);
createCubeFactsWithValidColumns(client);
} catch (Exception exc) {
- exc.printStackTrace();
+ log.error("Exception while creating sources.", exc);
throw exc;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 9274ccf..0f1ad6e 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -44,6 +44,9 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class TestCubeRewriter extends TestQueryRewrite {
private final String cubeName = CubeTestSetup.TEST_CUBE_NAME;
@@ -1451,7 +1454,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
assertTrue(hql.contains("`a measure`"));
System.out.println("@@ hql: " + hql);
} catch (NullPointerException npe) {
- npe.printStackTrace();
+ log.error("Not expecting null pointer exception", npe);
fail("Not expecting null pointer exception");
}
}
@@ -1502,7 +1505,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
System.out.println("@@HQL " + hql);
} catch (NullPointerException npe) {
fail(npe.getMessage());
- npe.printStackTrace();
+ log.error("Not expecting null pointer exception", npe);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
index a916dbf..6717b5b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDateUtil.java
@@ -45,10 +45,14 @@ import org.testng.annotations.Test;
import com.beust.jcommander.internal.Sets;
import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Unit tests for cube DateUtil class TestDateUtil.
*/
+@Slf4j
public class TestDateUtil {
+
public static final String[] TEST_PAIRS = {
"2013-Jan-01", "2013-Jan-31", "2013-Jan-01", "2013-May-31",
"2013-Jan-01", "2013-Dec-31", "2013-Feb-01", "2013-Apr-25",
@@ -67,7 +71,7 @@ public class TestDateUtil {
try {
pairs[i] = DATE_FMT.parse(TEST_PAIRS[i]);
} catch (ParseException e) {
- e.printStackTrace();
+ log.error("Parsing exception while setup.", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
index fef634d..b0b2e88 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestHQLParser.java
@@ -31,7 +31,11 @@ import org.apache.hadoop.hive.ql.parse.ParseException;
import org.testng.Assert;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class TestHQLParser {
+
HiveConf conf = new HiveConf();
@Test
public void testGroupByOrderByGetString() throws Exception {
@@ -238,7 +242,7 @@ public class TestHQLParser {
try {
ASTNode tree = HQLParser.parseHQL(query, conf);
} catch (NullPointerException exc) {
- exc.printStackTrace();
+ log.error("should not have thrown npe", exc);
Assert.fail("should not have thrown npe");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestMaxUpdateInterval.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestMaxUpdateInterval.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestMaxUpdateInterval.java
index a5f64ec..7e5f38d 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestMaxUpdateInterval.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestMaxUpdateInterval.java
@@ -32,10 +32,14 @@ import org.apache.lens.cube.metadata.UpdatePeriod;
import org.testng.Assert;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/*
* Unit test for maxUpdateIntervalIn method in CubeFactTable
*/
+@Slf4j
public class TestMaxUpdateInterval {
+
public static final String[] TEST_PAIRS = {
"2013-Jan-01", "2013-Jan-31", "2013-Jan-01", "2013-May-31",
"2013-Jan-01", "2013-Dec-31", "2013-Feb-01", "2013-Apr-25",
@@ -54,7 +58,7 @@ public class TestMaxUpdateInterval {
pairs[i] = DATE_FMT.parse(TEST_PAIRS[i]);
System.out.println(pairs[i].toString());
} catch (ParseException e) {
- e.printStackTrace();
+ log.error("Parsing exception while testing max update interval.", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
index 994a244..d16f6a5 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryRewrite.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
+
import org.testng.Assert;
import org.testng.annotations.AfterSuite;
import org.testng.annotations.BeforeClass;
@@ -103,7 +104,7 @@ public abstract class TestQueryRewrite {
// unreachable
return null;
} catch (SemanticException e) {
- e.printStackTrace();
+ log.error("Semantic exception in Rewrite.", e);
return e;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java
index 106660e..453a102 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriter.java
@@ -33,6 +33,9 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.testng.Assert;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public abstract class TestTimeRangeWriter {
public abstract TimeRangeWriter getTimerangeWriter();
@@ -59,7 +62,7 @@ public abstract class TestTimeRangeWriter {
try {
whereClause = getTimerangeWriter().getTimeRangeWhereClause(null, "test", answeringParts);
} catch (SemanticException e) {
- e.printStackTrace();
+ log.error("Semantic exception while testing disjoint parts.", e);
th = e;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
index b5aaeb2..0e22ce2 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
@@ -36,6 +36,10 @@ import org.testng.Assert;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
+
+@Slf4j
public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
private Configuration conf;
@@ -76,7 +80,7 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
rewrite("cube select" + " SUM(msr2) from testCube where " + TWO_DAYS_RANGE, conf);
} catch (SemanticException e) {
th = e;
- e.printStackTrace();
+ log.error("Semantic exception while testing cube query.", e);
}
if (!CubeTestSetup.isZerothHour()) {
Assert.assertNotNull(th);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 8bec5e0..052d0e2 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -61,19 +61,18 @@ import org.apache.hive.service.cli.*;
import org.apache.hive.service.cli.thrift.TOperationHandle;
import org.apache.hive.service.cli.thrift.TProtocolVersion;
import org.apache.hive.service.cli.thrift.TSessionHandle;
-import org.apache.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class HiveDriver.
*/
+@Slf4j
public class HiveDriver implements LensDriver {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(HiveDriver.class);
-
/** The Constant HIVE_CONNECTION_CLASS. */
public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
@@ -186,15 +185,15 @@ public class HiveDriver implements LensDriver {
if (thConn != null) {
try {
- LOG.info("Closed connection:" + expired.getConnId());
+ log.info("Closed connection:" + expired.getConnId());
thConn.close();
} catch (IOException e) {
- LOG.error("Error closing connection", e);
+ log.error("Error closing connection", e);
}
}
}
} catch (InterruptedException intr) {
- LOG.warn("Connection expiry thread interrupted", intr);
+ log.warn("Connection expiry thread interrupted", intr);
return;
}
}
@@ -309,7 +308,7 @@ public class HiveDriver implements LensDriver {
connectionExpiryThread.setName("HiveDriver-ConnectionExpiryThread");
connectionExpiryThread.start();
driverListeners = new ArrayList<LensEventListener<DriverEvent>>();
- LOG.info("Hive driver inited");
+ log.info("Hive driver inited");
}
@Override
@@ -350,7 +349,7 @@ public class HiveDriver implements LensDriver {
@Override
public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
- LOG.info("Estimate: " + qctx.getDriverQuery(this));
+ log.info("Estimate: " + qctx.getDriverQuery(this));
if (qctx.getDriverQuery(this) == null) {
throw new NullPointerException("Null driver query for " + qctx.getUserQuery());
}
@@ -382,7 +381,7 @@ public class HiveDriver implements LensDriver {
// explain called again and again
return (HiveQueryPlan) explainCtx.getDriverContext().getDriverQueryPlan(this);
}
- LOG.info("Explain: " + explainCtx.getDriverQuery(this));
+ log.info("Explain: " + explainCtx.getDriverQuery(this));
Configuration explainConf = new Configuration(explainCtx.getDriverConf(this));
explainConf.setClassLoader(explainCtx.getConf().getClassLoader());
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
@@ -461,7 +460,7 @@ public class HiveDriver implements LensDriver {
qdconf.set("mapred.job.name", ctx.getQueryHandle().toString());
OperationHandle op = getClient().executeStatement(getSession(ctx), ctx.getSelectedDriverQuery(),
qdconf.getValByRegex(".*"));
- LOG.info("The hive operation handle: " + op);
+ log.info("The hive operation handle: " + op);
ctx.setDriverOpHandle(op.toString());
hiveHandles.put(ctx.getQueryHandle(), op);
updateStatus(ctx);
@@ -498,24 +497,24 @@ public class HiveDriver implements LensDriver {
Configuration qdconf = ctx.getDriverConf(this);
qdconf.set("mapred.job.name", ctx.getQueryHandle().toString());
//Query is already explained.
- LOG.info("whetherCalculatePriority: " + whetherCalculatePriority);
+ log.info("whetherCalculatePriority: " + whetherCalculatePriority);
if (whetherCalculatePriority) {
try {
// Inside try since non-data fetching queries can also be executed by async method.
String priority = queryPriorityDecider.decidePriority(ctx).toString();
qdconf.set("mapred.job.priority", priority);
- LOG.info("set priority to " + priority);
+ log.info("set priority to " + priority);
} catch (Exception e) {
// not failing query launch when setting priority fails
// priority will be set to usually NORMAL - the default in underlying system.
- LOG.error("could not set priority for lens session id:" + ctx.getLensSessionIdentifier()
+ log.error("could not set priority for lens session id:" + ctx.getLensSessionIdentifier()
+ "User query: " + ctx.getUserQuery(), e);
}
}
OperationHandle op = getClient().executeStatementAsync(getSession(ctx), ctx.getSelectedDriverQuery(),
qdconf.getValByRegex(".*"));
ctx.setDriverOpHandle(op.toString());
- LOG.info("QueryHandle: " + ctx.getQueryHandle() + " HiveHandle:" + op);
+ log.info("QueryHandle: " + ctx.getQueryHandle() + " HiveHandle:" + op);
hiveHandles.put(ctx.getQueryHandle(), op);
} catch (IOException e) {
throw new LensException("Error adding persistent path", e);
@@ -532,7 +531,7 @@ public class HiveDriver implements LensDriver {
*/
@Override
public void updateStatus(QueryContext context) throws LensException {
- LOG.debug("GetStatus: " + context.getQueryHandle());
+ log.debug("GetStatus: " + context.getQueryHandle());
if (context.getDriverStatus().isFinished()) {
return;
}
@@ -540,9 +539,9 @@ public class HiveDriver implements LensDriver {
ByteArrayInputStream in = null;
try {
// Get operation status from hive server
- LOG.debug("GetStatus hiveHandle: " + hiveHandle);
+ log.debug("GetStatus hiveHandle: " + hiveHandle);
OperationStatus opStatus = getClient().getOperationStatus(hiveHandle);
- LOG.debug("GetStatus on hiveHandle: " + hiveHandle + " returned state:" + opStatus.getState().name());
+ log.debug("GetStatus on hiveHandle: " + hiveHandle + " returned state:" + opStatus.getState().name());
switch (opStatus.getState()) {
case CANCELED:
@@ -605,7 +604,7 @@ public class HiveDriver implements LensDriver {
progress = taskStatuses.size() == 0 ? 0 : (float) completedTasks / taskStatuses.size();
errorMsg = errorMessage.toString();
} else {
- LOG.warn("Empty task statuses");
+ log.warn("Empty task statuses");
}
String error = null;
if (StringUtils.isNotBlank(errorMsg)) {
@@ -619,7 +618,7 @@ public class HiveDriver implements LensDriver {
context.getDriverStatus().setDriverStartTime(opStatus.getOperationStarted());
context.getDriverStatus().setDriverFinishTime(opStatus.getOperationCompleted());
} catch (Exception e) {
- LOG.error("Error getting query status", e);
+ log.error("Error getting query status", e);
handleHiveServerError(context, e);
throw new LensException("Error getting query status", e);
} finally {
@@ -627,7 +626,7 @@ public class HiveDriver implements LensDriver {
try {
in.close();
} catch (IOException e) {
- e.printStackTrace();
+ log.error("Error closing stream.", e);
}
}
}
@@ -654,7 +653,7 @@ public class HiveDriver implements LensDriver {
*/
@Override
public LensResultSet fetchResultSet(QueryContext ctx) throws LensException {
- LOG.info("FetchResultSet: " + ctx.getQueryHandle());
+ log.info("FetchResultSet: " + ctx.getQueryHandle());
// This should be applicable only for a async query
return createResultSet(ctx, false);
}
@@ -679,10 +678,10 @@ public class HiveDriver implements LensDriver {
if (handle == null) {
return;
}
- LOG.info("CloseQuery: " + handle);
+ log.info("CloseQuery: " + handle);
OperationHandle opHandle = hiveHandles.remove(handle);
if (opHandle != null) {
- LOG.info("CloseQuery: " + opHandle);
+ log.info("CloseQuery: " + opHandle);
try {
getClient().closeOperation(opHandle);
} catch (HiveSQLException e) {
@@ -699,10 +698,10 @@ public class HiveDriver implements LensDriver {
*/
@Override
public boolean cancelQuery(QueryHandle handle) throws LensException {
- LOG.info("CancelQuery: " + handle);
+ log.info("CancelQuery: " + handle);
OperationHandle hiveHandle = getHiveHandle(handle);
try {
- LOG.info("CancelQuery hiveHandle: " + hiveHandle);
+ log.info("CancelQuery hiveHandle: " + hiveHandle);
getClient().cancelOperation(hiveHandle);
return true;
} catch (HiveSQLException e) {
@@ -718,7 +717,7 @@ public class HiveDriver implements LensDriver {
*/
@Override
public void close() {
- LOG.info("CloseDriver");
+ log.info("CloseDriver");
// Close this driver and release all resources
sessionLock.lock();
try {
@@ -727,7 +726,7 @@ public class HiveDriver implements LensDriver {
getClient().closeSession(lensToHiveSession.get(lensSessionDbKey));
} catch (Exception e) {
checkInvalidSession(e);
- LOG.warn("Error closing session for lens session: " + lensSessionDbKey + ", hive session: "
+ log.warn("Error closing session for lens session: " + lensSessionDbKey + ", hive session: "
+ lensToHiveSession.get(lensSessionDbKey), e);
}
}
@@ -756,7 +755,7 @@ public class HiveDriver implements LensDriver {
} catch (Exception e) {
throw new LensException(e);
}
- LOG.info("New thrift connection " + connectionClass);
+ log.info("New thrift connection " + connectionClass);
}
return embeddedConnection.getClient();
} else {
@@ -774,7 +773,7 @@ public class HiveDriver implements LensDriver {
connection = new ExpirableConnection(tconn, connectionExpiryTimeout);
thriftConnExpiryQueue.offer(connection);
threadConnections.put(connectionKey, connection);
- LOG.info("New thrift connection " + connectionClass + " for thread:" + Thread.currentThread().getId()
+ log.info("New thrift connection " + connectionClass + " for thread:" + Thread.currentThread().getId()
+ " for user:" + user + " connection ID=" + connection.getConnId());
} catch (Exception e) {
throw new LensException(e);
@@ -799,7 +798,7 @@ public class HiveDriver implements LensDriver {
*/
private LensResultSet createResultSet(QueryContext context, boolean closeAfterFetch) throws LensException {
OperationHandle op = getHiveHandle(context.getQueryHandle());
- LOG.info("Creating result set for hiveHandle:" + op);
+ log.info("Creating result set for hiveHandle:" + op);
try {
if (context.isDriverPersistent()) {
return new HivePersistentResultSet(new Path(context.getHdfsoutPath()), op, getClient());
@@ -846,7 +845,7 @@ public class HiveDriver implements LensDriver {
context.unSetDriverPersistent();
hiveQuery = context.getSelectedDriverQuery();
}
- LOG.info("Hive driver query:" + hiveQuery);
+ log.info("Hive driver query:" + hiveQuery);
context.setSelectedDriverQuery(hiveQuery);
}
@@ -875,14 +874,14 @@ public class HiveDriver implements LensDriver {
try {
hiveSession = getClient().openSession(ctx.getClusterUser(), "");
lensToHiveSession.put(sessionDbKey, hiveSession);
- LOG.info("New hive session for user: " + ctx.getClusterUser() + ", lens session: " + sessionDbKey
+ log.info("New hive session for user: " + ctx.getClusterUser() + ", lens session: " + sessionDbKey
+ " hive session handle: " + hiveSession.getHandleIdentifier());
for (LensEventListener<DriverEvent> eventListener : driverListeners) {
try {
eventListener.onEvent(new DriverSessionStarted(System.currentTimeMillis(), this, lensSession, hiveSession
.getSessionId().toString()));
} catch (Exception exc) {
- LOG.error("Error sending driver start event to listener " + eventListener, exc);
+ log.error("Error sending driver start event to listener " + eventListener, exc);
}
}
} catch (Exception e) {
@@ -969,7 +968,7 @@ public class HiveDriver implements LensDriver {
}
error = "timedout";
} catch (Exception e) {
- LOG.warn("Error while polling for status", e);
+ log.warn("Error while polling for status", e);
error = "error polling";
}
listener.onError(handle, error);
@@ -1025,9 +1024,9 @@ public class HiveDriver implements LensDriver {
QueryHandle qhandle = (QueryHandle) in.readObject();
OperationHandle opHandle = new OperationHandle((TOperationHandle) in.readObject());
hiveHandles.put(qhandle, opHandle);
- LOG.debug("Hive driver recovered " + qhandle + ":" + opHandle);
+ log.debug("Hive driver recovered " + qhandle + ":" + opHandle);
}
- LOG.info("HiveDriver recovered " + hiveHandles.size() + " queries");
+ log.info("HiveDriver recovered " + hiveHandles.size() + " queries");
int numSessions = in.readInt();
for (int i = 0; i < numSessions; i++) {
String lensId = in.readUTF();
@@ -1035,7 +1034,7 @@ public class HiveDriver implements LensDriver {
TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6);
lensToHiveSession.put(lensId, sHandle);
}
- LOG.info("HiveDriver recovered " + lensToHiveSession.size() + " sessions");
+ log.info("HiveDriver recovered " + lensToHiveSession.size() + " sessions");
}
}
@@ -1052,15 +1051,15 @@ public class HiveDriver implements LensDriver {
for (Map.Entry<QueryHandle, OperationHandle> entry : hiveHandles.entrySet()) {
out.writeObject(entry.getKey());
out.writeObject(entry.getValue().toTOperationHandle());
- LOG.debug("Hive driver persisted " + entry.getKey() + ":" + entry.getValue());
+ log.debug("Hive driver persisted " + entry.getKey() + ":" + entry.getValue());
}
- LOG.info("HiveDriver persisted " + hiveHandles.size() + " queries");
+ log.info("HiveDriver persisted " + hiveHandles.size() + " queries");
out.writeInt(lensToHiveSession.size());
for (Map.Entry<String, SessionHandle> entry : lensToHiveSession.entrySet()) {
out.writeUTF(entry.getKey());
out.writeObject(entry.getValue().toTSessionHandle());
}
- LOG.info("HiveDriver persisted " + lensToHiveSession.size() + " sessions");
+ log.info("HiveDriver persisted " + lensToHiveSession.size() + " sessions");
}
}
@@ -1122,14 +1121,14 @@ public class HiveDriver implements LensDriver {
for (SessionHandle session : hiveSessionsToCheck) {
if (isSessionInvalid(exc, session)) {
// We have to expire previous session
- LOG.info("Hive server session " + session + " for lens session " + lensSession + " has become invalid");
+ log.info("Hive server session " + session + " for lens session " + lensSession + " has become invalid");
sessionLock.lock();
try {
// We should close all connections and clear the session map since
// most likely all sessions are gone
closeAllConnections();
lensToHiveSession.clear();
- LOG.info("Cleared all sessions");
+ log.info("Cleared all sessions");
} finally {
sessionLock.unlock();
}
@@ -1146,12 +1145,12 @@ public class HiveDriver implements LensDriver {
protected void checkInvalidOperation(QueryHandle queryHandle, HiveSQLException exc) {
final OperationHandle operation = hiveHandles.get(queryHandle);
if (operation == null) {
- LOG.info("No hive operation available for " + queryHandle);
+ log.info("No hive operation available for " + queryHandle);
return;
}
if (exc.getMessage() != null && exc.getMessage().contains("Invalid OperationHandle:")
&& exc.getMessage().contains(operation.toString())) {
- LOG.info("Hive operation " + operation + " for query " + queryHandle + " has become invalid");
+ log.info("Hive operation " + operation + " for query " + queryHandle + " has become invalid");
hiveHandles.remove(queryHandle);
return;
}
@@ -1193,10 +1192,10 @@ public class HiveDriver implements LensDriver {
if (hiveSession != null) {
try {
getClient().closeSession(hiveSession);
- LOG.info("Closed Hive session " + hiveSession.getHandleIdentifier() + " for lens session "
+ log.info("Closed Hive session " + hiveSession.getHandleIdentifier() + " for lens session "
+ sessionDbKey);
} catch (Exception e) {
- LOG.error("Error closing hive session " + hiveSession.getHandleIdentifier()
+ log.error("Error closing hive session " + hiveSession.getHandleIdentifier()
+ " for lens session " + sessionDbKey, e);
}
resourcesAddedForSession.remove(hiveSession);
@@ -1217,7 +1216,7 @@ public class HiveDriver implements LensDriver {
try {
connection.getConnection().close();
} catch (Exception ce) {
- LOG.warn("Error closing connection to hive server");
+ log.warn("Error closing connection to hive server");
}
}
threadConnections.clear();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
index 4d631ec..3d1f20a 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
@@ -30,9 +30,12 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.hive.service.cli.*;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class HiveInMemoryResultSet.
*/
+@Slf4j
public class HiveInMemoryResultSet extends InMemoryResultSet {
/** The client. */
@@ -120,7 +123,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
noMoreResults = rowSet.numRows() == 0;
if (noMoreResults) {
if (closeAfterFecth) {
- HiveDriver.LOG.info("No more results closing the query");
+ log.info("No more results closing the query");
client.closeOperation(opHandle);
}
return false;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
index 9d43fa8..b0a04df 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
@@ -34,9 +34,12 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class HiveQueryPlan.
*/
+@Slf4j
public class HiveQueryPlan extends DriverQueryPlan {
/** The explain output. */
@@ -164,7 +167,7 @@ public class HiveQueryPlan extends DriverQueryPlan {
Table tbl = metastore.getTable(table, false);
if (tbl == null) {
// table not found, possible case if query is create table
- HiveDriver.LOG.info("Table " + table + " not found while extracting plan details");
+ log.info("Table " + table + " not found while extracting plan details");
continue;
}
tablesQueried.add(table);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index f0566c8..b4086b8 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -35,8 +35,6 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.Service;
@@ -47,14 +45,14 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestRemoteHiveDriver.
*/
+@Slf4j
public class TestRemoteHiveDriver extends TestHiveDriver {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(TestRemoteHiveDriver.class);
-
/** The Constant HS2_HOST. */
static final String HS2_HOST = "localhost";
@@ -121,7 +119,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
try {
server.stop();
} catch (Exception e) {
- e.printStackTrace();
+ log.error("Error stopping hive service", e);
}
}
@@ -151,7 +149,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
*/
@Test
public void testMultiThreadClient() throws Exception {
- LOG.info("@@ Starting multi thread test");
+ log.info("@@ Starting multi thread test");
// Launch two threads
createTestTable("test_multithreads");
HiveConf thConf = new HiveConf(conf, TestRemoteHiveDriver.class);
@@ -175,10 +173,10 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
thrDriver.executeAsync(qctx);
} catch (LensException e) {
errCount.incrementAndGet();
- LOG.info(q + " executeAsync error: " + e.getCause());
+ log.info(q + " executeAsync error: " + e.getCause());
continue;
}
- LOG.info("@@ Launched query: " + q + " " + qctx.getQueryHandle());
+ log.info("@@ Launched query: " + q + " " + qctx.getQueryHandle());
launchedQueries++;
// Launch many threads to poll for status
final QueryHandle handle = qctx.getQueryHandle();
@@ -192,18 +190,17 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
try {
thrDriver.updateStatus(qctx);
if (qctx.getDriverStatus().isFinished()) {
- LOG.info("@@ " + handle.getHandleId() + " >> " + qctx.getDriverStatus().getState());
+ log.info("@@ " + handle.getHandleId() + " >> " + qctx.getDriverStatus().getState());
thrDriver.closeQuery(handle);
break;
}
Thread.sleep(POLL_DELAY);
} catch (LensException e) {
- LOG.error("Got Exception", e.getCause());
- e.printStackTrace();
+ log.error("Got Exception " +e.getCause(), e);
errCount.incrementAndGet();
break;
} catch (InterruptedException e) {
- e.printStackTrace();
+ log.error("Encountred Interrupted exception", e);
break;
}
}
@@ -219,11 +216,11 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
try {
th.join(10000);
} catch (InterruptedException e) {
- LOG.warn("Not ended yet: " + th.getName());
+ log.warn("Not ended yet: " + th.getName());
}
}
Assert.assertEquals(0, thrDriver.getHiveHandleSize());
- LOG.info("@@ Completed all pollers. Total thrift errors: " + errCount.get());
+ log.info("@@ Completed all pollers. Total thrift errors: " + errCount.get());
assertEquals(launchedQueries, QUERIES);
assertEquals(thrs.size(), QUERIES * THREADS);
assertEquals(errCount.get(), 0);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index dc30bb9..556ca2c 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -46,9 +46,12 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestColumnarSQLRewriter.
*/
+@Slf4j
public class TestColumnarSQLRewriter {
HiveConf hconf = new HiveConf();
@@ -186,7 +189,7 @@ public class TestColumnarSQLRewriter {
createHiveTable("default", "branch_dim", branchdimColumns);
createHiveTable("default", "location_dim", locationdimColumns);
} catch (HiveException e) {
- e.printStackTrace();
+ log.error("Encountered hive exception.", e);
}
}
@@ -204,7 +207,7 @@ public class TestColumnarSQLRewriter {
Hive.get().dropTable("default.branch_dim");
Hive.get().dropTable("default.location_dim");
} catch (HiveException e) {
- e.printStackTrace();
+ log.error("Encountered hive exception", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestDataSourceConnectionProvider.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestDataSourceConnectionProvider.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestDataSourceConnectionProvider.java
index e526700..21e5f50 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestDataSourceConnectionProvider.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestDataSourceConnectionProvider.java
@@ -28,19 +28,18 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.Logger;
import org.testng.Assert;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestDataSourceConnectionProvider.
*/
+@Slf4j
public class TestDataSourceConnectionProvider {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(TestDataSourceConnectionProvider.class);
-
/**
* Test get connection hsql.
*
@@ -75,20 +74,20 @@ public class TestDataSourceConnectionProvider {
Assert.assertNotNull(st);
passed.incrementAndGet();
} catch (SQLException e) {
- LOG.error("error getting connection to db!", e);
+ log.error("error getting connection to db!", e);
} finally {
if (st != null) {
try {
st.close();
} catch (SQLException e) {
- e.printStackTrace();
+ log.error("Encountered SQL ecxception", e);
}
}
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
- e.printStackTrace();
+ log.error("Encountered SQL exception", e);
}
}
}
@@ -134,7 +133,7 @@ public class TestDataSourceConnectionProvider {
}
} catch (SQLException sqlEx) {
if (i != MAX_CONNECTIONS) {
- LOG.error("Unexpected getConnection error", sqlEx);
+ log.error("Unexpected getConnection error", sqlEx);
}
assertEquals(i, MAX_CONNECTIONS, "Failed before last getConnection call: " + sqlEx.getMessage());
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 75c428a..4f03f23 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -46,8 +46,6 @@ import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.util.LensUtil;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.session.SessionState;
@@ -62,11 +60,13 @@ import org.testng.annotations.Test;
import com.codahale.metrics.MetricRegistry;
import com.mchange.v2.c3p0.ComboPooledDataSource;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestJdbcDriver.
*/
+@Slf4j
public class TestJdbcDriver {
- public static final Log LOG = LogFactory.getLog(TestJdbcDriver.class);
/** The base conf. */
Configuration baseConf;
@@ -211,7 +211,7 @@ public class TestJdbcDriver {
try {
driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
- e.printStackTrace();
+ log.error("Error running DDL query.", e);
th = e;
}
Assert.assertNotNull(th);
@@ -222,7 +222,7 @@ public class TestJdbcDriver {
try {
driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
- e.printStackTrace();
+ log.error("Error running DDL query", e);
th = e;
}
Assert.assertNotNull(th);
@@ -233,7 +233,7 @@ public class TestJdbcDriver {
try {
driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
- e.printStackTrace();
+ log.error("Error running DDL query", e);
th = e;
}
Assert.assertNotNull(th);
@@ -244,7 +244,7 @@ public class TestJdbcDriver {
try {
driver.rewriteQuery(createQueryContext(query));
} catch (LensException e) {
- e.printStackTrace();
+ log.error("Error running DDL query", e);
th = e;
}
Assert.assertNotNull(th);
@@ -272,7 +272,7 @@ public class TestJdbcDriver {
driver.getEstimateConnectionConf().getInt(JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE, 50);
for (int i = 0; i < maxEstimateConnections + 10; i++) {
try {
- LOG.info("Iteration#" + (i + 1));
+ log.info("Iteration#" + (i + 1));
String query = i > maxEstimateConnections ? "SELECT * FROM estimate_test" : "CREATE TABLE FOO(ID INT)";
ExplainQueryContext context = createExplainContext(query, baseConf);
cost = driver.estimate(context);
@@ -578,8 +578,8 @@ public class TestJdbcDriver {
try {
listenerNotificationLatch.await(1, TimeUnit.SECONDS);
} catch (Exception e) {
- fail("query completion listener was not notified - " + e.getMessage());
- e.printStackTrace();
+ fail("Query completion listener was not notified - " + e.getMessage());
+ log.error("Query completion listener was not notified.", e);
}
LensResultSet grs = driver.fetchResultSet(context);
@@ -738,7 +738,7 @@ public class TestJdbcDriver {
LensResultSet rs = driver.execute(ctx);
fail("Should have thrown exception");
} catch (LensException e) {
- e.printStackTrace();
+ log.error("Encountered Lens exception.", e);
}
final CountDownLatch listenerNotificationLatch = new CountDownLatch(1);
@@ -778,7 +778,7 @@ public class TestJdbcDriver {
driver.fetchResultSet(ctx);
fail("should have thrown error");
} catch (LensException e) {
- e.printStackTrace();
+ log.error("Encountered Lens exception", e);
}
driver.closeQuery(handle);
}
@@ -816,8 +816,8 @@ public class TestJdbcDriver {
DataSourceConnectionProvider.DriverConfig queryCfg =
queryCp.getDriverConfigfromConf(driver.getConf());
- LOG.info("@@@ ESTIMATE_CFG " + estimateCfg);
- LOG.info("@@@ QUERY CFG " + queryCfg);
+ log.info("@@@ ESTIMATE_CFG " + estimateCfg);
+ log.info("@@@ QUERY CFG " + queryCfg);
// Get connection from each so that pools get initialized
try {
@@ -825,14 +825,14 @@ public class TestJdbcDriver {
estimateConn.close();
} catch (SQLException e) {
// Ignore exception
- LOG.error("Error getting connection from estimate pool", e);
+ log.error("Error getting connection from estimate pool", e);
}
try {
Connection queryConn = queryCp.getConnection(driver.getConf());
queryConn.close();
} catch (SQLException e) {
- LOG.error("Error getting connection from query pool", e);
+ log.error("Error getting connection from query pool", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
index 2f28e5d..dee43ef 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
@@ -31,7 +31,11 @@ import org.apache.lens.api.metastore.ObjectFactory;
import org.apache.lens.client.LensClientSingletonWrapper;
import org.apache.lens.client.LensMetadataClient;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class SampleMetastore {
+
private LensMetadataClient metaClient;
public static final Unmarshaller JAXB_UNMARSHALLER;
private APIResult result;
@@ -117,7 +121,7 @@ public class SampleMetastore {
System.out.println("Created DB storages");
} catch (Exception e) {
retCode = 1;
- e.printStackTrace();
+ log.error("Creating DB storage failed", e);
System.err.println("Creating DB storage failed");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
index 8ffe853..6dd0ecf 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
@@ -40,16 +40,17 @@ import org.apache.lens.ml.api.TestReport;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Client side implementation of LensML
*/
+@Slf4j
public class LensMLClient implements LensML, Closeable {
- private static final Log LOG = LogFactory.getLog(LensMLClient.class);
/** The client. */
private LensMLJerseyClient client;
@@ -179,7 +180,7 @@ public class LensMLClient implements LensML, Closeable {
try {
in.close();
} catch (IOException e) {
- e.printStackTrace();
+ log.error("Error closing stream.", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
index 51979d8..00f20fc 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
@@ -27,12 +27,12 @@ import java.util.Map;
import org.apache.lens.ml.algo.api.AlgoParam;
import org.apache.lens.ml.algo.api.MLAlgo;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class AlgoArgParser.
*/
+@Slf4j
public final class AlgoArgParser {
private AlgoArgParser() {
}
@@ -53,9 +53,6 @@ public final class AlgoArgParser {
public abstract E parse(String value);
}
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(AlgoArgParser.class);
-
/**
* Extracts feature names. If the algo has any parameters associated with @AlgoParam annotation, those are set
* as well.
@@ -104,12 +101,12 @@ public final class AlgoArgParser {
CustomArgParser<?> parser = clz.newInstance();
f.set(algo, parser.parse(value));
} else {
- LOG.warn("Ignored param " + key + "=" + value + " as no parser found");
+ log.warn("Ignored param " + key + "=" + value + " as no parser found");
}
}
}
} catch (Exception exc) {
- LOG.error("Error while setting param " + key + " to " + value + " for algo " + algo);
+ log.error("Error while setting param " + key + " to " + value + " for algo " + algo, exc);
}
}
return featureColumns;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
index e4c03c8..de76603 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
@@ -52,8 +52,6 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -66,15 +64,14 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class LensMLImpl.
*/
+@Slf4j
public class LensMLImpl implements LensML {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensMLImpl.class);
-
/** The drivers. */
protected List<MLDriver> drivers;
@@ -144,7 +141,7 @@ public class LensMLImpl implements LensML {
String modelId = UUID.randomUUID().toString();
- LOG.info("Begin training model " + modelId + ", algo=" + algorithm + ", table=" + table + ", params="
+ log.info("Begin training model " + modelId + ", algo=" + algorithm + ", table=" + table + ", params="
+ Arrays.toString(args));
String database = null;
@@ -156,7 +153,7 @@ public class LensMLImpl implements LensML {
MLModel model = algo.train(toLensConf(conf), database, table, modelId, args);
- LOG.info("Done training model: " + modelId);
+ log.info("Done training model: " + modelId);
model.setCreatedAt(new Date());
model.setAlgoName(algorithm);
@@ -164,7 +161,7 @@ public class LensMLImpl implements LensML {
Path modelLocation = null;
try {
modelLocation = persistModel(model);
- LOG.info("Model saved: " + modelId + ", algo: " + algorithm + ", path: " + modelLocation);
+ log.info("Model saved: " + modelId + ", algo: " + algorithm + ", path: " + modelLocation);
return model.getId();
} catch (IOException e) {
throw new LensException("Error saving model " + modelId + " for algo " + algorithm, e);
@@ -207,7 +204,7 @@ public class LensMLImpl implements LensML {
outputStream.writeObject(model);
outputStream.flush();
} catch (IOException io) {
- LOG.error("Error saving model " + model.getId() + " reason: " + io.getMessage());
+ log.error("Error saving model " + model.getId() + " reason: " + io.getMessage(), io);
throw io;
} finally {
IOUtils.closeQuietly(outputStream);
@@ -272,7 +269,7 @@ public class LensMLImpl implements LensML {
throw new RuntimeException("No ML Drivers specified in conf");
}
- LOG.info("Loading drivers " + Arrays.toString(driverClasses));
+ log.info("Loading drivers " + Arrays.toString(driverClasses));
drivers = new ArrayList<MLDriver>(driverClasses.length);
for (String driverClass : driverClasses) {
@@ -280,12 +277,12 @@ public class LensMLImpl implements LensML {
try {
cls = Class.forName(driverClass);
} catch (ClassNotFoundException e) {
- LOG.error("Driver class not found " + driverClass);
+ log.error("Driver class not found " + driverClass, e);
continue;
}
if (!MLDriver.class.isAssignableFrom(cls)) {
- LOG.warn("Not a driver class " + driverClass);
+ log.warn("Not a driver class " + driverClass);
continue;
}
@@ -294,16 +291,16 @@ public class LensMLImpl implements LensML {
MLDriver driver = mlDriverClass.newInstance();
driver.init(toLensConf(conf));
drivers.add(driver);
- LOG.info("Added driver " + driverClass);
+ log.info("Added driver " + driverClass);
} catch (Exception e) {
- LOG.error("Failed to create driver " + driverClass + " reason: " + e.getMessage(), e);
+ log.error("Failed to create driver " + driverClass + " reason: " + e.getMessage(), e);
}
}
if (drivers.isEmpty()) {
throw new RuntimeException("No ML drivers loaded");
}
- LOG.info("Inited ML service");
+ log.info("Inited ML service");
}
/**
@@ -317,14 +314,14 @@ public class LensMLImpl implements LensML {
}
driver.start();
} catch (LensException e) {
- LOG.error("Failed to start driver " + driver, e);
+ log.error("Failed to start driver " + driver, e);
}
}
udfStatusExpirySvc = Executors.newSingleThreadScheduledExecutor();
udfStatusExpirySvc.scheduleAtFixedRate(new UDFStatusExpiryRunnable(), 60, 60, TimeUnit.SECONDS);
- LOG.info("Started ML service");
+ log.info("Started ML service");
}
/**
@@ -335,12 +332,12 @@ public class LensMLImpl implements LensML {
try {
driver.stop();
} catch (LensException e) {
- LOG.error("Failed to stop driver " + driver, e);
+ log.error("Failed to stop driver " + driver, e);
}
}
drivers.clear();
udfStatusExpirySvc.shutdownNow();
- LOG.info("Stopped ML service");
+ log.info("Stopped ML service");
}
public synchronized HiveConf getHiveConf() {
@@ -455,18 +452,18 @@ public class LensMLImpl implements LensML {
}
if (!spec.isOutputTableExists()) {
- LOG.info("Output table '" + testTable + "' does not exist for test algorithm = " + algorithm + " modelid="
+ log.info("Output table '" + testTable + "' does not exist for test algorithm = " + algorithm + " modelid="
+ modelID + ", Creating table using query: " + spec.getCreateOutputTableQuery());
// create the output table
String createOutputTableQuery = spec.getCreateOutputTableQuery();
queryRunner.runQuery(createOutputTableQuery);
- LOG.info("Table created " + testTable);
+ log.info("Table created " + testTable);
}
// Check if ML UDF is registered in this session
registerPredictUdf(sessionHandle, queryRunner);
- LOG.info("Running evaluation query " + testQuery);
+ log.info("Running evaluation query " + testQuery);
queryRunner.setQueryName("model_test_" + modelID);
QueryHandle testQueryHandle = queryRunner.runQuery(testQuery);
@@ -483,7 +480,7 @@ public class LensMLImpl implements LensML {
// Save test report
persistTestReport(testReport);
- LOG.info("Saved test report " + testReport.getReportID());
+ log.info("Saved test report " + testReport.getReportID());
return testReport;
}
@@ -494,12 +491,12 @@ public class LensMLImpl implements LensML {
* @throws LensException the lens exception
*/
private void persistTestReport(MLTestReport testReport) throws LensException {
- LOG.info("saving test report " + testReport.getReportID());
+ log.info("saving test report " + testReport.getReportID());
try {
ModelLoader.saveTestReport(conf, testReport);
- LOG.info("Saved report " + testReport.getReportID());
+ log.info("Saved report " + testReport.getReportID());
} catch (IOException e) {
- LOG.error("Error saving report " + testReport.getReportID() + " reason: " + e.getMessage());
+ log.error("Error saving report " + testReport.getReportID() + " reason: " + e.getMessage(), e);
}
}
@@ -529,7 +526,7 @@ public class LensMLImpl implements LensML {
}
return reports;
} catch (IOException e) {
- LOG.error("Error reading report list for " + algorithm, e);
+ log.error("Error reading report list for " + algorithm, e);
return null;
}
}
@@ -566,9 +563,9 @@ public class LensMLImpl implements LensML {
public void deleteModel(String algorithm, String modelID) throws LensException {
try {
ModelLoader.deleteModel(conf, algorithm, modelID);
- LOG.info("DELETED model " + modelID + " algorithm=" + algorithm);
+ log.info("DELETED model " + modelID + " algorithm=" + algorithm);
} catch (IOException e) {
- LOG.error(
+ log.error(
"Error deleting model file. algorithm=" + algorithm + " model=" + modelID + " reason: " + e.getMessage(), e);
throw new LensException("Unable to delete model " + modelID + " for algorithm " + algorithm, e);
}
@@ -582,9 +579,9 @@ public class LensMLImpl implements LensML {
public void deleteTestReport(String algorithm, String reportID) throws LensException {
try {
ModelLoader.deleteTestReport(conf, algorithm, reportID);
- LOG.info("DELETED report=" + reportID + " algorithm=" + algorithm);
+ log.info("DELETED report=" + reportID + " algorithm=" + algorithm);
} catch (IOException e) {
- LOG.error("Error deleting report " + reportID + " algorithm=" + algorithm + " reason: " + e.getMessage(), e);
+ log.error("Error deleting report " + reportID + " algorithm=" + algorithm + " reason: " + e.getMessage(), e);
throw new LensException("Unable to delete report " + reportID + " for algorithm " + algorithm, e);
}
}
@@ -599,7 +596,7 @@ public class LensMLImpl implements LensML {
try {
algo = getAlgoForName(algorithm);
} catch (LensException e) {
- LOG.error("Error getting algo description : " + algorithm, e);
+ log.error("Error getting algo description : " + algorithm, e);
return null;
}
if (algo instanceof BaseSparkAlgo) {
@@ -693,15 +690,15 @@ public class LensMLImpl implements LensML {
return;
}
- LOG.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
+ log.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
String regUdfQuery = "CREATE TEMPORARY FUNCTION " + HiveMLUDF.UDF_NAME + " AS '" + HiveMLUDF.class
.getCanonicalName() + "'";
queryRunner.setQueryName("register_predict_udf_" + sessionHandle.getPublicId().toString());
QueryHandle udfQuery = queryRunner.runQuery(regUdfQuery);
- LOG.info("udf query handle is " + udfQuery);
+ log.info("udf query handle is " + udfQuery);
predictUdfStatus.put(sessionHandle, true);
- LOG.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
+ log.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
}
protected boolean isUdfRegisterd(LensSessionHandle sessionHandle) {
@@ -719,12 +716,12 @@ public class LensMLImpl implements LensML {
List<LensSessionHandle> sessions = new ArrayList<LensSessionHandle>(predictUdfStatus.keySet());
for (LensSessionHandle sessionHandle : sessions) {
if (!sessionService.isOpen(sessionHandle)) {
- LOG.info("Session closed, removing UDF status: " + sessionHandle);
+ log.info("Session closed, removing UDF status: " + sessionHandle);
predictUdfStatus.remove(sessionHandle);
}
}
} catch (Exception exc) {
- LOG.warn("Error clearing UDF statuses", exc);
+ log.warn("Error clearing UDF statuses", exc);
}
}
}
[44/51] [abbrv] incubator-lens git commit: LENS-487: Implementation
for consistent error response display for Lens Client
Posted by jd...@apache.org.
LENS-487: Implementation for consistent error response display for Lens Client
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d1d99708
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d1d99708
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d1d99708
Branch: refs/heads/current-release-line
Commit: d1d99708026d17d43f3f9ea4f9ae088cb151fbd2
Parents: 5d20751
Author: Himanshu Gahlaut <ga...@gmail.com>
Authored: Thu Jun 11 11:49:37 2015 +0530
Committer: Himanshu Gahlaut <ga...@gmail.com>
Committed: Sat Jun 20 14:23:53 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/api/query/LensQuery.java | 11 ++
.../org/apache/lens/api/query/QueryStatus.java | 12 ++
.../query/SupportedQuerySubmitOperations.java | 59 +++++++++
.../apache/lens/api/response/LensErrorTO.java | 119 ------------------
.../api/response/LensJAXBContextResolver.java | 77 ------------
.../apache/lens/api/response/LensResponse.java | 110 ----------------
.../lens/api/response/NoErrorPayload.java | 33 -----
.../api/response/NoSuccessResponseData.java | 30 -----
.../apache/lens/api/result/LensAPIResult.java | 125 +++++++++++++++++++
.../org/apache/lens/api/result/LensErrorTO.java | 122 ++++++++++++++++++
.../api/result/LensJAXBContextResolver.java | 77 ++++++++++++
.../apache/lens/api/result/NoErrorPayload.java | 33 +++++
.../apache/lens/api/result/NoResultData.java | 30 +++++
.../apache/lens/api/result/PrettyPrintable.java | 32 +++++
lens-api/src/main/resources/lens-errors.conf | 2 +-
.../lens/cli/commands/LensQueryCommands.java | 33 +++--
.../apache/lens/cli/ExecuteQueryCommandIT.java | 42 +++++++
.../apache/lens/cli/TestLensQueryCommands.java | 3 +-
lens-client/pom.xml | 20 +++
.../java/org/apache/lens/client/LensClient.java | 19 ++-
.../org/apache/lens/client/LensStatement.java | 64 ++++++----
.../client/exceptions/LensAPIException.java | 48 +++++++
.../exceptions/LensBriefErrorException.java | 34 +++++
.../client/exceptions/LensClientException.java | 24 +---
.../lens/client/jdbc/LensJdbcStatement.java | 10 +-
.../apache/lens/client/model/BriefError.java | 47 +++++++
.../lens/client/model/IdBriefErrorTemplate.java | 53 ++++++++
.../client/model/IdBriefErrorTemplateKey.java | 35 ++++++
.../lens/client/model/BriefErrorTest.java | 60 +++++++++
.../client/model/IdBriefErrorTemplateTest.java | 74 +++++++++++
.../ColUnAvailableInTimeRangeException.java | 2 +-
.../FieldsCannotBeQueriedTogetherException.java | 2 +-
.../org/apache/lens/examples/SampleQueries.java | 7 +-
.../org/apache/lens/ml/impl/LensMLImpl.java | 5 +-
.../java/org/apache/lens/rdd/LensRDDClient.java | 7 +-
.../regression/core/helpers/QueryHelper.java | 12 +-
.../server/api/driver/DriverQueryStatus.java | 4 +-
.../lens/server/api/error/LensException.java | 8 +-
.../api/error/LensMultiCauseException.java | 2 +-
.../server/api/query/FinishedLensQuery.java | 2 +-
.../lens/server/api/query/QueryContext.java | 2 +-
.../java/org/apache/lens/server/LensServer.java | 2 +-
.../lens/server/error/LensExceptionMapper.java | 6 +-
.../UnSupportedQuerySubmitOpException.java | 4 +-
.../server/query/QueryExecutionServiceImpl.java | 43 +++++--
.../lens/server/query/QueryServiceResource.java | 9 +-
.../lens/server/query/ResultFormatter.java | 2 +-
.../query/SupportedQuerySubmitOperations.java | 61 ---------
.../org/apache/lens/server/LensTestUtil.java | 9 +-
.../org/apache/lens/server/TestServerMode.java | 5 +-
.../apache/lens/server/TestServerRestart.java | 9 +-
.../common/ErrorResponseExpectedData.java | 11 +-
.../lens/server/common/TestDataUtils.java | 15 +--
.../metrics/TestResourceMethodMetrics.java | 5 +-
.../server/query/QueryAPIErrorResponseTest.java | 9 +-
.../server/query/TestQueryEndEmailNotifier.java | 5 +-
.../lens/server/query/TestQueryService.java | 39 +++---
.../lens/server/query/TestResultFormatting.java | 5 +-
pom.xml | 2 +-
59 files changed, 1131 insertions(+), 601 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
index c22d39a..f594133 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
@@ -209,4 +209,15 @@ public class LensQuery {
@Getter
private String queryName;
+ public Integer getErrorCode() {
+ return (this.status != null) ? this.status.getErrorCode() : null;
+ }
+
+ public String getErrorMessage() {
+ return (this.status!=null) ? this.status.getLensErrorTOErrorMsg() : null;
+ }
+
+ public String getQueryHandleString() {
+ return (this.queryHandle != null) ? this.queryHandle.getHandleIdString() : null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index f927375..539f0b0 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -26,6 +26,8 @@ import java.io.Serializable;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.lens.api.result.LensErrorTO;
+
import lombok.*;
/**
@@ -153,6 +155,9 @@ public class QueryStatus implements Serializable {
@Setter
private String errorMessage;
+ @XmlElement
+ private LensErrorTO lensErrorTO;
+
/*
* (non-Javadoc)
*
@@ -253,4 +258,11 @@ public class QueryStatus implements Serializable {
return isValidTransition(this.status, newState);
}
+ public Integer getErrorCode() {
+ return (this.lensErrorTO != null) ? this.lensErrorTO.getCode() : null;
+ }
+
+ public String getLensErrorTOErrorMsg() {
+ return (this.lensErrorTO != null) ? this.lensErrorTO.getMessage() : null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/query/SupportedQuerySubmitOperations.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/SupportedQuerySubmitOperations.java b/lens-api/src/main/java/org/apache/lens/api/query/SupportedQuerySubmitOperations.java
new file mode 100644
index 0000000..5f246e5
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/query/SupportedQuerySubmitOperations.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.query;
+
+import static org.apache.lens.api.query.SubmitOp.*;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import javax.xml.bind.annotation.*;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+@XmlRootElement
+@EqualsAndHashCode
+@ToString
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SupportedQuerySubmitOperations {
+
+ private static final String SEP = ", ";
+
+ @XmlElementWrapper(name = "supportedOperations")
+ @XmlElement(name = "operation")
+ private List<String> supportedOps = new LinkedList<String>();
+
+ public SupportedQuerySubmitOperations() {
+ ImmutableSet<SubmitOp> supportedOps = Sets.immutableEnumSet(ESTIMATE, EXECUTE, EXPLAIN, EXECUTE_WITH_TIMEOUT);
+
+ for (SubmitOp supportedOp : supportedOps) {
+ this.supportedOps.add(supportedOp.toString().toLowerCase());
+ }
+ }
+
+ public String getSupportedOperationsAsString() {
+ return StringUtils.join(supportedOps, SEP);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/response/LensErrorTO.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/response/LensErrorTO.java b/lens-api/src/main/java/org/apache/lens/api/response/LensErrorTO.java
deleted file mode 100644
index 313566c..0000000
--- a/lens-api/src/main/java/org/apache/lens/api/response/LensErrorTO.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.response;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlElementWrapper;
-
-import org.apache.commons.lang.StringUtils;
-
-import lombok.*;
-
-/**
- *
- * Transport object for lens error information.
- *
- * @param <PAYLOAD> represents type of error payload transferred in failure lens response
- */
-@EqualsAndHashCode(exclude = { "stackTrace" })
-@ToString
-@NoArgsConstructor(access = AccessLevel.PACKAGE)
-@XmlAccessorType(XmlAccessType.FIELD)
-public class LensErrorTO<PAYLOAD> {
-
- @XmlElement
- private int code;
-
- @XmlElement
- private String message;
-
- @XmlElement
- private String stackTrace;
-
- @XmlElement
- @Getter
- private PAYLOAD payload;
-
- @Getter
- @XmlElementWrapper(name = "childErrors")
- @XmlElement(name = "error")
- private List<LensErrorTO> childErrors;
-
- public static <PAYLOAD> LensErrorTO<PAYLOAD> composedOf(final int code, final String message,
- final String stackTrace, final PAYLOAD payload, final List<LensErrorTO> childErrors) {
-
- return new LensErrorTO<PAYLOAD>(code, message, stackTrace, payload, childErrors);
- }
-
- public static <PAYLOAD> LensErrorTO<PAYLOAD> composedOf(final int code, final String message,
- final String stackTrace, final PAYLOAD payload) {
-
- return new LensErrorTO<PAYLOAD>(code, message, stackTrace, payload, null);
- }
-
- public static LensErrorTO<NoErrorPayload> composedOf(final int code, final String message,
- final String stackTrace) {
-
- return new LensErrorTO<NoErrorPayload>(code, message, stackTrace, null, null);
- }
-
- public static LensErrorTO<NoErrorPayload> composedOf(final int code, final String message,
- final String stackTrace, final List<LensErrorTO> childErrors) {
-
- return new LensErrorTO<NoErrorPayload>(code, message, stackTrace, null, childErrors);
- }
-
- private LensErrorTO(final int code, final String message, final String stackTrace, final PAYLOAD errorPayload,
- final List<LensErrorTO> childErrors) {
-
- checkArgument(code > 0);
- checkArgument(StringUtils.isNotBlank(message));
- checkArgument(StringUtils.isNotBlank(stackTrace));
-
- this.code = code;
- this.message = message;
- this.stackTrace = stackTrace;
- this.payload = errorPayload;
- this.childErrors = childErrors;
- }
-
- public boolean areValidStackTracesPresent() {
-
- /* if stack trace of first level error is not valid, then return false */
- if (StringUtils.isBlank(stackTrace)) {
- return false;
- }
-
- /* validate stack traces of child Errors */
- if (childErrors != null) {
- for (LensErrorTO childError : childErrors) {
- if (!childError.areValidStackTracesPresent()) {
- return false;
- }
- }
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/response/LensJAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/response/LensJAXBContextResolver.java b/lens-api/src/main/java/org/apache/lens/api/response/LensJAXBContextResolver.java
deleted file mode 100644
index a401b53..0000000
--- a/lens-api/src/main/java/org/apache/lens/api/response/LensJAXBContextResolver.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.response;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
-import javax.ws.rs.ext.ContextResolver;
-import javax.ws.rs.ext.Provider;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-
-import org.apache.lens.api.error.ErrorCollection;
-import org.apache.lens.api.error.ErrorCollectionFactory;
-
-import com.google.common.collect.Sets;
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * @see javax.ws.rs.ext.ContextResolver
- */
-@Provider
-@Slf4j
-public class LensJAXBContextResolver implements ContextResolver<JAXBContext> {
-
- private Map<Class, JAXBContext> jaxbContextCache = new ConcurrentHashMap<Class, JAXBContext>();
-
- @Override
- public JAXBContext getContext(Class<?> type) {
-
- JAXBContext jaxbContext = jaxbContextCache.get(type);
-
- if (jaxbContext == null) {
-
- log.debug("JAXB instance to be created for {}", type);
- try {
- if (type.equals(LensResponse.class)) {
-
- ErrorCollection errorCollection = new ErrorCollectionFactory().createErrorCollection();
- Set<Class> classesToBeBound = Sets.newHashSet(errorCollection.getErrorPayloadClasses());
- log.debug("classesToBeBound:{}", classesToBeBound);
- classesToBeBound.add(type);
-
- Class[] classesToBeBoundArray = classesToBeBound.toArray(new Class[classesToBeBound.size()]);
- jaxbContext = JAXBContext.newInstance(classesToBeBoundArray);
- } else {
-
- jaxbContext = JAXBContext.newInstance(type);
- }
- jaxbContextCache.put(type, jaxbContext);
-
- } catch (JAXBException e) {
- log.error("JAXBContext not initialized for "+type, e);
- } catch (ClassNotFoundException e) {
- log.error("JAXBContext not initialized for "+type, e);
- }
- }
- return jaxbContext;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java b/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
deleted file mode 100644
index 818ae40..0000000
--- a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.response;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-import javax.ws.rs.core.Response.Status;
-import javax.xml.bind.annotation.*;
-
-import org.apache.lens.api.query.QuerySubmitResult;
-
-import org.apache.commons.lang.StringUtils;
-
-import lombok.*;
-
-/**
- * Transport object for LensResponse
- *
- * DATA represents type of data transferred in success response.
- * PAYLOAD represents type of error payload transferred in error response.
- *
- */
-@XmlRootElement
-@XmlSeeAlso({NoSuccessResponseData.class, NoErrorPayload.class, QuerySubmitResult.class})
-@NoArgsConstructor(access=AccessLevel.PACKAGE)
-@ToString
-@XmlAccessorType(XmlAccessType.FIELD)
-public class LensResponse<DATA, PAYLOAD> {
-
- @XmlElement
- private String apiVersion;
-
- @XmlElement
- private String id;
-
- @XmlElement(name = "data")
- @Getter
- private DATA data;
-
- @XmlElement(name = "error")
- @Getter
- private LensErrorTO<PAYLOAD> lensErrorTO;
-
- @XmlTransient
- private Status httpStatusCode;
-
- public static <DATA> LensResponse<DATA, NoErrorPayload> composedOf(final String apiVersion,
- final String id, @NonNull final DATA data) {
-
- return new LensResponse<DATA, NoErrorPayload>(apiVersion, id, data, null, Status.OK);
- }
-
- public static <DATA> LensResponse<DATA, NoErrorPayload> composedOf(final String apiVersion,
- final String id, @NonNull final DATA data, @NonNull final Status httpStatusCode) {
-
- return new LensResponse<DATA, NoErrorPayload>(apiVersion, id, data, null, httpStatusCode);
- }
-
- public static <PAYLOAD> LensResponse<NoSuccessResponseData, PAYLOAD> composedOf(
- final String apiVersion, final String id, @NonNull final LensErrorTO<PAYLOAD> lensErrorTO,
- @NonNull final Status httpStatusCode) {
-
- return new LensResponse<NoSuccessResponseData, PAYLOAD>(apiVersion, id, null, lensErrorTO, httpStatusCode);
- }
-
- private LensResponse(final String apiVersion, final String id, final DATA data,
- final LensErrorTO lensErrorTO, @NonNull final Status httpStatusCode) {
-
- /* The check commented below should be enabled in future, once story of apiVersion is clear. Right now there could
- be REST APIs throwing LensException without initializing apiVersion
-
- checkArgument(StringUtils.isNotBlank(apiVersion)); */
-
- checkArgument(StringUtils.isNotBlank(id));
-
- this.apiVersion = apiVersion;
- this.id = id;
- this.data = data;
- this.lensErrorTO = lensErrorTO;
- this.httpStatusCode = httpStatusCode;
- }
-
- public boolean areValidStackTracesPresent() {
- return (lensErrorTO == null) ? false : lensErrorTO.areValidStackTracesPresent();
- }
-
- public Status getHttpStatusCode() {
- return this.httpStatusCode;
- }
-
- public boolean isSuccessResponse() {
- return lensErrorTO == null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/response/NoErrorPayload.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/response/NoErrorPayload.java b/lens-api/src/main/java/org/apache/lens/api/response/NoErrorPayload.java
deleted file mode 100644
index afc1a05..0000000
--- a/lens-api/src/main/java/org/apache/lens/api/response/NoErrorPayload.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.response;
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import lombok.EqualsAndHashCode;
-
-/**
- * NoErrorPayload type is to be used while creating LensResponse for success responses.
- * Success Responses will not have any error payload.
- *
- */
-@XmlRootElement
-@EqualsAndHashCode
-public final class NoErrorPayload {
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/response/NoSuccessResponseData.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/response/NoSuccessResponseData.java b/lens-api/src/main/java/org/apache/lens/api/response/NoSuccessResponseData.java
deleted file mode 100644
index 1775584..0000000
--- a/lens-api/src/main/java/org/apache/lens/api/response/NoSuccessResponseData.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.api.response;
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * NoSuccessResponseData type is to be used while creating LensResponse for error responses.
- * Error Responses will not have any data related to successful execution of API.
- *
- */
-@XmlRootElement
-public final class NoSuccessResponseData {
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/result/LensAPIResult.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/result/LensAPIResult.java b/lens-api/src/main/java/org/apache/lens/api/result/LensAPIResult.java
new file mode 100644
index 0000000..41805eb
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/result/LensAPIResult.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.result;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import javax.ws.rs.core.Response.Status;
+import javax.xml.bind.annotation.*;
+
+import org.apache.lens.api.query.QuerySubmitResult;
+
+import org.apache.commons.lang.StringUtils;
+
+import lombok.*;
+
+/**
+ * Transport object for results returned by Lens APIs
+ *
+ * DATA represents type of data in success result.
+ *
+ */
+@XmlRootElement
+@XmlSeeAlso({NoResultData.class, NoErrorPayload.class, QuerySubmitResult.class})
+@NoArgsConstructor(access=AccessLevel.PACKAGE)
+@ToString
+@XmlAccessorType(XmlAccessType.FIELD)
+public class LensAPIResult<DATA> {
+
+ @XmlElement
+ private String apiVersion;
+
+ @XmlElement
+ @Getter
+ private String id;
+
+ @XmlElement(name = "data")
+ @Getter
+ private DATA data;
+
+ @XmlElement(name = "error")
+ @Getter
+ private LensErrorTO lensErrorTO;
+
+ @XmlTransient
+ private Status httpStatusCode;
+
+ public static <DATA> LensAPIResult<DATA> composedOf(final String apiVersion,
+ final String id, @NonNull final DATA data) {
+
+ return new LensAPIResult<DATA>(apiVersion, id, data, null, Status.OK);
+ }
+
+ public static <DATA> LensAPIResult<DATA> composedOf(final String apiVersion,
+ final String id, @NonNull final DATA data, @NonNull final Status httpStatusCode) {
+
+ return new LensAPIResult<DATA>(apiVersion, id, data, null, httpStatusCode);
+ }
+
+ public static <PAYLOAD> LensAPIResult<NoResultData> composedOf(
+ final String apiVersion, final String id, @NonNull final LensErrorTO lensErrorTO,
+ @NonNull final Status httpStatusCode) {
+
+ return new LensAPIResult<NoResultData>(apiVersion, id, null, lensErrorTO, httpStatusCode);
+ }
+
+ private LensAPIResult(final String apiVersion, final String id, final DATA data, final LensErrorTO lensErrorTO,
+ @NonNull final Status httpStatusCode) {
+
+ /* The check commented below should be enabled in future, once story of apiVersion is clear. Right now there could
+ be REST APIs throwing LensException without initializing apiVersion
+
+ checkArgument(StringUtils.isNotBlank(apiVersion)); */
+
+ checkArgument(StringUtils.isNotBlank(id));
+
+ this.apiVersion = apiVersion;
+ this.id = id;
+ this.data = data;
+ this.lensErrorTO = lensErrorTO;
+ this.httpStatusCode = httpStatusCode;
+ }
+
+ public boolean areValidStackTracesPresent() {
+ return (lensErrorTO == null) ? false : lensErrorTO.areValidStackTracesPresent();
+ }
+
+ public Status getHttpStatusCode() {
+ return this.httpStatusCode;
+ }
+
+ public boolean isSuccessResult() {
+ return lensErrorTO == null;
+ }
+
+ public boolean isErrorResult() {
+ return !isSuccessResult();
+ }
+
+ public int getErrorCode() {
+ checkState(isErrorResult());
+ return lensErrorTO.getCode();
+ }
+
+ public String getErrorMessage() {
+ checkState(isErrorResult());
+ return lensErrorTO.getMessage();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/result/LensErrorTO.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/result/LensErrorTO.java b/lens-api/src/main/java/org/apache/lens/api/result/LensErrorTO.java
new file mode 100644
index 0000000..a2aca36
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/result/LensErrorTO.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.result;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.io.Serializable;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+
+import org.apache.commons.lang.StringUtils;
+
+import lombok.*;
+
+/**
+ *
+ * Transport object for lens error information.
+ *
+ * @param <PAYLOAD> represents type of error payload transferred in failure lens response
+ */
+@EqualsAndHashCode(exclude = { "stackTrace" })
+@ToString
+@NoArgsConstructor(access = AccessLevel.PACKAGE)
+@XmlAccessorType(XmlAccessType.FIELD)
+public class LensErrorTO<PAYLOAD> implements Serializable {
+
+ @XmlElement
+ @Getter
+ private int code;
+
+ @XmlElement
+ @Getter
+ private String message;
+
+ @XmlElement
+ private String stackTrace;
+
+ @XmlElement
+ @Getter
+ private PAYLOAD payload;
+
+ @Getter
+ @XmlElementWrapper(name = "childErrors")
+ @XmlElement(name = "error")
+ private List<LensErrorTO> childErrors;
+
+ public static <PAYLOAD> LensErrorTO<PAYLOAD> composedOf(final int code, final String message,
+ final String stackTrace, final PAYLOAD payload, final List<LensErrorTO> childErrors) {
+
+ return new LensErrorTO<PAYLOAD>(code, message, stackTrace, payload, childErrors);
+ }
+
+ public static <PAYLOAD> LensErrorTO<PAYLOAD> composedOf(final int code, final String message,
+ final String stackTrace, final PAYLOAD payload) {
+
+ return new LensErrorTO<PAYLOAD>(code, message, stackTrace, payload, null);
+ }
+
+ public static LensErrorTO<NoErrorPayload> composedOf(final int code, final String message,
+ final String stackTrace) {
+
+ return new LensErrorTO<NoErrorPayload>(code, message, stackTrace, null, null);
+ }
+
+ public static LensErrorTO<NoErrorPayload> composedOf(final int code, final String message,
+ final String stackTrace, final List<LensErrorTO> childErrors) {
+
+ return new LensErrorTO<NoErrorPayload>(code, message, stackTrace, null, childErrors);
+ }
+
+ private LensErrorTO(final int code, final String message, final String stackTrace, final PAYLOAD errorPayload,
+ final List<LensErrorTO> childErrors) {
+
+ checkArgument(code > 0);
+ checkArgument(StringUtils.isNotBlank(message));
+ checkArgument(StringUtils.isNotBlank(stackTrace));
+
+ this.code = code;
+ this.message = message;
+ this.stackTrace = stackTrace;
+ this.payload = errorPayload;
+ this.childErrors = childErrors;
+ }
+
+ public boolean areValidStackTracesPresent() {
+
+ /* if stack trace of first level error is not valid, then return false */
+ if (StringUtils.isBlank(stackTrace)) {
+ return false;
+ }
+
+ /* validate stack traces of child Errors */
+ if (childErrors != null) {
+ for (LensErrorTO childError : childErrors) {
+ if (!childError.areValidStackTracesPresent()) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/result/LensJAXBContextResolver.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/result/LensJAXBContextResolver.java b/lens-api/src/main/java/org/apache/lens/api/result/LensJAXBContextResolver.java
new file mode 100644
index 0000000..5b48578
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/result/LensJAXBContextResolver.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.result;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+
+import org.apache.lens.api.error.ErrorCollection;
+import org.apache.lens.api.error.ErrorCollectionFactory;
+
+import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * @see javax.ws.rs.ext.ContextResolver
+ */
+@Provider
+@Slf4j
+public class LensJAXBContextResolver implements ContextResolver<JAXBContext> {
+
+ private Map<Class, JAXBContext> jaxbContextCache = new ConcurrentHashMap<Class, JAXBContext>();
+
+ @Override
+ public JAXBContext getContext(Class<?> type) {
+
+ JAXBContext jaxbContext = jaxbContextCache.get(type);
+
+ if (jaxbContext == null) {
+
+ log.debug("JAXB instance to be created for {}", type);
+ try {
+ if (type.equals(LensAPIResult.class)) {
+
+ ErrorCollection errorCollection = new ErrorCollectionFactory().createErrorCollection();
+ Set<Class> classesToBeBound = Sets.newHashSet(errorCollection.getErrorPayloadClasses());
+ log.debug("classesToBeBound:{}", classesToBeBound);
+ classesToBeBound.add(type);
+
+ Class[] classesToBeBoundArray = classesToBeBound.toArray(new Class[classesToBeBound.size()]);
+ jaxbContext = JAXBContext.newInstance(classesToBeBoundArray);
+ } else {
+
+ jaxbContext = JAXBContext.newInstance(type);
+ }
+ jaxbContextCache.put(type, jaxbContext);
+
+ } catch (JAXBException e) {
+ log.error("JAXBContext not initialized for "+type, e);
+ } catch (ClassNotFoundException e) {
+ log.error("JAXBContext not initialized for "+type, e);
+ }
+ }
+ return jaxbContext;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/result/NoErrorPayload.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/result/NoErrorPayload.java b/lens-api/src/main/java/org/apache/lens/api/result/NoErrorPayload.java
new file mode 100644
index 0000000..77810d3
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/result/NoErrorPayload.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.result;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import lombok.EqualsAndHashCode;
+
+/**
+ * NoErrorPayload type is to be used while creating LensResponse for success responses.
+ * Success Responses will not have any error payload.
+ *
+ */
+@XmlRootElement
+@EqualsAndHashCode
+public final class NoErrorPayload {
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/result/NoResultData.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/result/NoResultData.java b/lens-api/src/main/java/org/apache/lens/api/result/NoResultData.java
new file mode 100644
index 0000000..1b47d02
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/result/NoResultData.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.result;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * NoResultData type is to be used while creating LensAPIResult for error scenarios.
+ * Error scenarios will not have any data related to successful execution of API.
+ *
+ */
+@XmlRootElement
+public final class NoResultData {
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java b/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
new file mode 100644
index 0000000..e6152b8
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/result/PrettyPrintable.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.result;
+
+/**
+ * Contents of a class which implements this interface can be represented in a pretty formatted string.
+ */
+public interface PrettyPrintable {
+
+ /**
+ * Returns the contents of this object in the form of a pretty formatted string.
+ *
+ * @return
+ */
+ String toPrettyString();
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-api/src/main/resources/lens-errors.conf
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf
index 26bda1f..6130ad7 100644
--- a/lens-api/src/main/resources/lens-errors.conf
+++ b/lens-api/src/main/resources/lens-errors.conf
@@ -60,7 +60,7 @@ lensServerErrors = [
errorCode = 2003
httpStatusCode = ${BAD_REQUEST}
errorMsg = "Provided Operation is not supported. Supported Operations are: [%s]"
- payloadClass = org.apache.lens.server.query.SupportedQuerySubmitOperations
+ payloadClass = org.apache.lens.api.query.SupportedQuerySubmitOperations
}
]
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 928531e..c48fabd 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -26,8 +26,14 @@ import java.util.UUID;
import javax.ws.rs.core.Response;
import org.apache.lens.api.query.*;
+import org.apache.lens.api.result.PrettyPrintable;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
import org.apache.lens.client.LensClient;
+import org.apache.lens.client.exceptions.LensAPIException;
+import org.apache.lens.client.exceptions.LensBriefErrorException;
+import org.apache.lens.client.model.BriefError;
+import org.apache.lens.client.model.IdBriefErrorTemplate;
+import org.apache.lens.client.model.IdBriefErrorTemplateKey;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -74,17 +80,26 @@ public class LensQueryCommands extends BaseLensCommand {
@CliOption(key = {"async"}, mandatory = false, unspecifiedDefaultValue = "false",
specifiedDefaultValue = "true", help = "<async>") boolean async,
@CliOption(key = {"name"}, mandatory = false, help = "<query-name>") String queryName) {
- if (async) {
- QueryHandle handle = getClient().executeQueryAsynch(sql, queryName);
- return handle.getHandleId().toString();
- } else {
- try {
- LensClient.LensClientResultSetWithStats result = getClient().getResults(sql, queryName);
- return formatResultSet(result);
- } catch (Throwable t) {
- return t.getMessage();
+
+ PrettyPrintable cliOutput;
+
+ try {
+ if (async) {
+ QueryHandle queryHandle = getClient().executeQueryAsynch(sql, queryName).getData();
+ return queryHandle.getHandleIdString();
+ } else {
+ return formatResultSet(getClient().getResults(sql, queryName));
}
+ } catch (final LensAPIException e) {
+
+ BriefError briefError = new BriefError(e.getLensAPIErrorCode(), e.getLensAPIErrorMessage());
+ cliOutput = new IdBriefErrorTemplate(IdBriefErrorTemplateKey.REQUEST_ID, e.getLensAPIRequestId(), briefError);
+
+ } catch (final LensBriefErrorException e) {
+ cliOutput = e.getIdBriefErrorTemplate();
}
+
+ return cliOutput.toPrettyString();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-cli/src/test/java/org/apache/lens/cli/ExecuteQueryCommandIT.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/ExecuteQueryCommandIT.java b/lens-cli/src/test/java/org/apache/lens/cli/ExecuteQueryCommandIT.java
new file mode 100644
index 0000000..b3fcccb
--- /dev/null
+++ b/lens-cli/src/test/java/org/apache/lens/cli/ExecuteQueryCommandIT.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli;
+
+import static org.testng.Assert.assertTrue;
+
+import org.apache.lens.cli.commands.LensQueryCommands;
+import org.apache.lens.client.LensClient;
+
+import org.testng.annotations.Test;
+
+public class ExecuteQueryCommandIT extends LensCliApplicationTest {
+
+ @Test
+ public void testExecuteSyncQueryWithSyntaxError() {
+
+ LensQueryCommands lensQueryCommands = new LensQueryCommands();
+ lensQueryCommands.setClient(new LensClient());
+
+ final String actualResult = lensQueryCommands.executeQuery("mock-query", false, "testQuery");
+
+ assertTrue(actualResult.contains("Query Id: "));
+ assertTrue(actualResult.contains("\n" + "Error Code: 3001\n"
+ + "Error Message: Syntax Error: line 1:0 cannot recognize input near 'mock' '-' 'query'"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index 97f0cf0..46e28a9 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -225,7 +225,8 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String sql = "cube select id,name from test_dim";
long submitTime = System.currentTimeMillis();
String qh = qCom.executeQuery(sql, true, "testQuery1");
- String user = qCom.getClient().getLensStatement(new QueryHandle(UUID.fromString(qh))).getQuery().getSubmittedUser();
+ String user = qCom.getClient().getLensStatement(new QueryHandle(UUID.fromString(qh)))
+ .getQuery().getSubmittedUser();
String result = qCom.getAllQueries("", "testQuery1", user, -1, Long.MAX_VALUE);
// this is because previous query has run two query handle will be there
assertTrue(result.contains(qh), result);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/pom.xml
----------------------------------------------------------------------
diff --git a/lens-client/pom.xml b/lens-client/pom.xml
index e27a6ce..540c203 100644
--- a/lens-client/pom.xml
+++ b/lens-client/pom.xml
@@ -123,5 +123,25 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 7399d9e..51f586a 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -27,6 +27,12 @@ import javax.ws.rs.core.Response;
import org.apache.lens.api.APIResult;
import org.apache.lens.api.metastore.*;
import org.apache.lens.api.query.*;
+import org.apache.lens.api.result.LensAPIResult;
+import org.apache.lens.client.exceptions.LensAPIException;
+import org.apache.lens.client.exceptions.LensBriefErrorException;
+import org.apache.lens.client.model.BriefError;
+import org.apache.lens.client.model.IdBriefErrorTemplate;
+import org.apache.lens.client.model.IdBriefErrorTemplateKey;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -86,13 +92,13 @@ public class LensClient {
return mc;
}
- public QueryHandle executeQueryAsynch(String sql, String queryName) {
+ public LensAPIResult<QueryHandle> executeQueryAsynch(String sql, String queryName) throws LensAPIException {
LOG.debug("Executing query " + sql);
- statement.execute(sql, false, queryName);
+ LensAPIResult<QueryHandle> lensAPIResult = statement.execute(sql, false, queryName);
LensQuery query = statement.getQuery();
LOG.debug("Adding query to statementMap " + query.getQueryHandle());
statementMap.put(query.getQueryHandle(), statement);
- return query.getQueryHandle();
+ return lensAPIResult;
}
public Date getLatestDateOfCube(String cubeName, String timePartition) {
@@ -123,7 +129,7 @@ public class LensClient {
}
}
- public LensClientResultSetWithStats getResults(String sql, String queryName) {
+ public LensClientResultSetWithStats getResults(String sql, String queryName) throws LensAPIException {
LOG.debug("Executing query " + sql);
statement.execute(sql, true, queryName);
return getResultsFromStatement(statement);
@@ -132,8 +138,9 @@ public class LensClient {
private LensClientResultSetWithStats getResultsFromStatement(LensStatement statement) {
QueryStatus.Status status = statement.getStatus().getStatus();
if (status != QueryStatus.Status.SUCCESSFUL) {
- throw new IllegalStateException(statement.getStatus().getStatusMessage()
- + " cause:" + statement.getStatus().getErrorMessage());
+ IdBriefErrorTemplate errorResult = new IdBriefErrorTemplate(IdBriefErrorTemplateKey.QUERY_ID,
+ statement.getQueryHandleString(), new BriefError(statement.getErrorCode(), statement.getErrorMessage()));
+ throw new LensBriefErrorException(errorResult);
}
LensClientResultSet result = null;
if (statement.getStatus().isResultSetAvailable()) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
index 40e2b86..f7305fb 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
@@ -32,10 +32,8 @@ import org.apache.lens.api.APIResult;
import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
-import org.apache.lens.api.response.LensJAXBContextResolver;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
-
+import org.apache.lens.api.result.LensAPIResult;
+import org.apache.lens.client.exceptions.LensAPIException;
import org.apache.commons.lang.StringUtils;
@@ -71,9 +69,11 @@ public class LensStatement {
* @param waitForQueryToComplete the wait for query to complete
* @param queryName the query name
*/
- public void execute(String sql, boolean waitForQueryToComplete, String queryName) {
- QueryHandle handle = executeQuery(sql, waitForQueryToComplete, queryName);
- this.query = getQuery(handle);
+ public LensAPIResult<QueryHandle> execute(String sql, boolean waitForQueryToComplete,
+ String queryName) throws LensAPIException {
+ LensAPIResult<QueryHandle> lensAPIResult = executeQuery(sql, waitForQueryToComplete, queryName);
+ this.query = getQuery(lensAPIResult.getData());
+ return lensAPIResult;
}
/**
@@ -82,8 +82,8 @@ public class LensStatement {
* @param sql the sql
* @param queryName the query name
*/
- public void execute(String sql, String queryName) {
- QueryHandle handle = executeQuery(sql, true, queryName);
+ public void execute(String sql, String queryName) throws LensAPIException {
+ QueryHandle handle = executeQuery(sql, true, queryName).getData();
this.query = getQuery(handle);
}
@@ -95,13 +95,15 @@ public class LensStatement {
* @param queryName the query name
* @return the query handle
*/
- public QueryHandle executeQuery(String sql, boolean waitForQueryToComplete, String queryName) {
- QueryHandle handle = executeQuery(sql, queryName);
+ public LensAPIResult<QueryHandle> executeQuery(String sql, boolean waitForQueryToComplete,
+ String queryName) throws LensAPIException {
+
+ LensAPIResult<QueryHandle> lensAPIResult = executeQuery(sql, queryName);
if (waitForQueryToComplete) {
- waitForQueryToComplete(handle);
+ waitForQueryToComplete(lensAPIResult.getData());
}
- return handle;
+ return lensAPIResult;
}
/**
@@ -137,8 +139,8 @@ public class LensStatement {
WebTarget target = getPreparedQueriesWebTarget(client);
QueryPrepareHandle handle = target.request().post(
- Entity.entity(prepareForm(sql, "PREPARE", queryName), MediaType.MULTIPART_FORM_DATA_TYPE),
- QueryPrepareHandle.class);
+ Entity.entity(prepareForm(sql, "PREPARE", queryName), MediaType.MULTIPART_FORM_DATA_TYPE),
+ QueryPrepareHandle.class);
getPreparedQuery(handle);
return handle;
}
@@ -288,14 +290,12 @@ public class LensStatement {
* @param queryName the query name
* @return the query handle
*/
- private QueryHandle executeQuery(String sql, String queryName) {
+ private LensAPIResult<QueryHandle> executeQuery(String sql, String queryName) throws LensAPIException {
if (!connection.isOpen()) {
throw new IllegalStateException("Lens Connection has to be " + "established before querying");
}
- Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).register(LensJAXBContextResolver.class)
- .build();
-
+ Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).build();
FormDataMultiPart mp = new FormDataMultiPart();
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), connection
.getSessionHandle(), MediaType.APPLICATION_XML_TYPE));
@@ -306,8 +306,13 @@ public class LensStatement {
WebTarget target = getQueryWebTarget(client);
- return target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE));
+
+ if (response.getStatus() == Response.Status.OK.getStatusCode()) {
+ return response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {});
+ }
+
+ throw new LensAPIException(response.readEntity(LensAPIResult.class));
}
/**
@@ -358,7 +363,7 @@ public class LensStatement {
WebTarget target = getQueryWebTarget(client);
QueryPlan handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryPlan, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryPlan>>() {}).getData();
return handle;
}
@@ -396,8 +401,7 @@ public class LensStatement {
WebTarget target = getPreparedQueriesWebTarget(client);
List<QueryPrepareHandle> handles = target.queryParam("sessionid", connection.getSessionHandle())
.queryParam("user", userName).queryParam("queryName", queryName).queryParam("fromDate", fromDate)
- .queryParam("toDate", toDate).request().get(new GenericType<List<QueryPrepareHandle>>() {
- });
+ .queryParam("toDate", toDate).request().get(new GenericType<List<QueryPrepareHandle>>() {});
return handles;
}
@@ -561,4 +565,16 @@ public class LensStatement {
public LensQuery getQuery() {
return this.query;
}
+
+ public int getErrorCode() {
+ return this.query.getErrorCode();
+ }
+
+ public String getErrorMessage() {
+ return this.query.getErrorMessage();
+ }
+
+ public String getQueryHandleString() {
+ return this.query.getQueryHandleString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
new file mode 100644
index 0000000..fcc53c5
--- /dev/null
+++ b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensAPIException.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.client.exceptions;
+
+import static com.google.common.base.Preconditions.checkState;
+
+import org.apache.lens.api.result.LensAPIResult;
+
+import lombok.ToString;
+
+@ToString
+public class LensAPIException extends Exception {
+
+ private LensAPIResult errorResult;
+
+ public LensAPIException(final LensAPIResult lensAPIErrorResult) {
+ checkState(lensAPIErrorResult.isErrorResult());
+ this.errorResult = lensAPIErrorResult;
+ }
+
+ public int getLensAPIErrorCode() {
+ return this.errorResult.getErrorCode();
+ }
+
+ public String getLensAPIErrorMessage() {
+ return this.errorResult.getErrorMessage();
+ }
+
+ public String getLensAPIRequestId() {
+ return this.errorResult.getId();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/exceptions/LensBriefErrorException.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensBriefErrorException.java b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensBriefErrorException.java
new file mode 100644
index 0000000..ddfd777
--- /dev/null
+++ b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensBriefErrorException.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.client.exceptions;
+
+import org.apache.lens.client.model.IdBriefErrorTemplate;
+
+import lombok.Getter;
+import lombok.NonNull;
+
+public class LensBriefErrorException extends RuntimeException {
+
+ @Getter
+ private final IdBriefErrorTemplate idBriefErrorTemplate;
+
+ public LensBriefErrorException(@NonNull final IdBriefErrorTemplate idBriefErrorTemplate) {
+ this.idBriefErrorTemplate = idBriefErrorTemplate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/exceptions/LensClientException.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensClientException.java b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensClientException.java
index 1150726..1c03be7 100644
--- a/lens-client/src/main/java/org/apache/lens/client/exceptions/LensClientException.java
+++ b/lens-client/src/main/java/org/apache/lens/client/exceptions/LensClientException.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.client.exceptions;
+import org.apache.lens.api.result.LensAPIResult;
+
/**
* The Class LensClientException.
*/
@@ -26,12 +28,7 @@ public class LensClientException extends RuntimeException {
/** The Constant serialVersionUID. */
private static final long serialVersionUID = 1L;
- /** The message. */
- private final String message;
-
- /** The cause. */
- private Exception cause;
-
+ private LensAPIResult lensAPIErrorResult;
/**
* Instantiates a new lens client exception.
*
@@ -39,8 +36,7 @@ public class LensClientException extends RuntimeException {
* @param cause the cause
*/
public LensClientException(String message, Exception cause) {
- this.message = message;
- this.cause = cause;
+ super(message, cause);
}
/**
@@ -49,16 +45,6 @@ public class LensClientException extends RuntimeException {
* @param message the message
*/
public LensClientException(String message) {
- this.message = message;
- }
-
- @Override
- public String getMessage() {
- return message;
- }
-
- @Override
- public Exception getCause() {
- return cause;
+ super(message);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java b/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java
index 95d0ff7..10f7155 100644
--- a/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/jdbc/LensJdbcStatement.java
@@ -21,10 +21,14 @@ package org.apache.lens.client.jdbc;
import java.sql.*;
import org.apache.lens.client.LensStatement;
+import org.apache.lens.client.exceptions.LensAPIException;
+
+import lombok.extern.slf4j.Slf4j;
/**
* The Class LensJdbcStatement.
*/
+@Slf4j
public class LensJdbcStatement implements Statement {
/** The connection. */
@@ -53,7 +57,11 @@ public class LensJdbcStatement implements Statement {
*/
@Override
public ResultSet executeQuery(String s) throws SQLException {
- statement.execute(s, null);
+ try {
+ statement.execute(s, null);
+ } catch (LensAPIException e) {
+ log.error("Execution Failed for Statement:{}", s, e);
+ }
return new LensJdbcResultSet(statement.getResultSet(), statement.getResultSetMetaData(), this);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/model/BriefError.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/model/BriefError.java b/lens-client/src/main/java/org/apache/lens/client/model/BriefError.java
new file mode 100644
index 0000000..f2b37ea
--- /dev/null
+++ b/lens-client/src/main/java/org/apache/lens/client/model/BriefError.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.client.model;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.lens.api.result.PrettyPrintable;
+
+import org.apache.commons.lang.StringUtils;
+
+public class BriefError implements PrettyPrintable {
+
+ private final int errorCode;
+ private final String errorMsg;
+
+ public BriefError(final int errorCode, final String errorMsg) {
+
+ checkArgument(errorCode > 0);
+ checkArgument(StringUtils.isNotBlank(errorMsg));
+ this.errorCode = errorCode;
+ this.errorMsg = errorMsg;
+ }
+
+ @Override
+ public String toPrettyString() {
+
+ StringBuilder sb = new StringBuilder("Error Code: ").append(this.errorCode).append("\n").append("Error Message: ")
+ .append(this.errorMsg);
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplate.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplate.java b/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplate.java
new file mode 100644
index 0000000..8a485b5
--- /dev/null
+++ b/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplate.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.client.model;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import org.apache.lens.api.result.PrettyPrintable;
+
+import org.apache.commons.lang.StringUtils;
+
+import lombok.NonNull;
+
+public class IdBriefErrorTemplate implements PrettyPrintable {
+
+ private final IdBriefErrorTemplateKey idKey;
+ private final String idValue;
+ private final BriefError briefError;
+
+ public IdBriefErrorTemplate(@NonNull final IdBriefErrorTemplateKey idKey, final String idValue,
+ @NonNull BriefError briefError) {
+
+ checkArgument(StringUtils.isNotBlank(idValue));
+ this.idKey = idKey;
+ this.idValue = idValue;
+ this.briefError = briefError;
+ }
+
+ @Override
+ public String toPrettyString() {
+
+ StringBuilder sb = new StringBuilder(idKey.getConstant()).append(": ").append(this.idValue).append("\n")
+ .append(this.briefError.toPrettyString());
+
+ return sb.toString();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplateKey.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplateKey.java b/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplateKey.java
new file mode 100644
index 0000000..41e7177
--- /dev/null
+++ b/lens-client/src/main/java/org/apache/lens/client/model/IdBriefErrorTemplateKey.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.client.model;
+
+public enum IdBriefErrorTemplateKey {
+
+ REQUEST_ID("Request Id"),
+ QUERY_ID("Query Id");
+
+ IdBriefErrorTemplateKey(final String key) {
+ this.key = key;
+ }
+
+ public String getConstant() {
+ return this.key;
+ }
+
+ private final String key;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/test/java/org/apache/lens/client/model/BriefErrorTest.java
----------------------------------------------------------------------
diff --git a/lens-client/src/test/java/org/apache/lens/client/model/BriefErrorTest.java b/lens-client/src/test/java/org/apache/lens/client/model/BriefErrorTest.java
new file mode 100644
index 0000000..90e2fbe
--- /dev/null
+++ b/lens-client/src/test/java/org/apache/lens/client/model/BriefErrorTest.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.client.model;
+
+import static org.testng.Assert.assertEquals;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class BriefErrorTest {
+
+ @Test
+ public void testToPrettyString() {
+
+ final int testErrCode = 1001;
+ final String testErrorMsg = "Test Error Msg, adfg-asdfk $ %, \n";
+ BriefError briefError = new BriefError(testErrCode, testErrorMsg);
+
+ final String actualPrettyString = briefError.toPrettyString();
+ final String expectedPrettyString = "Error Code: 1001\nError Message: Test Error Msg, adfg-asdfk $ %, \n";
+
+ assertEquals(actualPrettyString, expectedPrettyString);
+ }
+
+ @DataProvider(name="dpInvalidErrorCodes")
+ public Object[][] dpInvalidErrorCodes() {
+ return new Object[][] {{-1}, {0}};
+ }
+
+ @Test(dataProvider = "dpInvalidStrings", expectedExceptions = IllegalArgumentException.class)
+ public void testBriefErrorMustNotAcceptInvalidErrorCodes(final int invalidErrCode) {
+ new BriefError(invalidErrCode, "Valid Error Message");
+ }
+
+ @DataProvider(name="dpInvalidStrings")
+ public Object[][] dpInvalidStrings() {
+ return new Object[][] {{null}, {""}, {" "}};
+ }
+
+ @Test(dataProvider = "dpInvalidStrings", expectedExceptions = IllegalArgumentException.class)
+ public void testBriefErrorMustNotAcceptInvalidErrorMsg(final String invalidErrMsg) {
+ new BriefError(1001, invalidErrMsg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-client/src/test/java/org/apache/lens/client/model/IdBriefErrorTemplateTest.java
----------------------------------------------------------------------
diff --git a/lens-client/src/test/java/org/apache/lens/client/model/IdBriefErrorTemplateTest.java b/lens-client/src/test/java/org/apache/lens/client/model/IdBriefErrorTemplateTest.java
new file mode 100644
index 0000000..9d9ec0e
--- /dev/null
+++ b/lens-client/src/test/java/org/apache/lens/client/model/IdBriefErrorTemplateTest.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.client.model;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class IdBriefErrorTemplateTest {
+
+ @Test
+ public void testToPrettyString() {
+
+ /* Setup Dependencies */
+ final IdBriefErrorTemplateKey testQueryIdKey = IdBriefErrorTemplateKey.QUERY_ID;
+ final String testQueryIdValue = "TestQueryIdValue";
+ final BriefError mockBriefError = mock(BriefError.class);
+
+ /* Stubbing */
+ final String testBriefErrorPrettyString = "TestLensBriefErrorPrettyString";
+ when(mockBriefError.toPrettyString()).thenReturn(testBriefErrorPrettyString);
+
+ /* Execution */
+ IdBriefErrorTemplate template = new IdBriefErrorTemplate(testQueryIdKey, testQueryIdValue, mockBriefError);
+ final String actualPrettyString = template.toPrettyString();
+
+ /* Verfication */
+ final String expectedPrettyString = "Query Id: TestQueryIdValue\nTestLensBriefErrorPrettyString";
+ assertEquals(actualPrettyString, expectedPrettyString);
+ }
+
+ @DataProvider(name="dpInvalidStrings")
+ public Object[][] dpInvalidStrings() {
+ return new Object[][] {{null}, {""}, {" "}};
+ }
+
+ @Test(expectedExceptions = NullPointerException.class)
+ public void testIdBriefErrorTemplateMustNotAcceptNullIdKey() {
+
+ final BriefError mockBriefError = mock(BriefError.class);
+ new IdBriefErrorTemplate(null, "ValidIdValue", mockBriefError);
+ }
+
+ @Test(dataProvider = "dpInvalidStrings", expectedExceptions = IllegalArgumentException.class)
+ public void testIdBriefErrorTemplateMustNotAcceptInvalidIdValue(final String invalidIdValue) {
+
+ final BriefError mockBriefError = mock(BriefError.class);
+ new IdBriefErrorTemplate(IdBriefErrorTemplateKey.QUERY_ID, invalidIdValue, mockBriefError);
+ }
+
+ @Test(expectedExceptions = NullPointerException.class)
+ public void testIdBriefErrorTemplateMustNotAcceptNullBriefError() {
+ new IdBriefErrorTemplate(IdBriefErrorTemplateKey.QUERY_ID, "ValidIdValue", null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-cube/src/main/java/org/apache/lens/cube/error/ColUnAvailableInTimeRangeException.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/ColUnAvailableInTimeRangeException.java b/lens-cube/src/main/java/org/apache/lens/cube/error/ColUnAvailableInTimeRangeException.java
index 99a78f6..dd3bb72 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/ColUnAvailableInTimeRangeException.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/ColUnAvailableInTimeRangeException.java
@@ -22,7 +22,7 @@ import static org.apache.lens.cube.error.LensCubeErrorCode.COLUMN_UNAVAILABLE_IN
import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.error.LensError;
-import org.apache.lens.api.response.LensErrorTO;
+import org.apache.lens.api.result.LensErrorTO;
import org.apache.lens.server.api.error.LensException;
import lombok.EqualsAndHashCode;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-cube/src/main/java/org/apache/lens/cube/error/FieldsCannotBeQueriedTogetherException.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/error/FieldsCannotBeQueriedTogetherException.java b/lens-cube/src/main/java/org/apache/lens/cube/error/FieldsCannotBeQueriedTogetherException.java
index f246246..65b96d7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/error/FieldsCannotBeQueriedTogetherException.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/error/FieldsCannotBeQueriedTogetherException.java
@@ -22,7 +22,7 @@ import static org.apache.lens.cube.error.LensCubeErrorCode.FIELDS_CANNOT_BE_QUER
import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.error.LensError;
-import org.apache.lens.api.response.LensErrorTO;
+import org.apache.lens.api.result.LensErrorTO;
import org.apache.lens.server.api.error.LensException;
import lombok.EqualsAndHashCode;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
index 6c820a4..6b662e7 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleQueries.java
@@ -29,6 +29,7 @@ import org.apache.lens.api.query.*;
import org.apache.lens.client.LensClientSingletonWrapper;
import org.apache.lens.client.LensMetadataClient;
import org.apache.lens.client.LensStatement;
+import org.apache.lens.client.exceptions.LensAPIException;
import org.apache.commons.lang.StringUtils;
@@ -99,7 +100,7 @@ public class SampleQueries {
*
* @throws IOException Signals that an I/O exception has occurred.
*/
- public void queryAll() throws IOException {
+ public void queryAll() throws IOException, LensAPIException {
runQueries("dimension-queries.sql");
runQueries("cube-queries.sql");
System.out.println("Successful queries " + success + " out of " + total + "queries");
@@ -117,7 +118,7 @@ public class SampleQueries {
* @param fileName the file name
* @throws IOException Signals that an I/O exception has occurred.
*/
- public void runQueries(String fileName) throws IOException {
+ public void runQueries(String fileName) throws IOException, LensAPIException {
InputStream file = SampleMetastore.class.getClassLoader().getResourceAsStream(fileName);
BufferedReader reader = new BufferedReader(new InputStreamReader(file, "UTF-8"));
String query;
@@ -131,7 +132,7 @@ public class SampleQueries {
}
total++;
System.out.println("Query:" + query);
- QueryHandle handle = queryClient.executeQuery(query, true, null);
+ QueryHandle handle = queryClient.executeQuery(query, true, null).getData();
System.out.println("Status:" + queryClient.getQuery().getStatus());
System.out.println("Total time in millis:"
+ (queryClient.getQuery().getFinishTime() - queryClient.getQuery().getSubmissionTime()));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
index de76603..50c22f4 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
@@ -38,8 +38,7 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.ml.algo.api.MLAlgo;
import org.apache.lens.ml.algo.api.MLDriver;
import org.apache.lens.ml.algo.api.MLModel;
@@ -647,7 +646,7 @@ public class LensMLImpl implements LensML {
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", sessionHandle).request()
.get(LensQuery.class);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
index fe4d926..0efa10d 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
@@ -28,6 +28,7 @@ import java.util.UUID;
import org.apache.lens.api.query.*;
import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientResultSet;
+import org.apache.lens.client.exceptions.LensAPIException;
import org.apache.lens.ml.algo.spark.HiveTableRDD;
import org.apache.lens.server.api.error.LensException;
@@ -169,8 +170,8 @@ public class LensRDDClient {
* @return the query handle
* @throws LensException the lens exception
*/
- public QueryHandle createLensRDDAsync(String query) throws LensException {
- return getClient().executeQueryAsynch(query, "");
+ public QueryHandle createLensRDDAsync(String query) throws LensAPIException {
+ return getClient().executeQueryAsynch(query, "").getData();
}
/**
@@ -305,7 +306,7 @@ public class LensRDDClient {
* @return the lens rdd result
* @throws LensException the lens exception
*/
- public LensRDDResult createLensRDD(String query) throws LensException {
+ public LensRDDResult createLensRDD(String query) throws LensAPIException, LensException {
QueryHandle queryHandle = createLensRDDAsync(query);
while (!isReadyForRDD(queryHandle)) {
try {
[50/51] [abbrv] incubator-lens git commit: LENS-513 - add jar should
be able to take regex path and should be able to add multiple jars
Posted by jd...@apache.org.
LENS-513 - add jar should be able to take regex path and should be able to add multiple jars
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/4dcff2c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/4dcff2c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/4dcff2c2
Branch: refs/heads/current-release-line
Commit: 4dcff2c2658c2ba9c25839e67f6faf1fb0f451b3
Parents: 1a06e77
Author: Yash Sharma <ya...@gmail.com>
Authored: Mon Jun 22 19:25:41 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Jun 22 19:25:41 2015 +0530
----------------------------------------------------------------------
.../lens/server/session/SessionResource.java | 28 +-
.../apache/lens/server/util/ScannedPaths.java | 209 ++++++-------
.../lens/server/util/TestScannedPaths.java | 302 ++++++++++++++++++-
3 files changed, 405 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4dcff2c2/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
index 4d82a06..a5033fe 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
@@ -19,7 +19,6 @@
package org.apache.lens.server.session;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -143,17 +142,14 @@ public class SessionResource {
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
public APIResult addResource(@FormDataParam("sessionid") LensSessionHandle sessionid,
@FormDataParam("type") String type, @FormDataParam("path") String path) {
- Iterator<String> foundFiles = new ScannedPaths(path, type).iterator();
- String matchedPath = null;
+ ScannedPaths scannedPaths = new ScannedPaths(path, type);
int matchedPathsCount = 0;
- if (foundFiles == null) {
- return new APIResult(Status.FAILED, "No matching resources found for provided path.");
- }
-
int numAdded = 0;
- while (foundFiles.hasNext()) {
- matchedPath = foundFiles.next();
+ for (String matchedPath : scannedPaths) {
+ if (matchedPath.startsWith("file:") && !matchedPath.startsWith("file://")) {
+ matchedPath = "file://" + matchedPath.substring("file:".length());
+ }
numAdded += sessionService.addResourceToAllServices(sessionid, type, matchedPath);
matchedPathsCount++;
}
@@ -203,20 +199,16 @@ public class SessionResource {
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
public APIResult deleteResource(@FormDataParam("sessionid") LensSessionHandle sessionid,
@FormDataParam("type") String type, @FormDataParam("path") String path) {
- Iterator<String> foundFiles = new ScannedPaths(path, type).iterator();
- String matchedPath = null;
-
- if (foundFiles == null) {
- return new APIResult(Status.PARTIAL, "No matching resources found for provided path.");
- }
+ ScannedPaths scannedPaths = new ScannedPaths(path, type);
int numDeleted = 0;
- while(foundFiles.hasNext()) {
- matchedPath = foundFiles.next();
-
+ for(String matchedPath : scannedPaths) {
for (LensService service : LensServices.get().getLensServices()) {
try {
+ if (matchedPath.startsWith("file:") && !matchedPath.startsWith("file://")) {
+ matchedPath = "file://" + matchedPath.substring("file:".length());
+ }
service.deleteResource(sessionid, type, matchedPath);
numDeleted++;
} catch (LensException e) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4dcff2c2/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java
index ac773fe..92687c3 100644
--- a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java
+++ b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java
@@ -26,36 +26,26 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-
import org.apache.hadoop.fs.Path;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
+
@Slf4j
public class ScannedPaths implements Iterable<String> {
private String path = null;
private String type = null;
- private Path fsPath;
-
- /** Is the provided expression parsed **/
- private boolean isScanned;
-
- /* Keep all matched paths so we don't have to query filesystem multiple times */
- private Map<String, String> matchedPaths = null;
-
/* The Chosen Ones */
- @Getter(lazy=true) private final List<String> finalPaths = getMatchedPaths();
+ @Getter(lazy=true) private final List<String> finalPaths = getMatchedPaths(path, type);
public ScannedPaths(String path, String type) {
this.path = path;
@@ -64,8 +54,10 @@ public class ScannedPaths implements Iterable<String> {
@Override
public Iterator<String> iterator() {
- /** Does all the pattern matching and returns the iterator to finalPaths collection **/
- return (getFinalPaths() == null) ? null : getFinalPaths().iterator();
+ /** Does all the pattern matching and returns the iterator to finalPaths collection.
+ * finalPaths should never be null.
+ **/
+ return getFinalPaths().iterator();
}
/**
@@ -75,123 +67,114 @@ public class ScannedPaths implements Iterable<String> {
*
* Updates finalPaths List with matched paths and returns an iterator for matched paths.
*/
- private List<String> getMatchedPaths() {
- List<String> finalPaths = null;
+ private List<String> getMatchedPaths(String path, String type) {
+ List<String> finalPaths = new ArrayList<>();
+ FileSystem fs;
+
try {
- FileSystem fs = FileSystem.get(new URI(path), new Configuration());
- fsPath = new Path(new URI(path).getPath());
-
- if (fs.isDirectory(fsPath)) {
- findAllMatchedPaths(true);
- filterByJarType();
- /* Updates finalPaths List with restrictions imposed
- by jar_order/glob_order file */
- finalPaths = getMatchedPathsFilteredByOrder();
+ fs = FileSystem.get(new URI(path), new Configuration());
+ Path pt = new Path(new URI(path));
+
+ if (fs.exists(pt) && fs.isFile(pt)) {
+ /**
+ * CASE 1 : Direct FILE provided in path
+ **/
+ finalPaths.add(pt.toUri().toString());
+ } else if (fs.exists(pt) && fs.isDirectory(pt)) {
+ /**
+ * CASE 2 : DIR provided in path
+ **/
+ Path resourceOrderFile = null;
+ InputStream resourceOrderIStream = null;
+ FileStatus[] statuses;
+ List<String> newMatches;
+ List<String> resources;
+ boolean resourceFileFound = false;
+
+ fs = pt.getFileSystem(new Configuration());
+ resourceOrderFile = new Path(pt, "jar_order");
+ /** Add everything in dir if no jar_order or glob_order is present **/
+ if (!fs.exists(resourceOrderFile)) {
+ resourceOrderFile = new Path(pt, "glob_order");
+ if (!fs.exists(resourceOrderFile)) {
+ /** Get matched resources recursively for all files **/
+ statuses = fs.globStatus(new Path(pt, "*"));
+ for (FileStatus st : statuses) {
+ finalPaths.add(st.getPath().toUri().toString());
+ }
+ } else {
+ resourceFileFound = true;
+ }
+ } else {
+ resourceFileFound = true;
+ }
+
+ if (resourceFileFound) {
+ /** Else get jars as per order specified in jar_order/glob_order **/
+ resourceOrderIStream = fs.open(resourceOrderFile);
+ resources = IOUtils.readLines(resourceOrderIStream, Charset.forName("UTF-8"));
+ for (String resource : resources) {
+ if (resource == null || resource.isEmpty()) {
+ continue;
+ }
+
+ /** Get matched resources recursively for provided path/pattern **/
+ if (resource.startsWith("/") || resource.contains(":/")) {
+ newMatches = getMatchedPaths(new Path(resource).toString(), type);
+ } else {
+ newMatches = getMatchedPaths(new Path(pt, resource).toString(), type);
+ }
+
+ if (newMatches != null) {
+ finalPaths.addAll(newMatches);
+ }
+ }
+ }
} else {
- findAllMatchedPaths(false);
- filterByJarType();
- if (matchedPaths != null) {
- finalPaths = new ArrayList<String>(matchedPaths.values());
+ /**
+ * CASE 3 : REGEX provided in path
+ * */
+ FileStatus[] statuses = fs.globStatus(pt);
+ for (FileStatus st : statuses) {
+ List<String> newMatches = getMatchedPaths(st.getPath().toString(), type);
+ if (newMatches != null) {
+ finalPaths.addAll(newMatches);
+ }
}
}
+
+ filterDirsAndJarType(fs, finalPaths);
+
} catch (FileNotFoundException fex) {
log.error("File not found while scanning path.", fex);
} catch (URISyntaxException | IOException ex) {
log.error("Exception while initializing PathScanner.", ex);
+ } catch (Exception e) {
+ log.error("Exception while initializing PathScanner.", e);
}
+
return finalPaths;
}
- /**
- * Populates the matchedPaths[] with all paths matching the pattern.
- */
- private void findAllMatchedPaths(boolean isDir) {
- try {
- Path path = isDir ? new Path(fsPath, "*") : fsPath;
- FileStatus[] statuses = path.getFileSystem(new Configuration()).globStatus(path);
-
- if (statuses == null || statuses.length == 0) {
- log.info("No matched paths found for expression " + path);
- return;
- }
- matchedPaths = new HashMap<String, String>();
- for (int count = 0; count < statuses.length; count++) {
- matchedPaths.put(statuses[count].getPath().getName(), statuses[count].getPath().toString());
- }
- } catch (FileNotFoundException fex) {
- log.error("File not found while scanning path.", fex);
- return;
- } catch (IOException ioex) {
- log.error("IOException while scanning path.", ioex);
- return;
- }
- }
/**
- * Filters the matchedPaths by "jar" type.
- * Removes non-jar resources if type is specified as "jar".
- */
- private void filterByJarType() {
- if (matchedPaths == null) {
- return;
- } else if (type.equalsIgnoreCase("jar")) {
- Iterator<Map.Entry<String, String>> iter = matchedPaths.entrySet().iterator();
-
+ * Skip Dirs from matched regex.
+ * We are interested only in file resources.
+ **/
+ private void filterDirsAndJarType(FileSystem fs, List<String> matches) {
+ try {
+ Iterator<String> iter = matches.iterator();
+ String path;
while (iter.hasNext()) {
- Map.Entry<String, String> entry = iter.next();
- if (!entry.getKey().endsWith(".jar")) {
+ path = iter.next();
+ if (fs.isDirectory(new Path(path))) {
+ iter.remove();
+ } else if (type.equalsIgnoreCase("jar") && !path.endsWith(".jar")) {
iter.remove();
}
}
+ } catch (IOException e) {
+ log.error("Exception while initializing filtering dirs.", e);
}
}
-
- /**
- * Filters the matchedPath[] to remove unwanted resources
- * and apply ordering to the resources as specified in jar_order or glob_order file.
- * Bypasses filtering if none of the files is present in the directory.
- */
- private List<String> getMatchedPathsFilteredByOrder() {
- if (matchedPaths == null) {
- return null;
- }
-
- List<String> finalPaths = null;
- InputStream resourceOrderIStream = null;
- List<String> resources;
- try {
- FileSystem fs = fsPath.getFileSystem(new Configuration());
- Path resourceOrderFile = new Path(fsPath.toUri().getPath(), "jar_order");
-
- if (!fs.exists(resourceOrderFile)) {
- resourceOrderFile = new Path(fsPath.toUri().getPath(), "glob_order");
- if (!fs.exists(resourceOrderFile)) {
- /* No order file present. Bypass filtering and Add all resource matching pattern */
- return new ArrayList<>(matchedPaths.values());
- }
- }
-
- resourceOrderIStream = fs.open(resourceOrderFile);
- resources = IOUtils.readLines(resourceOrderIStream, Charset.forName("UTF-8"));
- finalPaths = new ArrayList<>();
- for(String resource : resources) {
- if (resource == null || resource.isEmpty()) {
- continue;
- }
-
- if (matchedPaths.containsKey(resource)) {
- finalPaths.add(matchedPaths.get(resource));
- }
- }
- } catch (FileNotFoundException fex) {
- log.error("File not found while scanning path.", fex);
- finalPaths = null;
- } catch (IOException ioex) {
- log.error("IOException while scanning path.", ioex);
- finalPaths = null;
- } finally {
- IOUtils.closeQuietly(resourceOrderIStream);
- }
- return finalPaths;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4dcff2c2/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java b/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
index 5073634..ac9faf2 100644
--- a/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
+++ b/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
@@ -29,6 +29,7 @@ import org.apache.commons.io.FileUtils;
import org.testng.Assert;
import org.testng.annotations.Test;
+import com.google.common.base.Joiner;
import com.google.common.collect.Iterators;
import lombok.extern.slf4j.Slf4j;
@@ -64,7 +65,7 @@ public class TestScannedPaths {
fileRegex = tempPath + "tempdata_unknown_*";
sc = new ScannedPaths(fileRegex, "file");
- Assert.assertNull(sc.iterator(), "Iterator should be null for unmatched path patterns.");
+ Assert.assertFalse(sc.iterator().hasNext(), "Iterator should be empty for unmatched path patterns.");
} catch (Exception e) {
Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
} finally {
@@ -126,7 +127,303 @@ public class TestScannedPaths {
}
}
- private File createNewPath(String fileName) {
+ /**
+ * Private method used by testScannedPathsMultipleJarGlobOrder.
+ * Creates temp dir structure with files.
+ *
+ * Dir structure created:
+ * sourceDirPath/tempfiles/tempdata_a.jar
+ * sourceDirPath/tempfiles/tempdata_c.jar
+ * sourceDirPath/tempfiles/dir1/tempdata_a.jar
+ * sourceDirPath/tempfiles/dir1/tempdata_b.jar
+ * sourceDirPath/tempfiles/dir1/tempdata_c.data
+ * sourceDirPath/tempfiles/dir2/tempdata_a.data
+ * sourceDirPath/tempfiles/dir2/tempdata_b.data
+ * sourceDirPath/tempfiles/dir2/tempdata_c.jar
+ * sourceDirPath/tempfiles/dir3/tempdata_a.jar
+ * sourceDirPath/tempfiles/dir3/tempdata_b.data
+ * sourceDirPath/tempfiles/dir3/tempdata_c-duplicate.jar
+ * sourceDirPath/tempfiles/dir3/tempdata_c.jar
+ *
+ * // Additional regex checks for regex in jar_order/glob_order
+ * sourceDirPath/tempfiles/dir3/innerDirA/inceptionLevel2/tempdata_c.jar
+ * sourceDirPath/tempfiles/dir3/innerDirA/inceptionLevel2/tempdata_c-duplicate.jar
+ * sourceDirPath/tempfiles/dir3/innerDirB/inceptionLevel2/tempdata_c-duplicate-2.jar
+ * @param sourceDirPath
+ */
+ private void createTempDirStructure1(String sourceDirPath) {
+ File jarFile = null, globFile = null;
+ String filenameA, filenameB, filenameC;
+ String tempPath = sourceDirPath + "/tempfiles/";
+ FileUtils.deleteQuietly(new File(tempPath));
+
+ String jarExtension = ".jar";
+ String dataExtension = ".data";
+ String dir1 = "dir1/", dir2 = "dir2/", dir3 = "dir3/";
+
+ filenameA = "tempdata_a";
+ filenameB = "tempdata_b";
+ filenameC = "tempdata_c";
+
+ PrintWriter writer;
+
+ try {
+ createNewPath(tempPath, filenameC, jarExtension);
+ createNewPath(tempPath, filenameA, jarExtension);
+ createNewPath(tempPath, dir1, filenameA, jarExtension);
+ createNewPath(tempPath, dir1, filenameB, jarExtension);
+ createNewPath(tempPath, dir1, filenameC, dataExtension);
+ createNewPath(tempPath, dir2, filenameA, dataExtension);
+ createNewPath(tempPath, dir2, filenameB, dataExtension);
+ createNewPath(tempPath, dir2, filenameC, jarExtension);
+ createNewPath(tempPath, dir3, filenameA, jarExtension);
+ createNewPath(tempPath, dir3, filenameB, dataExtension);
+ createNewPath(tempPath, dir3, filenameC, jarExtension);
+ createNewPath(tempPath, dir3, filenameC, "-duplicate", jarExtension);
+
+ /** Additional paths for inner dirs **/
+ createNewPath(tempPath, dir3, "innerDirA/inceptionLevel2/", filenameC, jarExtension);
+ createNewPath(tempPath, dir3, "innerDirA/inceptionLevel2/", filenameC, "-duplicate", jarExtension);
+ createNewPath(tempPath, dir3, "innerDirB/inceptionLevel2/", filenameC, "-duplicate-2", jarExtension);
+
+ /** Create jar_order **/
+ jarFile = createNewPath(tempPath, dir1, "jar_order");
+ writer = new PrintWriter(jarFile, "UTF-8");
+ writer.println(filenameB + jarExtension);
+ writer.println(filenameA + jarExtension);
+ writer.close();
+ jarFile = createNewPath(tempPath, dir2, "jar_order");
+ writer = new PrintWriter(jarFile, "UTF-8");
+ writer.println(filenameC + jarExtension);
+ writer.close();
+ jarFile = createNewPath(tempPath, dir3, "jar_order");
+ writer = new PrintWriter(jarFile, "UTF-8");
+ writer.println(filenameC + "-duplicate" + jarExtension);
+ writer.println(filenameA + jarExtension);
+ writer.close();
+
+ /** Create glob_order **/
+ globFile = createNewPath(tempPath, dir1, "glob_order");
+ writer = new PrintWriter(globFile, "UTF-8");
+ writer.println(filenameC + dataExtension);
+ writer.println(filenameB + jarExtension);
+ writer.println(filenameA + jarExtension);
+ writer.close();
+
+ globFile = createNewPath(tempPath, dir3, "glob_order");
+ writer = new PrintWriter(globFile, "UTF-8");
+ writer.println(filenameC + jarExtension);
+ writer.println(filenameC + "-duplicate" + jarExtension);
+ /** Check regex compatibility in order files **/
+ writer.println("inner*/*Level*/" + filenameC + "-duplicate*" + jarExtension);
+ writer.close();
+
+
+
+ } catch (Exception ex) {
+ Assert.fail("Exception while creating temp paths : " + ex.getMessage());
+ }
+ }
+
+
+ /**
+ * Private method used by testScannedPathsRecursive.
+ * Creates temp dir structure with files.
+ *
+ * Dir structure created:
+ * sourceDirPath/tempfiles/a_dir/jar_1
+ * sourceDirPath/tempfiles/a_dir/jar_2
+ * sourceDirPath/tempfiles/a_dir/jar_order (*1\n*2)
+ *
+ * sourceDirPath/tempfiles/b_dir/jar_3
+ * sourceDirPath/tempfiles/b_dir/jar_4
+ * sourceDirPath/tempfiles/b_dir/glob_order (*4\n*3)
+ *
+ * sourceDirPath/tempfiles/c_dir/jar_5
+ * sourceDirPath/tempfiles/c_dir/jar_6
+ *
+ * sourceDirPath/tempfiles/jar_order (a*\nb*\nc*)
+ *
+ * @param sourceDirPath
+ */
+ private void createTempDirStructure2(String sourceDirPath) {
+ File orderFile = null;
+ String filenameA, filenameB, filenameC;
+ String tempPath = sourceDirPath + "/parent_dir/";
+ FileUtils.deleteQuietly(new File(tempPath));
+
+ PrintWriter writer;
+
+ try {
+ createNewPath(tempPath, "a_dir/jar_1");
+ createNewPath(tempPath, "a_dir/jar_2");
+ createNewPath(tempPath, "b_dir/jar_3");
+ createNewPath(tempPath, "b_dir/jar_4");
+ createNewPath(tempPath, "c_dir/jar_5");
+ createNewPath(tempPath, "c_dir/jar_6");
+
+
+ /** Create jar_order **/
+ orderFile = createNewPath(tempPath, "a_dir/jar_order");
+ writer = new PrintWriter(orderFile, "UTF-8");
+ writer.println("*1");
+ writer.println("*2");
+ writer.close();
+
+ orderFile = createNewPath(tempPath, "b_dir/glob_order");
+ writer = new PrintWriter(orderFile, "UTF-8");
+ writer.println("*4");
+ writer.println("*3");
+ writer.close();
+
+ orderFile = createNewPath(tempPath, "jar_order");
+ writer = new PrintWriter(orderFile, "UTF-8");
+ writer.println("a*");
+ writer.println("b*");
+ writer.println("c*");
+ writer.close();
+
+ } catch (Exception ex) {
+ Assert.fail("Exception while creating temp paths : " + ex.getMessage());
+ }
+ }
+
+ public void testScannedPathsMultipleJarGlobOrder() throws Exception {
+ ScannedPaths sc;
+ Iterator<String> iter = null;
+ String tempPath = "target/test/";
+
+ String filenameA = "tempdata_a";
+ String filenameB = "tempdata_b";
+ String filenameC = "tempdata_c";
+
+ String fileRegex1 = tempPath + "*";
+ String fileRegex2 = tempPath + "*/*";
+
+ try {
+ createTempDirStructure1(tempPath);
+
+ sc = new ScannedPaths(fileRegex1, "jar");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains(filenameC));
+ Assert.assertTrue(iter.next().contains(filenameA));
+
+ sc = new ScannedPaths(fileRegex2, "jar");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 7, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains(filenameB));
+ Assert.assertTrue(iter.next().contains(filenameA));
+ Assert.assertTrue(iter.next().contains(filenameC)); // Direct matched tempdata_c.jar
+ Assert.assertTrue(iter.next().contains(filenameC));
+ Assert.assertTrue(iter.next().contains(filenameC + "-duplicate"));
+ Assert.assertTrue(iter.next().contains(filenameA));
+ Assert.assertTrue(iter.next().contains(filenameA)); // Direct matched tempdata_a.jar
+
+ /** Remove jar_order files from temp dir **/
+ FileUtils.deleteQuietly(new File(tempPath + "tempfiles/dir1/jar_order"));
+ FileUtils.deleteQuietly(new File(tempPath + "tempfiles/dir2/jar_order"));
+ FileUtils.deleteQuietly(new File(tempPath + "tempfiles/dir3/jar_order"));
+
+ sc = new ScannedPaths(fileRegex1, "file");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains(filenameC));
+ Assert.assertTrue(iter.next().contains(filenameA));
+
+ sc = new ScannedPaths(fileRegex2, "file");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 12, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains(filenameC));
+ Assert.assertTrue(iter.next().contains(filenameB));
+ Assert.assertTrue(iter.next().contains(filenameA));
+ Assert.assertTrue(iter.next().contains(filenameC)); // Direct matched tempdata_c.jar
+ Assert.assertTrue(iter.next().contains(filenameC));
+ Assert.assertTrue(iter.next().contains(filenameB));
+ Assert.assertTrue(iter.next().contains(filenameA));
+ Assert.assertTrue(iter.next().contains(filenameC));
+ Assert.assertTrue(iter.next().contains(filenameC + "-duplicate"));
+ Assert.assertTrue(iter.next().contains(filenameC + "-duplicate-2")); // Inner dir regex match
+ Assert.assertTrue(iter.next().contains(filenameC + "-duplicate")); // Inner dir regex match
+ Assert.assertTrue(iter.next().contains(filenameA)); // Direct matched tempdata_a.jar
+
+ } catch (Exception e) {
+ Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
+ } finally {
+ FileUtils.deleteQuietly(new File(tempPath));
+ }
+ }
+
+
+ public void testScannedPathsRecursive() throws Exception {
+ ScannedPaths sc;
+ Iterator<String> iter = null;
+ String tempPath = "target/test/";
+
+ String fileRegex = tempPath + "parent_*";
+
+ try {
+ createTempDirStructure2(tempPath);
+
+ sc = new ScannedPaths(fileRegex, "file");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 6, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains("jar_1"));
+ Assert.assertTrue(iter.next().contains("jar_2"));
+ Assert.assertTrue(iter.next().contains("jar_4"));
+ Assert.assertTrue(iter.next().contains("jar_3"));
+ Assert.assertTrue(iter.next().contains("jar_6"));
+ Assert.assertTrue(iter.next().contains("jar_5"));
+
+ /** Now also enforce order for c_dir **/
+ File orderFile = createNewPath(tempPath, "parent_dir/c_dir/glob_order");
+ PrintWriter writer = new PrintWriter(orderFile, "UTF-8");
+ writer.println("*5");
+ writer.println("*6");
+ writer.close();
+
+ sc = new ScannedPaths(fileRegex, "file");
+ Assert.assertEquals(Iterators.size(sc.iterator()), 6, "Incorrect number of matches found");
+ iter = sc.iterator();
+ Assert.assertTrue(iter.next().contains("jar_1"));
+ Assert.assertTrue(iter.next().contains("jar_2"));
+ Assert.assertTrue(iter.next().contains("jar_4"));
+ Assert.assertTrue(iter.next().contains("jar_3"));
+ Assert.assertTrue(iter.next().contains("jar_5"));
+ Assert.assertTrue(iter.next().contains("jar_6"));
+
+ } catch (Exception e) {
+ Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
+ } finally {
+ FileUtils.deleteQuietly(new File(tempPath));
+ }
+ }
+
+
+ public void testScannedPathsNonExistent() throws Exception {
+ ScannedPaths sc;
+ Iterator<String> iter = null;
+ String tempPath = "target/test/";
+
+ String fileRegex = tempPath + "paradox/nopath";
+
+ try {
+ sc = new ScannedPaths(fileRegex, "file");
+ Assert.assertFalse(sc.iterator().hasNext(), "Iterator should be empty for unmatched path patterns.");
+
+ sc = new ScannedPaths(fileRegex, "jar");
+ Assert.assertFalse(sc.iterator().hasNext(), "Iterator should be empty for unmatched path patterns.");
+
+ } catch (Exception e) {
+ Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
+ } finally {
+ FileUtils.deleteQuietly(new File(tempPath));
+ }
+ }
+
+ private File createNewPath(String ... params) {
+ String fileName = Joiner.on("").join(params);
+
File f = new File(fileName);
try {
if (!f.getParentFile().exists()) {
@@ -140,5 +437,4 @@ public class TestScannedPaths {
}
return f;
}
-
}
[48/51] [abbrv] incubator-lens git commit: LENS-605 : Fix Candidate
pruning wrt sourcecolumns required for joinchains
Posted by jd...@apache.org.
LENS-605 : Fix Candidate pruning wrt sourcecolumns required for joinchains
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/5f859118
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/5f859118
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/5f859118
Branch: refs/heads/current-release-line
Commit: 5f859118c2ea49cc09dd59472d266064d36ca978
Parents: 0cf6c4e
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Mon Jun 22 14:22:13 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Jun 22 14:22:13 2015 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CandidateTableResolver.java | 124 +++++++++++++++----
.../lens/cube/parse/CubeQueryContext.java | 70 ++++++++++-
.../cube/parse/DenormalizationResolver.java | 6 +-
.../lens/cube/parse/ExpressionResolver.java | 27 ++--
.../apache/lens/cube/parse/JoinResolver.java | 4 +-
.../cube/parse/TestDenormalizationResolver.java | 32 ++++-
6 files changed, 218 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5f859118/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 79d6d43..1b5c09a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -23,6 +23,9 @@ import java.util.*;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.cube.parse.CubeQueryContext.OptionalDimCtx;
+import org.apache.lens.cube.parse.CubeQueryContext.QueriedExprColumn;
+import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext;
+import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -32,6 +35,8 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import com.google.common.collect.Sets;
+
/**
* This resolver prunes the candidate tables for following cases
* <p/>
@@ -493,39 +498,127 @@ class CandidateTableResolver implements ContextRewriter {
return;
}
// check for source columns for denorm columns
- Map<CandidateTable, Collection<String>> removedCandidates = new HashMap<CandidateTable, Collection<String>>();
+ Map<Dimension, Set<CandidateTable>> removedCandidates = new HashMap<Dimension, Set<CandidateTable>>();
for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
Dimension dim = optdimEntry.getKey();
OptionalDimCtx optdim = optdimEntry.getValue();
Iterator<CandidateTable> iter = optdim.requiredForCandidates.iterator();
+ // remove candidates from each optional dim if the dimension is not reachable from candidate
+ Set<CandidateTable> cTableSet = Sets.newHashSet();
+ removedCandidates.put(dim, cTableSet);
while (iter.hasNext()) {
CandidateTable candidate = iter.next();
boolean remove = false;
if (cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim) == null) {
- LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as no join paths"
+ LOG.info("Removing candidate " + candidate + " from requiredForCandidates of " + dim + ", as no join paths"
+ " exist");
remove = true;
- removedCandidates.put(candidate, optdim.colQueried);
} else {
List<String> colSet = cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim).get(candidate.getBaseTable());
if (!checkForColumnExists(candidate, colSet)) {
- LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as columns:"
+ LOG.info("Removing candidate " + candidate + " from requiredForCandidates of " + dim + ", as columns:"
+ colSet + " do not exist");
remove = true;
- removedCandidates.put(candidate, colSet);
}
}
if (remove) {
iter.remove();
+ cTableSet.add(candidate);
}
}
}
- Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<CandidateTable>();
+ // For each ref column required in cube query
+ // remove candidates which are not reachable
+ // For example:
+ // CTable | Col1 | Col2
+ // F1 | reachable through D1 | Not reachable
+ // F2 | Reachable through D2 | Reachable through D5
+ // F3 | Not reachable | Not reachable
+ // F4 | Not reachable | Reachable through D6
+ // F5 | Directly available | Directly available
+ // F6 | Directly available | Not reachable
+ // F3 and F4 will get pruned while iterating over col1 and F1, F6 will get pruned while iterating over col2.
+ for (Map.Entry<String, Set<Dimension>> dimColEntry : cubeql.getRefColToDim().entrySet()) {
+ Set<CandidateTable> candidatesReachableThroughRefs = new HashSet<CandidateTable>();
+ String col = dimColEntry.getKey();
+ Set<Dimension> dimSet = dimColEntry.getValue();
+ for (Dimension dim : dimSet) {
+ OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(dim);
+ candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates);
+ }
+ for (Dimension dim : dimSet) {
+ for (CandidateTable candidate : removedCandidates.get(dim)) {
+ if (!candidatesReachableThroughRefs.contains(candidate)) {
+ if (candidate instanceof CandidateFact) {
+ if (cubeql.getCandidateFacts().contains(candidate)) {
+ LOG.info("Not considering fact:" + candidate + " as its required optional dims are not reachable");
+ cubeql.getCandidateFacts().remove(candidate);
+ cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
+ CandidateTablePruneCause.columnNotFound(col));
+ }
+ } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
+ LOG.info("Not considering dimtable:" + candidate + " as its required optional dims are not reachable");
+ cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
+ cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
+ CandidateTablePruneCause.columnNotFound(col));
+ }
+ }
+ }
+ }
+ }
+
+ // For each expression column required in cube query
+ // remove candidates in which expressions are not evaluable - for expression column all dimensions accessed in
+ // expression should be reachable from candidate.
+ // For example:
+ // CTable | Col1 | Col2
+ // F1 | evaluable through D1, D3 | Not evaluable
+ // F2 | evaluable through D2, D4 | evaluable through D5
+ // F3 | Not evaluable | Not evaluable
+ // F4 | Not evaluable | evaluable through D6
+ // F5 | Directly available | Directly available
+ // F6 | Directly available | Not evaluable
+ for (Map.Entry<QueriedExprColumn, Set<Dimension>> exprColEntry : cubeql.getExprColToDim().entrySet()) {
+ QueriedExprColumn col = exprColEntry.getKey();
+ Set<Dimension> dimSet = exprColEntry.getValue();
+ ExpressionContext ec = cubeql.getExprCtx().getExpressionContext(col.getExprCol(), col.getAlias());
+ for (Dimension dim : dimSet) {
+ for (CandidateTable candidate : removedCandidates.get(dim)) {
+ // check if evaluable expressions of this candidate are no more evaluable because dimension is not reachable
+ // if no evaluable expressions exist, then remove the candidate
+ Iterator<ExprSpecContext> escIter = ec.getEvaluableExpressions().get(candidate).iterator();
+ while (escIter.hasNext()) {
+ ExprSpecContext esc = escIter.next();
+ if (esc.getExprDims().contains(dim)) {
+ escIter.remove();
+ }
+ }
+ if (cubeql.getExprCtx().isEvaluable(col.getExprCol(), candidate)) {
+ // candidate has other evaluable expressions
+ continue;
+ }
+ if (candidate instanceof CandidateFact) {
+ if (cubeql.getCandidateFacts().contains(candidate)) {
+ LOG.info("Not considering fact:" + candidate + " as is not reachable through any optional dim");
+ cubeql.getCandidateFacts().remove(candidate);
+ cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
+ CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol()));
+ }
+ } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
+ LOG.info("Not considering dimtable:" + candidate + " as is not reachable through any optional dim");
+ cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
+ cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
+ CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol()));
+ }
+ }
+ }
+ }
+
+ // remove optional dims which are not required any more.
Set<Dimension> tobeRemoved = new HashSet<Dimension>();
for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
Dimension dim = optdimEntry.getKey();
OptionalDimCtx optdim = optdimEntry.getValue();
- candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates);
if ((!optdim.colQueried.isEmpty() && optdim.requiredForCandidates.isEmpty()) && !optdim.isRequiredInJoinChain) {
LOG.info("Not considering optional dimension " + dim + " as all requiredForCandidates are removed");
tobeRemoved.add(dim);
@@ -534,23 +627,6 @@ class CandidateTableResolver implements ContextRewriter {
for (Dimension dim : tobeRemoved) {
removeOptionalDim(cubeql, dim);
}
- for (CandidateTable candidate : removedCandidates.keySet()) {
- if (!candidatesReachableThroughRefs.contains(candidate)) {
- if (candidate instanceof CandidateFact) {
- if (cubeql.getCandidateFacts().contains(candidate)) {
- LOG.info("Not considering fact:" + candidate + " as is not reachable through any optional dim");
- cubeql.getCandidateFacts().remove(candidate);
- cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
- CandidateTablePruneCause.columnNotFound(removedCandidates.get(candidate)));
- }
- } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
- LOG.info("Not considering dimtable:" + candidate + " as is not reachable through any optional dim");
- cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
- cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
- CandidateTablePruneCause.columnNotFound(removedCandidates.get(candidate)));
- }
- }
- }
}
private void resolveCandidateDimTables(CubeQueryContext cubeql) throws SemanticException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5f859118/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 33b1a62..57c075c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -45,7 +45,13 @@ import org.apache.hadoop.hive.ql.parse.*;
import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
+import lombok.AllArgsConstructor;
+
+import lombok.Data;
+
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@@ -346,11 +352,45 @@ public class CubeQueryContext implements TrackQueriedColumns {
return partsQueried;
}
+ // map of ref column in query to set of Dimension that have the column - which are added as optional dims
+ @Getter
+ private Map<String, Set<Dimension>> refColToDim = Maps.newHashMap();
+
+ public void updateRefColDim(String col, Dimension dim) {
+ Set<Dimension> refDims = refColToDim.get(col.toLowerCase());
+ if (refDims == null) {
+ refDims = Sets.newHashSet();
+ refColToDim.put(col.toLowerCase(), refDims);
+ }
+ refDims.add(dim);
+ }
+
+ @Data
+ @AllArgsConstructor
+ static class QueriedExprColumn {
+ private String exprCol;
+ private String alias;
+ }
+ // map of expression column in query to set of Dimension that are accessed in the expression column - which are added
+ // as optional dims
+ @Getter
+ private Map<QueriedExprColumn, Set<Dimension>> exprColToDim = Maps.newHashMap();
+
+ public void updateExprColDim(String tblAlias, String col, Dimension dim) {
+
+ QueriedExprColumn qexpr = new QueriedExprColumn(col, tblAlias);
+ Set<Dimension> exprDims = exprColToDim.get(qexpr);
+ if (exprDims == null) {
+ exprDims = Sets.newHashSet();
+ exprColToDim.put(qexpr, exprDims);
+ }
+ exprDims.add(dim);
+ }
+
// Holds the context of optional dimension
// A dimension is optional if it is not queried directly by the user, but is
// required by a candidate table to get a denormalized field from reference
// or required in a join chain
-
@ToString
static class OptionalDimCtx {
OptionalDimCtx() {
@@ -361,8 +401,22 @@ public class CubeQueryContext implements TrackQueriedColumns {
boolean isRequiredInJoinChain = false;
}
- public void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String... cols)
- throws SemanticException {
+ public void addOptionalJoinDimTable(String alias, boolean isRequired) throws SemanticException {
+ addOptionalDimTable(alias, null, isRequired, null, false, (String[])null);
+ }
+
+ public void addOptionalExprDimTable(String dimAlias, String queriedExpr, String srcTableAlias,
+ CandidateTable candidate, String... cols) throws SemanticException {
+ addOptionalDimTable(dimAlias, candidate, false, queriedExpr, false, srcTableAlias, cols);
+ }
+
+ public void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol,
+ boolean isRef, String... cols) throws SemanticException {
+ addOptionalDimTable(alias, candidate, isRequiredInJoin, cubeCol, true, null, cols);
+ }
+
+ private void addOptionalDimTable(String alias, CandidateTable candidate, boolean isRequiredInJoin, String cubeCol,
+ boolean isRef, String tableAlias, String... cols) throws SemanticException {
alias = alias.toLowerCase();
try {
if (!addQueriedTable(alias, true)) {
@@ -380,10 +434,18 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
optDim.requiredForCandidates.add(candidate);
}
+ if (cubeCol != null) {
+ if (isRef) {
+ updateRefColDim(cubeCol, dim);
+ } else {
+ updateExprColDim(tableAlias, cubeCol, dim);
+ }
+ }
if (!optDim.isRequiredInJoinChain) {
optDim.isRequiredInJoinChain = isRequiredInJoin;
}
- LOG.info("Adding optional dimension:" + dim + " optDim:" + optDim);
+ LOG.info("Adding optional dimension:" + dim + " optDim:" + optDim
+ + (cubeCol == null ? "" : " for column:" + cubeCol + " isRef:" + isRef));
} catch (HiveException e) {
throw new SemanticException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5f859118/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 3ed94fa..075cdb4 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -152,11 +152,13 @@ public class DenormalizationResolver implements ContextRewriter {
refCols.add(refer);
// Add to optional tables
if (refer.col.isChainedColumn()) {
- cubeql.addOptionalDimTable(refer.col.getChainName(), table, false, refer.col.getRefColumn());
+ cubeql.addOptionalDimTable(refer.col.getChainName(), table, false, refer.col.getName(), true,
+ refer.col.getRefColumn());
} else {
for (TableReference reference : refer.col.getReferences()) {
- cubeql.addOptionalDimTable(reference.getDestTable(), table, false, reference.getDestColumn());
+ cubeql.addOptionalDimTable(reference.getDestTable(), table, false, refer.col.getName(), true,
+ reference.getDestColumn());
}
}
return true;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5f859118/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index 8e199ea..b2997dd 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -68,6 +68,7 @@ class ExpressionResolver implements ContextRewriter {
@Getter
private Set<ExprSpecContext> allExprs = new LinkedHashSet<ExprSpecContext>();
private Set<CandidateTable> directlyAvailableIn = new HashSet<CandidateTable>();
+ @Getter
private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions =
new HashMap<CandidateTable, Set<ExprSpecContext>>();
private boolean hasMeasures = false;
@@ -95,7 +96,7 @@ class ExpressionResolver implements ContextRewriter {
try {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
cubeql.addOptionalDimTable(table, null,
- false, esc.getTblAliasToColumns().get(table).toArray(new String[0]));
+ false, null, false, esc.getTblAliasToColumns().get(table).toArray(new String[0]));
esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
} catch (HiveException e) {
@@ -132,7 +133,7 @@ class ExpressionResolver implements ContextRewriter {
for (String table : esc.getTblAliasToColumns().keySet()) {
try {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
- cubeql.addOptionalDimTable(table, null, false,
+ cubeql.addOptionalDimTable(table, null, false, null, false,
esc.getTblAliasToColumns().get(table).toArray(new String[0]));
esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
@@ -171,17 +172,17 @@ class ExpressionResolver implements ContextRewriter {
if (evalSet == null) {
evalSet = new LinkedHashSet<ExprSpecContext>();
evaluableExpressions.put(cTable, evalSet);
- // add optional dimensions involved in expressions
- for (String table : esc.getTblAliasToColumns().keySet()) {
- try {
- if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
- cubeql.addOptionalDimTable(table, cTable,
- false, esc.getTblAliasToColumns().get(table).toArray(new String[0]));
- esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
- }
- } catch (HiveException e) {
- throw new SemanticException(e);
+ }
+ // add optional dimensions involved in expressions
+ for (String table : esc.getTblAliasToColumns().keySet()) {
+ try {
+ if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
+ cubeql.addOptionalExprDimTable(table, exprCol.getName(), srcAlias, cTable,
+ esc.getTblAliasToColumns().get(table).toArray(new String[0]));
+ esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
+ } catch (HiveException e) {
+ throw new SemanticException(e);
}
}
evalSet.add(esc);
@@ -210,6 +211,7 @@ class ExpressionResolver implements ContextRewriter {
private Set<ExprSpec> exprSpecs = new LinkedHashSet<ExprSpec>();
@Getter
private ASTNode finalAST;
+ @Getter
private Set<Dimension> exprDims = new HashSet<Dimension>();
// for each expression store alias to columns queried
@Getter
@@ -367,6 +369,7 @@ class ExpressionResolver implements ContextRewriter {
}
for (ExprSpecContext esc : ec.allExprs) {
if (esc.getTblAliasToColumns().get(alias) == null) {
+ log.debug("{} = {} is evaluable in {}", expr, esc, cTable);
ec.addEvaluable(cubeql, cTable, esc);
} else {
Set<String> columns = esc.getTblAliasToColumns().get(alias);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5f859118/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
index a760599..97c7a46 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
@@ -1016,7 +1016,7 @@ class JoinResolver implements ContextRewriter {
for (JoinChain chain : cubeql.getJoinchains().values()) {
for (String dimName : chain.getIntermediateDimensions()) {
- cubeql.addOptionalDimTable(dimName, null, true, null);
+ cubeql.addOptionalJoinDimTable(dimName, true);
}
}
@@ -1122,7 +1122,7 @@ class JoinResolver implements ContextRewriter {
for (TableRelationship rel : joinPath.getEdges()) {
// Add the joined tables to the queries table sets so that they are
// resolved in candidate resolver
- cubeql.addOptionalDimTable(rel.getToTable().getName(), null, required, null);
+ cubeql.addOptionalJoinDimTable(rel.getToTable().getName(), required);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5f859118/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index e615ccc..8c3c219 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -23,7 +23,9 @@ import static org.apache.lens.cube.parse.CubeTestSetup.*;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.server.api.error.LensException;
@@ -164,7 +166,7 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
}))
);
put("summary1,cheapfact,testfactmonthly,testfact2,testfact",
- Arrays.asList(CandidateTablePruneCause.columnNotFound("dim2big1", "dim2")));
+ Arrays.asList(CandidateTablePruneCause.columnNotFound("dim2big2")));
}
}
));
@@ -225,6 +227,34 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
}
@Test
+ public void testCubeQueryWithTwoRefCols() throws Exception {
+ Configuration tConf = new Configuration(conf);
+ tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "");
+ CubeQueryContext cubeql = rewriteCtx("select dim2, test_time_dim2 from testcube where " + TWO_DAYS_RANGE, tConf);
+ Set<String> candidateFacts = new HashSet<String>();
+ for (CandidateFact cfact : cubeql.getCandidateFacts()) {
+ candidateFacts.add(cfact.getName().toLowerCase());
+ }
+ // testfact contains test_time_dim_hour_id, but not dim2 - it should have been removed.
+ Assert.assertFalse(candidateFacts.contains("testfact"));
+ // summary2 contains dim2, but not test_time_dim2 - it should have been removed.
+ Assert.assertFalse(candidateFacts.contains("summary2"));
+ }
+
+ @Test
+ public void testDimensionQueryWithTwoRefCols() throws Exception {
+ Configuration tConf = new Configuration(conf);
+ tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "");
+ CubeQueryContext cubeql = rewriteCtx("select citydim.zipcode, citydim.statename from" + " citydim", tConf);
+ Set<String> candidateDims = new HashSet<String>();
+ for (CandidateDim cdim : cubeql.getCandidateDims().get(cubeql.getMetastoreClient().getDimension("citydim"))) {
+ candidateDims.add(cdim.getName());
+ }
+ // city_table2 contains stateid, but not zipcode - it should have been removed.
+ Assert.assertFalse(candidateDims.contains("city_table2"));
+ }
+
+ @Test
public void testDimensionQueryWithExpressionHavingDenormColumn() throws Exception {
String hqlQuery = rewrite("select citydim.name, citydim.citystate from" + " citydim", conf);
String joinExpr =
[30/51] [abbrv] incubator-lens git commit: LENS-606: Ensuring order
of iteration in Column Time Range Availability check in different estimate
threads
Posted by jd...@apache.org.
LENS-606: Ensuring order of iteration in Column Time Range Availability check in different estimate threads
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c823ec14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c823ec14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c823ec14
Branch: refs/heads/current-release-line
Commit: c823ec14c5b3b52f1208fbe06e3f5dfcd2413f13
Parents: d089ce2
Author: Himanshu Gahlaut <hi...@inmobi.com>
Authored: Wed Jun 17 12:48:33 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed Jun 17 12:48:33 2015 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c823ec14/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index c5d8d91..b7900be 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -905,7 +905,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
public void addColumnsQueried(String alias, String column) {
Set<String> cols = tblAliasToColumns.get(alias.toLowerCase());
if (cols == null) {
- cols = new HashSet<String>();
+ cols = new LinkedHashSet<String>();
tblAliasToColumns.put(alias.toLowerCase(), cols);
}
cols.add(column);
[04/51] [abbrv] incubator-lens git commit: LENS-580: Add mailing list
subscribe/unsubscribe information on website for beginners
Posted by jd...@apache.org.
LENS-580: Add mailing list subscribe/unsubscribe information on website for beginners
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/2e90c29c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/2e90c29c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/2e90c29c
Branch: refs/heads/current-release-line
Commit: 2e90c29c4058ec11e905327ed16bb01de0260a9d
Parents: b0783db
Author: Yash Sharma <ya...@gmail.com>
Authored: Wed May 27 20:03:44 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed May 27 20:03:44 2015 +0530
----------------------------------------------------------------------
src/site/apt/lenshome/quick-start.apt | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e90c29c/src/site/apt/lenshome/quick-start.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/lenshome/quick-start.apt b/src/site/apt/lenshome/quick-start.apt
index 86bd60a..6f62b7b 100644
--- a/src/site/apt/lenshome/quick-start.apt
+++ b/src/site/apt/lenshome/quick-start.apt
@@ -19,6 +19,23 @@
Getting started with Lens
+*
+
+** Subscribe to mailing lists
+
+*----------+----------------------------------------------+-------------------------------------------------+
+| | Subscribe | Unsubscribe |
+*----------+----------------------------------------------+-------------------------------------------------+
+| User | user-subscribe@lens.incubator.apache.org | user-unsubscribe@lens.incubator.apache.org |
+*----------+----------------------------------------------+-------------------------------------------------+
+| Dev | dev-subscribe@lens.incubator.apache.org | dev-unsubscribe@lens.incubator.apache.org |
+*----------+----------------------------------------------+-------------------------------------------------+
+| Commits | commits-subscribe@lens.incubator.apache.org | commits-unsubscribe@lens.incubator.apache.org |
+*----------+----------------------------------------------+-------------------------------------------------+
+
+
+** Try out Lens
+
Start with installation steps described {{{./install-and-run.html}here}} and try out
{{{./pseudo-distributed-setup.html}pseudo distributed setup}}. Refer to
{{{../admin/index.html} Admin documentation}} for more information on server components
[31/51] [abbrv] incubator-lens git commit: LENS-611: Query does not
fail for conflicting time dimension and measure (Himanshu)
Posted by jd...@apache.org.
LENS-611: Query does not fail for conflicting time dimension and measure (Himanshu)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ede537e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ede537e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ede537e6
Branch: refs/heads/current-release-line
Commit: ede537e6bf22176dd689d5d727605e8c6442dae5
Parents: c823ec1
Author: Himanshu Gahlaut <ga...@gmail.com>
Authored: Wed Jun 17 14:45:47 2015 +0530
Committer: Himanshu Gahlaut <hi...@apache.org>
Committed: Wed Jun 17 14:45:47 2015 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/ColumnResolver.java | 47 +++++++---
.../lens/cube/parse/CubeQueryContext.java | 23 +++++
.../apache/lens/cube/parse/FieldValidator.java | 17 ++--
.../apache/lens/cube/parse/CubeTestSetup.java | 10 ++-
.../FieldsCannotBeQueriedTogetherTest.java | 90 +++++++++++++++-----
5 files changed, 149 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ede537e6/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
index 0849381..2ff5959 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ColumnResolver.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import com.google.common.base.Optional;
+
class ColumnResolver implements ContextRewriter {
public ColumnResolver(Configuration conf) {
@@ -111,7 +113,6 @@ class ColumnResolver implements ContextRewriter {
String column = colIdent.getText().toLowerCase();
String table = tabident.getText().toLowerCase();
-
tqc.addColumnsQueried(table, column);
}
}
@@ -188,7 +189,9 @@ class ColumnResolver implements ContextRewriter {
// column is an existing alias
return;
}
- cubeql.addColumnsQueried(CubeQueryContext.DEFAULT_TABLE, column);
+
+ addColumnQueriedWithTimeRangeFuncCheck(cubeql, parent, CubeQueryContext.DEFAULT_TABLE, column);
+
} else if (node.getToken().getType() == DOT) {
// This is for the case where column name is prefixed by table name
// or table alias
@@ -200,15 +203,12 @@ class ColumnResolver implements ContextRewriter {
String column = colIdent.getText().toLowerCase();
String table = tabident.getText().toLowerCase();
- cubeql.addColumnsQueried(table, column);
+ addColumnQueriedWithTimeRangeFuncCheck(cubeql, parent, table, column);
+
} else if (node.getToken().getType() == TOK_FUNCTION) {
ASTNode fname = HQLParser.findNodeByPath(node, Identifier);
if (fname != null && CubeQueryContext.TIME_RANGE_FUNC.equalsIgnoreCase(fname.getText())) {
- if (!cubeql.shouldReplaceTimeDimWithPart()) {
- // Skip columns in timerange clause if replace timedimension with part
- // column is on
- addColumnsForWhere(cubeql, (ASTNode) node.getChild(1), node);
- }
+ addColumnsForWhere(cubeql, (ASTNode) node.getChild(1), node);
} else {
for (int i = 0; i < node.getChildCount(); i++) {
addColumnsForWhere(cubeql, (ASTNode) node.getChild(i), node);
@@ -221,6 +221,35 @@ class ColumnResolver implements ContextRewriter {
}
}
+ private static void addColumnQueriedWithTimeRangeFuncCheck(final CubeQueryContext cubeql, final ASTNode parent,
+ final String table, final String column) {
+
+ if (isTimeRangeFunc(parent)) {
+ cubeql.addQueriedTimeDimensionCols(column);
+ cubeql.addColumnsQueriedWithTimeDimCheck(CubeQueryContext.DEFAULT_TABLE, column);
+ } else {
+ cubeql.addColumnsQueried(table, column);
+ }
+ }
+
+ private static boolean isTimeRangeFunc(final ASTNode node) {
+
+ Optional<String> funcNameOp = getNameIfFunc(node);
+ final String funcName = funcNameOp.isPresent() ? funcNameOp.get() : null;
+ return CubeQueryContext.TIME_RANGE_FUNC.equalsIgnoreCase(funcName);
+ }
+
+ private static Optional<String> getNameIfFunc(final ASTNode node) {
+
+ String funcName = null;
+ if (node.getToken().getType() == TOK_FUNCTION) {
+ ASTNode foundNode = HQLParser.findNodeByPath(node, Identifier);
+ if (foundNode != null) {
+ funcName = foundNode.getText();
+ }
+ }
+ return Optional.fromNullable(funcName);
+ }
private static void addColumnsForSelectExpr(final CubeQueryContext cubeql, ASTNode node, ASTNode parent,
Set<String> cols) {
if (node.getToken().getType() == TOK_TABLE_OR_COL && (parent != null && parent.getToken().getType() != DOT)) {
@@ -243,7 +272,6 @@ class ColumnResolver implements ContextRewriter {
String column = colIdent.getText().toLowerCase();
String table = tabident.getText().toLowerCase();
-
cubeql.addColumnsQueried(table, column);
cols.add(column);
} else {
@@ -252,5 +280,4 @@ class ColumnResolver implements ContextRewriter {
}
}
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ede537e6/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index b7900be..ae65287 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -23,6 +23,8 @@ import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE;
+import static com.google.common.base.Preconditions.checkArgument;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.*;
@@ -42,6 +44,7 @@ import org.apache.hadoop.hive.ql.parse.*;
import org.codehaus.jackson.map.ObjectMapper;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import lombok.Getter;
import lombok.Setter;
@@ -83,6 +86,8 @@ public class CubeQueryContext implements TrackQueriedColumns {
@Getter
private final Set<String> queriedExprs = new HashSet<String>();
+ private final Set<String> queriedTimeDimCols = new LinkedHashSet<String>();
+
@Getter
private final Set<String> queriedExprsWithMeasures = new HashSet<String>();
@@ -902,7 +907,15 @@ public class CubeQueryContext implements TrackQueriedColumns {
addColumnsQueried(getAliasForTableName(table.getName()), column);
}
+ public void addColumnsQueriedWithTimeDimCheck(String alias, String timeDimColumn) {
+
+ if (!shouldReplaceTimeDimWithPart()) {
+ addColumnsQueried(alias, timeDimColumn);
+ }
+ }
+
public void addColumnsQueried(String alias, String column) {
+
Set<String> cols = tblAliasToColumns.get(alias.toLowerCase());
if (cols == null) {
cols = new LinkedHashSet<String>();
@@ -1154,4 +1167,14 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
}
}
+
+ public void addQueriedTimeDimensionCols(final String timeDimColName) {
+
+ checkArgument(StringUtils.isNotBlank(timeDimColName));
+ this.queriedTimeDimCols.add(timeDimColName);
+ }
+
+ public ImmutableSet<String> getQueriedTimeDimCols() {
+ return ImmutableSet.copyOf(this.queriedTimeDimCols);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ede537e6/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
index 60a767d..03377dd 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/FieldValidator.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import com.google.common.collect.ImmutableSet;
+
/**
* Validate fields based on cube queryability
*/
@@ -57,23 +59,28 @@ public class FieldValidator implements ContextRewriter {
throw new SemanticException(e);
}
- // dim attributes and chained source columns should only come from WHERE and GROUP BY ASTs
- Set<String> queriedDimAttrs = new LinkedHashSet<String>();
+ ImmutableSet<String> queriedTimeDimCols = cubeql.getQueriedTimeDimCols();
+
+ Set<String> queriedDimAttrs = new LinkedHashSet<String>(queriedTimeDimCols);
+ Set<String> nonQueryableFields = new LinkedHashSet<String>(queriedTimeDimCols);
+
Set<String> queriedMsrs = new LinkedHashSet<String>(cubeql.getQueriedMsrs());
queriedMsrs.addAll(getMeasuresFromExprMeasures(cubeql));
Set<String> chainedSrcColumns = new HashSet<String>();
- Set<String> nonQueryableFields = new LinkedHashSet<String>();
- findDimAttrsAndChainSourceColumns(cubeql, cubeql.getGroupByAST(), queriedDimAttrs,
- chainedSrcColumns, nonQueryableFields);
+ // dim attributes and chained source columns should only come from WHERE and GROUP BY ASTs
+ findDimAttrsAndChainSourceColumns(cubeql, cubeql.getGroupByAST(), queriedDimAttrs, chainedSrcColumns,
+ nonQueryableFields);
findDimAttrsAndChainSourceColumns(cubeql, cubeql.getWhereAST(), queriedDimAttrs,
chainedSrcColumns, nonQueryableFields);
// do validation
// Find atleast one derived cube which contains all the dimensions
// queried.
+
boolean derivedCubeFound = false;
for (DerivedCube dcube : dcubes) {
+
if (dcube.getDimAttributeNames().containsAll(chainedSrcColumns)
&& dcube.getDimAttributeNames().containsAll(queriedDimAttrs)) {
// remove all the measures that are covered
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ede537e6/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 49fabff..432c5f4 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -833,9 +833,9 @@ public class CubeTestSetup {
};
// add ref dim through chain
- cubeDimensions2.add(new ReferencedDimAtrribute(
- new FieldSchema("cityStateCapital", "string", "State's capital thru city"), "State's capital thru city",
- "cityState", "capital", null, null, null));
+ cubeDimensions2.add(
+ new ReferencedDimAtrribute(new FieldSchema("cityStateCapital", "string", "State's capital thru city"),
+ "State's capital thru city", "cityState", "capital", null, null, null));
client.createCube(BASE_CUBE_NAME, cubeMeasures2, cubeDimensions2, exprs, joinchains, cubeProperties);
Map<String, String> derivedProperties = new HashMap<String, String>();
@@ -846,6 +846,7 @@ public class CubeTestSetup {
Set<String> dimensions = new HashSet<String>();
dimensions.add("dim1");
dimensions.add("dim11");
+ dimensions.add("d_time");
client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME1, measures, dimensions, derivedProperties, 5L);
measures = new HashSet<String>();
@@ -860,6 +861,8 @@ public class CubeTestSetup {
dimensions.add("dim2");
dimensions.add("dim11");
dimensions.add("dim12");
+ dimensions.add("d_time");
+ dimensions.add("test_time_dim");
client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME2, measures, dimensions, derivedProperties, 10L);
measures = new HashSet<String>();
measures.add("msr3");
@@ -867,6 +870,7 @@ public class CubeTestSetup {
dimensions = new HashSet<String>();
dimensions.add("dim1");
dimensions.add("location");
+ dimensions.add("d_time");
client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME3, measures, dimensions, derivedProperties, 20L);
// create base cube facts
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ede537e6/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
index 0a37c3d..398c45e 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.lens.cube.parse;
-import static org.apache.lens.cube.parse.CubeTestSetup.TWO_DAYS_RANGE;
+import static org.apache.lens.cube.parse.CubeTestSetup.*;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.fail;
@@ -59,7 +59,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
exception. */
testFieldsCannotBeQueriedTogetherError("select dim2, SUM(msr1) from basecube where " + TWO_DAYS_RANGE,
- Arrays.asList("dim2", "msr1"));
+ Arrays.asList("dim2", "d_time", "msr1"));
}
@Test
@@ -72,7 +72,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select dim2, sum(roundedmsr1) from basecube where " + TWO_DAYS_RANGE,
- Arrays.asList("dim2", "msr1"));
+ Arrays.asList("dim2", "d_time", "msr1"));
}
@Test
@@ -85,7 +85,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select substrexprdim2, SUM(msr1) from basecube where " + TWO_DAYS_RANGE,
- Arrays.asList("dim2", "msr1"));
+ Arrays.asList("dim2", "d_time", "msr1"));
}
@Test
@@ -98,7 +98,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
derived cube, hence query shall be disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select substrexprdim2, sum(roundedmsr1) from basecube where "
- + TWO_DAYS_RANGE, Arrays.asList("dim2", "msr1"));
+ + TWO_DAYS_RANGE, Arrays.asList("dim2", "d_time", "msr1"));
}
@Test
@@ -114,7 +114,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
query shall be disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select citystate.name, SUM(msr1) from basecube where " + TWO_DAYS_RANGE,
- Arrays.asList("citystate.name", "msr1"));
+ Arrays.asList("citystate.name", "d_time", "msr1"));
}
@Test
@@ -130,7 +130,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
in the same derived cube, hence query shall be disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select citystate.name, sum(roundedmsr1) from basecube where "
- + TWO_DAYS_RANGE, Arrays.asList("citystate.name", "msr1"));
+ + TWO_DAYS_RANGE, Arrays.asList("citystate.name", "d_time", "msr1"));
}
@Test
@@ -146,7 +146,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
in the same derived cube, hence query shall be disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select cubestateName, sum(roundedmsr1) from basecube where "
- + TWO_DAYS_RANGE, Arrays.asList("cubestate.name", "msr1"));
+ + TWO_DAYS_RANGE, Arrays.asList("cubestate.name", "d_time", "msr1"));
}
@Test
@@ -162,7 +162,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
shall be disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select SUM(msr1) from basecube where cityState.name = 'foo' and "
- + TWO_DAYS_RANGE, Arrays.asList("citystate.name", "msr1"));
+ + TWO_DAYS_RANGE, Arrays.asList("citystate.name", "d_time", "msr1"));
}
@Test
@@ -178,7 +178,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
same derived cube, hence query shall be disallowed with appropriate exception. */
testFieldsCannotBeQueriedTogetherError("select sum(roundedmsr1) from basecube where cityState.name = 'foo' and "
- + TWO_DAYS_RANGE, Arrays.asList("citystate.name", "msr1"));
+ + TWO_DAYS_RANGE, Arrays.asList("citystate.name", "d_time", "msr1"));
}
@Test
@@ -193,8 +193,9 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
queryable through chain source column cityid. cityid and roundedmsr1( expression over msr1) are not present in the
same derived cube, hence query shall be disallowed with appropriate exception. */
- testFieldsCannotBeQueriedTogetherError("select sum(roundedmsr1) from basecube where cubestatename = 'foo' and "
- + TWO_DAYS_RANGE, Arrays.asList("cubestate.name", "msr1"));
+ testFieldsCannotBeQueriedTogetherError(
+ "select sum(roundedmsr1) from basecube where cubestatename = 'foo' and " + TWO_DAYS_RANGE,
+ Arrays.asList("cubestate.name", "d_time", "msr1"));
}
@Test
@@ -237,7 +238,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
/* dim2 and countryid are not present in the same derived cube, hence query should be disallowed */
testFieldsCannotBeQueriedTogetherError("select dim2, countryid, SUM(msr2) from basecube where " + TWO_DAYS_RANGE,
- Arrays.asList("countryid", "dim2"));
+ Arrays.asList("countryid", "d_time", "dim2"));
}
@Test
@@ -248,7 +249,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
* disallowed */
testFieldsCannotBeQueriedTogetherError("select substrexprdim2, cubeStateName, countryid, SUM(msr2) from basecube"
- + " where " + TWO_DAYS_RANGE, Arrays.asList("countryid", "dim2", "cubestate.name"));
+ + " where " + TWO_DAYS_RANGE, Arrays.asList("countryid", "dim2", "cubestate.name", "d_time"));
}
@Test
@@ -256,8 +257,8 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
/* newmeasure is not present in any derived cube, hence the query should be disallowed. */
- testFieldsCannotBeQueriedTogetherError("select newmeasure from basecube where "
- + TWO_DAYS_RANGE, Arrays.asList("newmeasure"));
+ testFieldsCannotBeQueriedTogetherError("select newmeasure from basecube where " + TWO_DAYS_RANGE,
+ Arrays.asList( "d_time", "newmeasure"));
}
@Test
@@ -266,7 +267,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
/* newexpr : expression over newmeasure is not present in any derived cube, hence the query should be disallowed. */
testFieldsCannotBeQueriedTogetherError("select newexpr from basecube where "
- + TWO_DAYS_RANGE, Arrays.asList("newmeasure"));
+ + TWO_DAYS_RANGE, Arrays.asList("d_time", "newmeasure"));
}
@Test
@@ -283,16 +284,65 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
testFieldsCannotBeQueriedTogetherError(
"select citystatecapital, SUM(msr1) from basecube where " + TWO_DAYS_RANGE,
- Arrays.asList("citystatecapital", "msr1"));
+ Arrays.asList("citystatecapital", "d_time", "msr1"));
+ }
+
+ @Test
+ public void testQueryWtihTimeDimAndReplaceTimeDimSwitchTrue() throws ParseException, SemanticException,
+ LensException {
+
+ /* If a time dimension and measure are not present in the same derived cube, then query shall be disallowed.
+
+ The testQuery in this test uses d_time in time range func. d_time is a time dimension in basecube.
+ d_time is present as a dimension in derived cube where as msr4 is not present in the same derived cube, hence
+ the query shall be disallowed.
+
+ The testQuery in this test uses its own queryConf which has CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL
+ set to true. */
+
+ Configuration queryConf = new Configuration(conf);
+ queryConf.setBoolean(CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, true);
+
+ testFieldsCannotBeQueriedTogetherError("select msr4 from basecube where " + "time_range_in(d_time, '"
+ + getDateUptoHours(TWODAYS_BACK) + "','" + getDateUptoHours(CubeTestSetup.NOW) + "')",
+ Arrays.asList("d_time", "msr4"), queryConf);
+ }
+
+ @Test
+ public void testQueryWtihTimeDimAndReplaceTimeDimSwitchFalse() throws ParseException, SemanticException,
+ LensException {
+
+ /* If a time dimension and measure are not present in the same derived cube, then query shall be disallowed.
+
+ The testQuery in this test uses d_time in time range func. d_time is a time dimension in basecube.
+ d_time is present as a dimension in derived cube where as msr4 is not present in the same derived cube, hence
+ the query shall be disallowed.
+
+ The testQuery in this test uses its own queryConf which has CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL
+ set to false */
+
+ Configuration queryConf = new Configuration(conf);
+ queryConf.setBoolean(CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, false);
+
+ testFieldsCannotBeQueriedTogetherError("select msr4 from basecube where " + "time_range_in(d_time, '" +
+ getDateUptoHours(TWODAYS_BACK) + "','" + getDateUptoHours(CubeTestSetup.NOW) + "')",
+ Arrays.asList("d_time", "msr4"), queryConf);
}
private void testFieldsCannotBeQueriedTogetherError(final String testQuery, final List<String> conflictingFields)
throws ParseException, SemanticException, LensException {
+ testFieldsCannotBeQueriedTogetherError(testQuery, conflictingFields, conf);
+ }
+
+ private void testFieldsCannotBeQueriedTogetherError(final String testQuery, final List<String> conflictingFields,
+ final Configuration queryConf)
+ throws ParseException, SemanticException, LensException {
try {
- String hqlQuery = rewrite(testQuery, conf);
- fail("Expected FieldsCannotBeQueriedTogetherException but it didn't come, rewrittenQuery:" + hqlQuery);
+ String hqlQuery = rewrite(testQuery, queryConf);
+ fail("Expected Query Rewrite to fail with FieldsCannotBeQueriedTogetherException, however it didn't happen. "
+ + "Query got re-written to:" + hqlQuery);
} catch(FieldsCannotBeQueriedTogetherException actualException) {
SortedSet<String> expectedFields = new TreeSet<String>(conflictingFields);
[20/51] [abbrv] incubator-lens git commit: LENS-557: Add failure
reason for FAILED queries in QueryStatusLogger logger.
Posted by jd...@apache.org.
LENS-557: Add failure reason for FAILED queries in QueryStatusLogger logger.
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/1b1d3353
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/1b1d3353
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/1b1d3353
Branch: refs/heads/current-release-line
Commit: 1b1d3353414c9dc0544bb6714aa7686ea7672e07
Parents: 099d19e
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Jun 9 12:17:14 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:17:14 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/api/query/QueryEnded.java | 12 ++++++++++++
.../lens/server/query/QueryExecutionServiceImpl.java | 9 ++++++++-
2 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1b1d3353/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java
index 4bf43d7..3e9474c 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java
@@ -23,6 +23,8 @@ import java.util.EnumSet;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
+import org.apache.commons.lang.StringUtils;
+
import lombok.Getter;
/**
@@ -68,4 +70,14 @@ public class QueryEnded extends StatusChange {
throw new IllegalStateException("Not a valid end state: " + current + " query: " + handle);
}
}
+
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (StringUtils.isNotBlank(cause)) {
+ buf.append(" cause:").append(cause);
+ }
+ return buf.toString();
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1b1d3353/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index dbc52e9..7623743 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -715,7 +715,14 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
case CLOSED:
return new QueryClosed(ctx.getClosedTime(), prevState, currState, query, ctx.getSubmittedUser(), null);
case FAILED:
- return new QueryFailed(ctx.getEndTime(), prevState, currState, query, ctx.getSubmittedUser(), null);
+ StringBuilder msgBuilder = new StringBuilder();
+ msgBuilder.append(ctx.getStatus().getStatusMessage());
+ if (!StringUtils.isBlank(ctx.getStatus().getErrorMessage())) {
+ msgBuilder.append("\n Reason:\n");
+ msgBuilder.append(ctx.getStatus().getErrorMessage());
+ }
+ return new QueryFailed(ctx.getEndTime(), prevState, currState, query, ctx.getSubmittedUser(),
+ msgBuilder.toString());
case LAUNCHED:
return new QueryLaunched(ctx.getLaunchTime(), prevState, currState, query);
case QUEUED:
[09/51] [abbrv] incubator-lens git commit: LENS-484 : NPE when weight
is not set in XFactTable object while creating fact table (Yash Sharma via
Raghavendra Singh)
Posted by jd...@apache.org.
LENS-484 : NPE when weight is not set in XFactTable object while creating fact table (Yash Sharma via Raghavendra Singh)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/1584f001
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/1584f001
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/1584f001
Branch: refs/heads/current-release-line
Commit: 1584f001fa3a3e5bcbf8483bd1350587e3fff653
Parents: 6a4b247
Author: Yash Sharma <ya...@gmail.com>
Authored: Tue Jun 2 15:32:22 2015 +0530
Committer: Raghavendra Singh <ra...@C1MLK26ADTY3.local>
Committed: Tue Jun 2 15:32:22 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 14 +-
.../apache/lens/cli/TestLensFactCommands.java | 2 +
.../TestLensFactCommandsWithMissingWeight.java | 150 +++++++++++++++++++
.../resources/cube_with_no_weight_facts.xml | 47 ++++++
.../resources/dim-local-storage-element.xml | 7 +-
lens-cli/src/test/resources/dim_table.xml | 8 +-
lens-cli/src/test/resources/dim_table2.xml | 6 +-
.../resources/fact-local-storage-element.xml | 8 +-
.../src/test/resources/fact_without_weight.xml | 50 +++++++
lens-cli/src/test/resources/sample-cube.xml | 20 +--
lens-cli/src/test/resources/test-dimension.xml | 4 +-
.../apache/lens/client/LensMetadataClient.java | 17 ++-
.../src/main/resources/city_subset.xml | 2 +-
.../src/main/resources/customer_table.xml | 2 +-
lens-examples/src/main/resources/dim_table.xml | 2 +-
lens-examples/src/main/resources/dim_table2.xml | 2 +-
lens-examples/src/main/resources/dim_table3.xml | 2 +-
lens-examples/src/main/resources/dim_table4.xml | 2 +-
.../src/main/resources/product_db_table.xml | 2 +-
.../src/main/resources/product_table.xml | 2 +-
.../src/main/resources/sales-aggr-fact1.xml | 6 +-
.../src/main/resources/sales-aggr-fact2.xml | 6 +-
lens-examples/src/main/resources/sales-cube.xml | 40 ++---
.../src/main/resources/sample-cube.xml | 18 +--
24 files changed, 346 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 1e8d71d..719d10d 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -626,13 +626,18 @@
</xs:documentation>
</xs:annotation>
</xs:attribute>
- <xs:attribute name="weight" type="xs:double">
+ <xs:attribute name="weight" use="required" >
<xs:annotation>
<xs:documentation>
The weight of the dimension table. LENS will use this attribute to determine the lightest
table when there are more than one eligible table for answering a query.
</xs:documentation>
</xs:annotation>
+ <xs:simpleType>
+ <xs:restriction base="xs:double">
+ <xs:minExclusive value="0"></xs:minExclusive>
+ </xs:restriction>
+ </xs:simpleType>
</xs:attribute>
</xs:complexType>
@@ -1157,13 +1162,18 @@
</xs:documentation>
</xs:annotation>
</xs:attribute>
- <xs:attribute name="weight" type="xs:double">
+ <xs:attribute name="weight" use="required" >
<xs:annotation>
<xs:documentation>
The weight of the fact table. LENS will use this attribute to decide the lightest table to
query when there are more than one eligible tables.
</xs:documentation>
</xs:annotation>
+ <xs:simpleType>
+ <xs:restriction base="xs:double">
+ <xs:minExclusive value="0"></xs:minExclusive>
+ </xs:restriction>
+ </xs:simpleType>
</xs:attribute>
</xs:complexType>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
index 1ad234d..13bfbd2 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
@@ -69,6 +69,8 @@ public class TestLensFactCommands extends LensCliApplicationTest {
String cubeList = getCubeCommand().showCubes();
assertFalse(cubeList.contains("sample_cube"), cubeList);
getCubeCommand().createCube(new File(cubeSpec.toURI()).getAbsolutePath());
+ cubeList = getCubeCommand().showCubes();
+ assertTrue(cubeList.contains("sample_cube"), cubeList);
}
private void dropSampleCube() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java
new file mode 100644
index 0000000..73f3a78
--- /dev/null
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommandsWithMissingWeight.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli;
+
+import java.io.*;
+
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.apache.lens.cli.commands.LensCubeCommands;
+import org.apache.lens.cli.commands.LensFactCommands;
+import org.apache.lens.client.LensClient;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+/**
+ * The Class TestLensFactCommands.
+ */
+public class TestLensFactCommandsWithMissingWeight extends LensCliApplicationTest {
+
+ /** The Constant LOG. */
+ private static final Logger LOG = LoggerFactory.getLogger(TestLensFactCommandsWithMissingWeight.class);
+
+ /** The Constant FACT_LOCAL. */
+ public static final String FACT_LOCAL = "fact_local_without_wt";
+
+ /* The Constant for cube name */
+ public static final String CUBE_NAME = "cube_with_no_weight_facts";
+
+ /* The Constant for fact name */
+ public static final String FACT_NAME = "fact_without_wt";
+
+ /* The File name with cube details */
+ public static final String CUBE_XML_FILE = "cube_with_no_weight_facts.xml";
+
+ /* The File name with fact details */
+ public static final String FACT_XML_FILE = "fact_without_weight.xml";
+
+ /** The command. */
+ private static LensFactCommands command = null;
+ private static LensCubeCommands cubeCommands = null;
+
+ /**
+ * Test fact commands.
+ *
+ * @throws java.io.IOException
+ */
+ @Test
+ public void testFactCommands() throws IOException, URISyntaxException {
+ dropSampleCubeIfExists();
+ dropFactIfExists();
+
+ createSampleCube();
+ addFactTable();
+ dropSampleCube();
+ }
+
+ private void createSampleCube() throws URISyntaxException {
+ URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource(CUBE_XML_FILE);
+ String cubeList = getCubeCommand().showCubes();
+ Assert.assertFalse(cubeList.contains(CUBE_NAME));
+ getCubeCommand().createCube(new File(cubeSpec.toURI()).getAbsolutePath());
+ cubeList = getCubeCommand().showCubes();
+ Assert.assertTrue(cubeList.contains(CUBE_NAME));
+ }
+
+ private void dropSampleCube() {
+ getCubeCommand().dropCube(CUBE_NAME);
+ TestLensStorageCommands.dropStorage(FACT_LOCAL);
+ }
+
+ private static LensFactCommands getCommand() {
+ if (command == null) {
+ LensClient client = new LensClient();
+ command = new LensFactCommands();
+ command.setClient(client);
+ }
+ return command;
+ }
+
+ private static LensCubeCommands getCubeCommand() {
+ if (cubeCommands == null) {
+ LensClient client = new LensClient();
+ cubeCommands = new LensCubeCommands();
+ cubeCommands.setClient(client);
+ }
+ return cubeCommands;
+ }
+
+ /**
+ * Adds the fact_without_wt table.
+ *
+ * @throws java.io.IOException
+ */
+ public static void addFactTable() throws IOException, URISyntaxException {
+ LensFactCommands command = getCommand();
+ String factList = command.showFacts(null);
+ Assert.assertEquals(command.showFacts(CUBE_NAME), "No fact found for " + CUBE_NAME);
+ Assert.assertEquals("No fact found", factList, "Fact tables should not be found.");
+ // add local storage before adding fact table
+ TestLensStorageCommands.addLocalStorage(FACT_LOCAL);
+ URL factSpec = TestLensFactCommandsWithMissingWeight.class.getClassLoader().getResource(FACT_XML_FILE);
+ String response = null;
+ response = command.createFact(new File(factSpec.toURI()).getAbsolutePath());
+
+ Assert.assertEquals(response, "failed", "Fact table creation should not be successful.");
+ Assert.assertEquals(command.showFacts(CUBE_NAME), "No fact found for " + CUBE_NAME,
+ "Fact tables should not be found.");
+ }
+
+ /**
+ * Drop fact_without_wt table if exixsts.
+ */
+ public static void dropFactIfExists() {
+ LensFactCommands command = getCommand();
+ String factList = command.showFacts(null);
+ if (factList.contains(FACT_NAME)) {
+ command.dropFact(FACT_NAME, false);
+ TestLensStorageCommands.dropStorage(FACT_LOCAL);
+ }
+ }
+
+ private void dropSampleCubeIfExists() {
+ String cubeList = getCubeCommand().showCubes();
+ if (cubeList.contains(CUBE_NAME)) {
+ getCubeCommand().dropCube(CUBE_NAME);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/cube_with_no_weight_facts.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/cube_with_no_weight_facts.xml b/lens-cli/src/test/resources/cube_with_no_weight_facts.xml
new file mode 100644
index 0000000..263ca88
--- /dev/null
+++ b/lens-cli/src/test/resources/cube_with_no_weight_facts.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_base_cube name="cube_with_no_weight_facts" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <measures>
+ <measure name="measure1" type="BIGINT" />
+ <measure name="measure2" type="INT" default_aggr="SUM" />
+ <measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00' />
+ <measure name="measure4" type="DOUBLE" default_aggr="MIN" />
+ </measures>
+ <dim_attributes>
+ <dim_attribute name="dim1" type="INT" />
+ <dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00' />
+ <dim_attribute name="dim3" type="INT">
+ <ref_spec>
+ <table_references>
+ <table_reference table="dim_table" column="id" />
+ </table_references>
+ </ref_spec>
+ </dim_attribute>
+ </dim_attributes>
+ <expressions>
+ <expression name="expr_msr5" type="DOUBLE">
+ <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/>
+ <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/>
+ </expression>
+ </expressions>
+</x_base_cube>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/dim-local-storage-element.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/dim-local-storage-element.xml b/lens-cli/src/test/resources/dim-local-storage-element.xml
index 5a19cfa..7a58aa3 100644
--- a/lens-cli/src/test/resources/dim-local-storage-element.xml
+++ b/lens-cli/src/test/resources/dim-local-storage-element.xml
@@ -21,15 +21,14 @@
-->
<x_storage_table_element xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <update_periods>
+ <update_period>DAILY</update_period>
+ </update_periods>
<storage_name>dim_local</storage_name>
-
<table_desc external="true" field_delimiter="," table_location="/tmp/examples/local">
<part_cols>
<column comment="Time column" name="dt" type="STRING" />
</part_cols>
<time_part_cols>dt</time_part_cols>
</table_desc>
- <update_periods>
- <update_period>DAILY</update_period>
- </update_periods>
</x_storage_table_element>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/dim_table.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/dim_table.xml b/lens-cli/src/test/resources/dim_table.xml
index 932c975..96b2244 100644
--- a/lens-cli/src/test/resources/dim_table.xml
+++ b/lens-cli/src/test/resources/dim_table.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="test_dim" table_name="dim_table" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="test_dim" table_name="dim_table" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT" />
@@ -32,6 +32,9 @@
</properties>
<storage_tables>
<storage_table>
+ <update_periods>
+ <update_period>HOURLY</update_period>
+ </update_periods>
<storage_name>local</storage_name>
<table_desc external="true" field_delimiter="," table_location="/tmp/examples/local">
<part_cols>
@@ -39,9 +42,6 @@
</part_cols>
<time_part_cols>dt</time_part_cols>
</table_desc>
- <update_periods>
- <update_period>HOURLY</update_period>
- </update_periods>
</storage_table>
</storage_tables>
</x_dimension_table>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/dim_table2.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/dim_table2.xml b/lens-cli/src/test/resources/dim_table2.xml
index 1d6e138..000f061 100644
--- a/lens-cli/src/test/resources/dim_table2.xml
+++ b/lens-cli/src/test/resources/dim_table2.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="test_dim" table_name="dim_table2" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="test_dim" table_name="dim_table2" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT" />
@@ -32,7 +32,9 @@
</properties>
<storage_tables>
<storage_table>
- <update_periods>HOURLY</update_periods>
+ <update_periods>
+ <update_period>HOURLY</update_period>
+ </update_periods>
<storage_name>dim_local</storage_name>
<table_desc external="true" field_delimiter="," table_location="/tmp/examples/dim1">
<part_cols>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/fact-local-storage-element.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/fact-local-storage-element.xml b/lens-cli/src/test/resources/fact-local-storage-element.xml
index 8ec3b42..d9bb111 100644
--- a/lens-cli/src/test/resources/fact-local-storage-element.xml
+++ b/lens-cli/src/test/resources/fact-local-storage-element.xml
@@ -21,6 +21,10 @@
-->
<x_storage_table_element xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <update_periods>
+ <update_period>HOURLY</update_period>
+ <update_period>DAILY</update_period>
+ </update_periods>
<storage_name>fact_local</storage_name>
<table_desc external="true" field_delimiter="," table_location="/tmp/examples/local">
<part_cols>
@@ -28,8 +32,4 @@
</part_cols>
<time_part_cols>dt</time_part_cols>
</table_desc>
- <update_periods>
- <update_period>HOURLY</update_period>
- <update_period>DAILY</update_period>
- </update_periods>
</x_storage_table_element>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/fact_without_weight.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/fact_without_weight.xml b/lens-cli/src/test/resources/fact_without_weight.xml
new file mode 100644
index 0000000..fc0c670
--- /dev/null
+++ b/lens-cli/src/test/resources/fact_without_weight.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_fact_table cube_name="cube_with_no_weight_facts" name="fact_without_wt" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd " >
+ <columns>
+ <column comment="" name="dim1" type="INT" />
+ <column comment="" name="measure1" type="BIGINT" />
+ <column comment="" name="measure2" type="INT" />
+ <column comment="" name="measure3" type="FLOAT" />
+ </columns>
+ <properties>
+ <property name="fact_without_wt.prop" value="f1" />
+ <property name="cube.fact.is.aggregated" value="true" />
+ </properties>
+ <storage_tables>
+ <storage_table>
+ <update_periods>
+ <update_period>HOURLY</update_period>
+ <update_period>DAILY</update_period>
+ <update_period>MONTHLY</update_period>
+ </update_periods>
+ <storage_name>fact_local_without_wt</storage_name>
+ <table_desc external="true" field_delimiter="," table_location="/tmp/examples/fact_local_without_wt">
+ <part_cols>
+ <column comment="Time column" name="dt" type="STRING" />
+ </part_cols>
+ <time_part_cols>dt</time_part_cols>
+ </table_desc>
+ </storage_table>
+ </storage_tables>
+</x_fact_table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/sample-cube.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/sample-cube.xml b/lens-cli/src/test/resources/sample-cube.xml
index d54bcfb..f0eba57 100644
--- a/lens-cli/src/test/resources/sample-cube.xml
+++ b/lens-cli/src/test/resources/sample-cube.xml
@@ -21,18 +21,16 @@
-->
<x_base_cube name="sample_cube" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="sample_cube.prop" value="sample" />
+ <property name="cube.sample_cube.timed.dimensions.list" value="dt" />
+ </properties>
<measures>
<measure name="measure1" type="BIGINT" />
<measure name="measure2" type="INT" default_aggr="SUM" />
<measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00' />
<measure name="measure4" type="DOUBLE" default_aggr="MIN" />
</measures>
- <expressions>
- <expression name="expr_msr5" type="DOUBLE">
- <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/>
- <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/>
- </expression>
- </expressions>
<dim_attributes>
<dim_attribute name="dim1" type="INT" />
<dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00' />
@@ -44,8 +42,10 @@
</ref_spec>
</dim_attribute>
</dim_attributes>
- <properties>
- <property name="sample_cube.prop" value="sample" />
- <property name="cube.sample_cube.timed.dimensions.list" value="dt" />
- </properties>
+ <expressions>
+ <expression name="expr_msr5" type="DOUBLE">
+ <expr_spec expr = "measure3 + measure4" end_time='2013-12-12T00:00:00'/>
+ <expr_spec expr = "measure3 + measure4 + 0.01" start_time='2013-12-12T00:00:00'/>
+ </expression>
+ </expressions>
</x_base_cube>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-cli/src/test/resources/test-dimension.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/test-dimension.xml b/lens-cli/src/test/resources/test-dimension.xml
index d602569..a9b7c08 100644
--- a/lens-cli/src/test/resources/test-dimension.xml
+++ b/lens-cli/src/test/resources/test-dimension.xml
@@ -24,8 +24,8 @@
<attributes>
<dim_attribute name="id" type="INT" />
<dim_attribute name="name" type="STRING" />
- <dim_attribute name="detail" type="STRING" start_time='2013-12-01-00:00' />
- <dim_attribute name="d2id" type="INT" start_time='2013-12-01-00:00'>
+ <dim_attribute name="detail" type="STRING" start_time='2013-12-01T00:00:00' />
+ <dim_attribute name="d2id" type="INT" start_time='2013-12-01T00:00:00'>
<ref_spec>
<table_references>
<table_reference table="test_dim2" column="id" />
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index a6651f6..44f4f1f 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -21,6 +21,7 @@ package org.apache.lens.client;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
@@ -33,10 +34,13 @@ import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
+import javax.xml.XMLConstants;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
+import javax.xml.validation.Schema;
+import javax.xml.validation.SchemaFactory;
import org.apache.lens.api.APIResult;
import org.apache.lens.api.APIResult.Status;
@@ -52,6 +56,8 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import org.xml.sax.SAXException;
+
import com.google.common.base.Joiner;
import com.google.common.io.Files;
@@ -67,10 +73,17 @@ public class LensMetadataClient {
static {
try {
+ ClassLoader classLoader = LensMetadataClient.class.getClassLoader();
+ SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ URI uri = classLoader.getResource("cube-0.1.xsd").toURI();
+ LOG.info("URI for cube schema: " + uri.toString());
+ Schema schema = sf.newSchema(uri.toURL());
JAXBContext jaxbContext = JAXBContext.newInstance(ObjectFactory.class);
JAXB_UNMARSHALLER = jaxbContext.createUnmarshaller();
- } catch (JAXBException e) {
- throw new RuntimeException("Could not initialize JAXBCOntext");
+ JAXB_UNMARSHALLER.setSchema(schema);
+ } catch (JAXBException | URISyntaxException | MalformedURLException | SAXException e) {
+ LOG.error("Could not initialize JAXBContext. ", e);
+ throw new RuntimeException("Could not initialize JAXBContext. ", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/city_subset.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/city_subset.xml b/lens-examples/src/main/resources/city_subset.xml
index 5059e5e..18c7847 100644
--- a/lens-examples/src/main/resources/city_subset.xml
+++ b/lens-examples/src/main/resources/city_subset.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="city" table_name="city_subset" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="city" table_name="city_subset" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/customer_table.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/customer_table.xml b/lens-examples/src/main/resources/customer_table.xml
index 18c387f..af308c2 100644
--- a/lens-examples/src/main/resources/customer_table.xml
+++ b/lens-examples/src/main/resources/customer_table.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="customer" table_name="customer_table" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="customer" table_name="customer_table" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/dim_table.xml b/lens-examples/src/main/resources/dim_table.xml
index 25367af..10c9ac0 100644
--- a/lens-examples/src/main/resources/dim_table.xml
+++ b/lens-examples/src/main/resources/dim_table.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="sample_dim" table_name="dim_table" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="sample_dim" table_name="dim_table" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table2.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/dim_table2.xml b/lens-examples/src/main/resources/dim_table2.xml
index eef1513..e72f9c5 100644
--- a/lens-examples/src/main/resources/dim_table2.xml
+++ b/lens-examples/src/main/resources/dim_table2.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="sample_dim2" table_name="dim_table2" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="sample_dim2" table_name="dim_table2" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table3.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/dim_table3.xml b/lens-examples/src/main/resources/dim_table3.xml
index 386c394..7955426 100644
--- a/lens-examples/src/main/resources/dim_table3.xml
+++ b/lens-examples/src/main/resources/dim_table3.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="sample_db_dim" table_name="dim_table3" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="sample_db_dim" table_name="dim_table3" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/dim_table4.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/dim_table4.xml b/lens-examples/src/main/resources/dim_table4.xml
index 69e3c50..9de1cd7 100644
--- a/lens-examples/src/main/resources/dim_table4.xml
+++ b/lens-examples/src/main/resources/dim_table4.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="sample_dim" table_name="dim_table4" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="sample_dim" table_name="dim_table4" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/product_db_table.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/product_db_table.xml b/lens-examples/src/main/resources/product_db_table.xml
index d961c44..867d37b 100644
--- a/lens-examples/src/main/resources/product_db_table.xml
+++ b/lens-examples/src/main/resources/product_db_table.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="product" table_name="product_db_table" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="product" table_name="product_db_table" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/product_table.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/product_table.xml b/lens-examples/src/main/resources/product_table.xml
index 86a1115..303f3ad 100644
--- a/lens-examples/src/main/resources/product_table.xml
+++ b/lens-examples/src/main/resources/product_table.xml
@@ -19,7 +19,7 @@
under the License.
-->
-<x_dimension_table dimension_name="product" table_name="product_table" weight="0.0" xmlns="uri:lens:cube:0.1"
+<x_dimension_table dimension_name="product" table_name="product_table" weight="100.0" xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
<columns>
<column comment="ID" name="id" type="INT"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sales-aggr-fact1.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/sales-aggr-fact1.xml b/lens-examples/src/main/resources/sales-aggr-fact1.xml
index 073052d..728c775 100644
--- a/lens-examples/src/main/resources/sales-aggr-fact1.xml
+++ b/lens-examples/src/main/resources/sales-aggr-fact1.xml
@@ -64,14 +64,14 @@
<storage_name>mydb</storage_name>
<table_desc external="true" field_delimiter="," table_location="/tmp/db-storage.db"
storage_handler_name="org.apache.lens.storage.db.DBStorageHandler">
- <table_parameters>
- <property name="lens.metastore.native.db.name" value="default"/>
- </table_parameters>
<part_cols>
<column comment="Process time partition" name="pt" type="STRING"/>
<column comment="Order time partition" name="ot" type="STRING"/>
<column comment="Delivery time partition" name="dt" type="STRING"/>
</part_cols>
+ <table_parameters>
+ <property name="lens.metastore.native.db.name" value="default"/>
+ </table_parameters>
<time_part_cols>pt</time_part_cols>
<time_part_cols>ot</time_part_cols>
<time_part_cols>dt</time_part_cols>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sales-aggr-fact2.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/sales-aggr-fact2.xml b/lens-examples/src/main/resources/sales-aggr-fact2.xml
index 67b7171..61edf64 100644
--- a/lens-examples/src/main/resources/sales-aggr-fact2.xml
+++ b/lens-examples/src/main/resources/sales-aggr-fact2.xml
@@ -58,14 +58,14 @@
<storage_name>mydb</storage_name>
<table_desc external="true" field_delimiter="," table_location="/tmp/db-storage.db"
storage_handler_name="org.apache.lens.storage.db.DBStorageHandler">
- <table_parameters>
- <property name="lens.metastore.native.db.name" value="default"/>
- </table_parameters>
<part_cols>
<column comment="Process time partition" name="pt" type="STRING"/>
<column comment="Order time partition" name="ot" type="STRING"/>
<column comment="Delivery time partition" name="dt" type="STRING"/>
</part_cols>
+ <table_parameters>
+ <property name="lens.metastore.native.db.name" value="default"/>
+ </table_parameters>
<time_part_cols>pt</time_part_cols>
<time_part_cols>ot</time_part_cols>
<time_part_cols>dt</time_part_cols>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sales-cube.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/sales-cube.xml b/lens-examples/src/main/resources/sales-cube.xml
index e44141c..4923aa3 100644
--- a/lens-examples/src/main/resources/sales-cube.xml
+++ b/lens-examples/src/main/resources/sales-cube.xml
@@ -21,6 +21,13 @@
-->
<x_base_cube name="sales" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.sales.timed.dimensions.list" value="order_time,delivery_time" />
+ <property name="cube.timedim.partition.order_time" value="ot" />
+ <property name="cube.timedim.partition.delivery_time" value="dt" />
+ <property name="cube.timedim.relation.order_time" value="delivery_time+[-20 days,-1 hour]" />
+ <!-- means dt-20days <= ot <= dt-1hour -->
+ </properties>
<measures>
<measure name="unit_sales" type="BIGINT" default_aggr="SUM" display_string="Unit Sales" format_string="#,###"/>
<measure name="store_sales" type="DOUBLE" default_aggr="SUM" display_string="Store Sales" format_string="#,###.##"/>
@@ -36,16 +43,6 @@
<measure name="rating" type="FLOAT" default_aggr="AVG"
display_string="Rating"/>
</measures>
- <expressions>
- <expression name="profit" type="DOUBLE" display_string="Profit">
- <expr_spec expr="store_sales - store_cost"/>
- </expression>
- <expression name="promotion_sales" type="DOUBLE" display_string="Promotion sales">
- <expr_spec expr="sum(case when promotion_id = 0 then 0 else store_sales end)" start_time='2015-04-12T00:00:00'/>
- <expr_spec expr="sum(case when promotion_id = -1 then 0 when promotion_id = -1 then 0 else store_sales end)"
- end_time="2015-04-12T00:00:00"/>
- </expression>
- </expressions>
<dim_attributes>
<dim_attribute name="customer_id" type="INT" />
<dim_attribute name="product_id" type="INT" />
@@ -58,24 +55,34 @@
<dim_attribute name="production_city_id" type="INT" />
<dim_attribute name="delivery_city_id" type="INT" />
<dim_attribute name="customer_city_name" type="string" description="City name to which the customer belongs"
- display_string="Customer City">
+ display_string="Customer City">
<ref_spec>
<chain_ref_column chain_name="customer_city" ref_col="name" />
</ref_spec>
</dim_attribute>
<dim_attribute name="production_city_name" type="STRING" description="City name in which the product was produced"
- display_string="Production City">
+ display_string="Production City">
<ref_spec>
<chain_ref_column chain_name="production_city" ref_col="name" />
</ref_spec>
</dim_attribute>
<dim_attribute name="delivery_city_name" type="STRING" description="City name to which the product was delivered"
- display_string="Delivery City">
+ display_string="Delivery City">
<ref_spec>
<chain_ref_column chain_name="delivery_city" ref_col="name" />
</ref_spec>
</dim_attribute>
</dim_attributes>
+ <expressions>
+ <expression name="profit" type="DOUBLE" display_string="Profit">
+ <expr_spec expr="store_sales - store_cost"/>
+ </expression>
+ <expression name="promotion_sales" type="DOUBLE" display_string="Promotion sales">
+ <expr_spec expr="sum(case when promotion_id = 0 then 0 else store_sales end)" start_time='2015-04-12T00:00:00'/>
+ <expr_spec expr="sum(case when promotion_id = -1 then 0 when promotion_id = -1 then 0 else store_sales end)"
+ end_time="2015-04-12T00:00:00"/>
+ </expression>
+ </expressions>
<join_chains>
<join_chain name="customer_details">
<paths>
@@ -150,11 +157,4 @@
</paths>
</join_chain>
</join_chains>
- <properties>
- <property name="cube.sales.timed.dimensions.list" value="order_time,delivery_time" />
- <property name="cube.timedim.partition.order_time" value="ot" />
- <property name="cube.timedim.partition.delivery_time" value="dt" />
- <property name="cube.timedim.relation.order_time" value="delivery_time+[-20 days,-1 hour]" />
- <!-- means dt-20days <= ot <= dt-1hour -->
- </properties>
</x_base_cube>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1584f001/lens-examples/src/main/resources/sample-cube.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/sample-cube.xml b/lens-examples/src/main/resources/sample-cube.xml
index 0d97842..7b784ea 100644
--- a/lens-examples/src/main/resources/sample-cube.xml
+++ b/lens-examples/src/main/resources/sample-cube.xml
@@ -21,18 +21,15 @@
-->
<x_base_cube name="sample_cube" xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <properties>
+ <property name="cube.sample_cube.timed.dimensions.list" value="dt"/>
+ </properties>
<measures>
<measure name="measure1" type="BIGINT"/>
<measure name="measure2" type="INT" default_aggr="SUM"/>
<measure name="measure3" type="FLOAT" default_aggr="MAX" start_time='2013-12-12T00:00:00'/>
<measure name="measure4" type="DOUBLE" default_aggr="MIN"/>
</measures>
- <expressions>
- <expression name="expr_msr5" type="DOUBLE">
- <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' />
- <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/>
- </expression>
- </expressions>
<dim_attributes>
<dim_attribute name="dim1" type="INT"/>
<dim_attribute name="dim2" type="INT" start_time='2013-12-01T00:00:00'/>
@@ -44,7 +41,10 @@
</ref_spec>
</dim_attribute>
</dim_attributes>
- <properties>
- <property name="cube.sample_cube.timed.dimensions.list" value="dt"/>
- </properties>
+ <expressions>
+ <expression name="expr_msr5" type="DOUBLE">
+ <expr_spec expr = "measure3 + measure4" start_time='2013-12-12T00:00:00' />
+ <expr_spec expr = "measure3 + measure4 + 0.01" end_time='2013-12-12T00:00:00'/>
+ </expression>
+ </expressions>
</x_base_cube>
[25/51] [abbrv] incubator-lens git commit: LENS-414: Move
LeastPartitionResolver after LeastFactResolver
Posted by jd...@apache.org.
LENS-414: Move LeastPartitionResolver after LeastFactResolver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b8995ed3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b8995ed3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b8995ed3
Branch: refs/heads/current-release-line
Commit: b8995ed3ff3d25bca82448a12fdcbaafc6ada2d6
Parents: ff31ad9
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Jun 12 11:44:33 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Fri Jun 12 11:44:33 2015 +0530
----------------------------------------------------------------------
.../main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java | 5 ++++-
.../java/org/apache/lens/cube/parse/LeastPartitionResolver.java | 4 ++++
.../test/java/org/apache/lens/cube/parse/TestQueryMetrics.java | 4 ++--
3 files changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8995ed3/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
index 57c91e7..72dc64b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java
@@ -175,10 +175,13 @@ public class CubeQueryRewriter {
rewriters.add(denormResolver);
// Prune candidate facts without any valid expressions
rewriters.add(exprResolver);
- rewriters.add(new LeastPartitionResolver(conf));
+ // We can have LightestFactResolver before LeastPartitionResolver - that says
+ // "if two facts have the same least weight, then the fact with least number of time partitions queried will be
+ // picked". This will be useful, if users did not set fact weights.
if (!lightFactFirst) {
rewriters.add(new LightestFactResolver(conf));
}
+ rewriters.add(new LeastPartitionResolver(conf));
rewriters.add(new LightestDimensionResolver(conf));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8995ed3/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
index d0b3ccb..6e4e948 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
@@ -41,6 +41,10 @@ class LeastPartitionResolver implements ContextRewriter {
if (cubeql.getCube() != null && !cubeql.getCandidateFactSets().isEmpty()) {
Map<Set<CandidateFact>, Integer> factPartCount = new HashMap<Set<CandidateFact>, Integer>();
+ //The number of partitions being calculated is not the actual number of partitions,
+ // they are number of time values now instead of partitions.
+ // This seems fine, as the less number of time values actually represent the rollups on time. And with
+ // MaxCoveringFactResolver facts with less partitions which are not covering the range would be removed.
for (Set<CandidateFact> facts : cubeql.getCandidateFactSets()) {
factPartCount.put(facts, getPartCount(facts));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b8995ed3/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
index 39f70eb..255aade 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestQueryMetrics.java
@@ -57,9 +57,9 @@ public class TestQueryMetrics extends TestQueryRewrite {
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.FieldValidator-ITER-7",
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.GroupbyResolver-ITER-6",
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.JoinResolver-ITER-8",
- "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LeastPartitionResolver-ITER-17",
+ "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LeastPartitionResolver-ITER-18",
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestDimensionResolver-ITER-19",
- "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestFactResolver-ITER-18",
+ "lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.LightestFactResolver-ITER-17",
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.MaxCoveringFactResolver-ITER-13",
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-11",
"lens.MethodMetricGauge.testCubeRewriteStackName-org.apache.lens.cube.parse.StorageTableResolver-ITER-12",
[37/51] [abbrv] incubator-lens git commit: LENS-25: Print the query
execution status on CLI
Posted by jd...@apache.org.
LENS-25: Print the query execution status on CLI
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/25428747
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/25428747
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/25428747
Branch: refs/heads/current-release-line
Commit: 25428747d76428dadeb5a573dc751ed9b4b15432
Parents: 1fde486
Author: Raju Bairishetti <ra...@gmail.com>
Authored: Thu Jun 18 16:41:59 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jun 18 16:41:59 2015 +0530
----------------------------------------------------------------------
.../cli/commands/LensConnectionCommands.java | 125 +++++++++++++++++++
.../java/org/apache/lens/client/LensClient.java | 6 +
.../apache/lens/client/LensClientConfig.java | 2 +-
.../org/apache/lens/client/LensStatement.java | 21 ++++
src/site/apt/user/cli.apt | 4 +
tools/conf-pseudo-distr/client/log4j.properties | 28 ++---
tools/conf/client/log4j.properties | 28 ++---
7 files changed, 183 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/25428747/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
index 675e6c8..8b4face 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cli.commands;
+import java.util.Enumeration;
import java.util.List;
import javax.ws.rs.ProcessingException;
@@ -25,6 +26,10 @@ import javax.ws.rs.ProcessingException;
import org.apache.lens.api.APIResult;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
+import org.apache.lens.client.LensClient;
+
+import org.apache.log4j.*;
+
import org.springframework.shell.core.ExitShellRequest;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
@@ -153,6 +158,126 @@ public class LensConnectionCommands extends BaseLensCommand {
}
/**
+ * Enables to show all class level logs on cli
+ * @param enable
+ */
+ @CliCommand(value = {"debug"}, help = "prints all class level logs and verbose logs on cli for debugging purpose."
+ + " 'debug false' to turn off all class level logging and verbose level logging ")
+ public void debug(@CliOption(key = {"", "enable"},
+ mandatory = false, unspecifiedDefaultValue = "true",
+ help = "To print all logs on cli for debugging purpose") boolean enable) {
+ Logger logger = LoggerUtil.getRootLogger();
+ Logger cliLogger = LoggerUtil.getCliLogger();
+ if (enable) {
+ LoggerUtil.addConsoleAppenderIfNotPresent(logger);
+ LoggerUtil.addConsoleAppenderIfNotPresent(cliLogger);
+ } else {
+ LoggerUtil.removeConsoleAppender(logger);
+ LoggerUtil.removeConsoleAppender(cliLogger);
+ }
+ }
+
+ /**
+ * Enables to show only cliLogger(verbose) logs on cli
+ * @param enable
+ */
+ @CliCommand(value = {"verbose"}, help = "Show cliLogger logs on cli"
+ + "'verbose false' turns off the cliLogger logs on console")
+ public void verbose(@CliOption(key = {"", "enable"},
+ mandatory = false, unspecifiedDefaultValue = "true",
+ help = "Print the clilogger logs on cli") boolean enable) {
+ Logger cliLogger = LoggerUtil.getCliLogger();
+ if (enable) {
+ LoggerUtil.addConsoleAppenderIfNotPresent(cliLogger);
+ } else {
+ LoggerUtil.removeConsoleAppender(cliLogger);
+ }
+ }
+
+ private static class LoggerUtil {
+ private static Logger logger;
+ private static Logger cliLogger;
+
+ public static Logger getRootLogger() {
+ if (logger == null) {
+ logger = Logger.getRootLogger();
+ }
+ return logger;
+ }
+
+ public static Logger getCliLogger() {
+ if (cliLogger == null) {
+ cliLogger = Logger.getLogger(LensClient.CLILOGGER);
+ }
+ return cliLogger;
+ }
+
+ public static ConsoleAppender getConsoleAppender() {
+ return getConsoleAppender(getPatternLayout());
+ }
+
+ public static ConsoleAppender getConsoleAppender(Layout layout) {
+ ConsoleAppender consoleAppender = new ConsoleAppender();
+ consoleAppender.setLayout(layout);
+ consoleAppender.activateOptions();
+ return consoleAppender;
+ }
+
+ public static PatternLayout getPatternLayout() {
+ String conversionPattern = "%d [%t] %F %-7p - %m%n";
+ return getPatternLayout(conversionPattern);
+ }
+
+ public static PatternLayout getPatternLayout(String conversionPattern) {
+ PatternLayout layout = new PatternLayout();
+ layout.setConversionPattern(conversionPattern);
+ return layout;
+ }
+
+ /**
+ * Check whether ConsoleAppender was already added
+ * @param logger
+ * @return true if it was already added
+ * false if it does not contain ConsoleAppender
+ */
+ public static void addConsoleAppenderIfNotPresent(Logger logger) {
+ boolean isConsoleAppenderAdded = false;
+ Layout layout = null;
+ Enumeration appenderSeries = logger.getAllAppenders();
+ while (appenderSeries.hasMoreElements()) {
+ Appender appender = (Appender) appenderSeries.nextElement();
+ if (appender instanceof ConsoleAppender) {
+ isConsoleAppenderAdded = true;
+ break;
+ } else {
+ layout = appender.getLayout();
+ }
+ }
+ if (!isConsoleAppenderAdded) {
+ if (layout == null) {
+ layout = LoggerUtil.getPatternLayout();
+ }
+ if (logger.getLevel() == null) {
+ logger.setLevel(Level.DEBUG);
+ }
+ ConsoleAppender consoleAppender = LoggerUtil.getConsoleAppender(layout);
+ logger.addAppender(consoleAppender);
+ }
+ }
+
+ public static void removeConsoleAppender(Logger logger) {
+ Enumeration appenderSeries = logger.getAllAppenders();
+ while (appenderSeries.hasMoreElements()) {
+ Appender appender = (Appender) appenderSeries.nextElement();
+ if (appender instanceof ConsoleAppender) {
+ logger.removeAppender(appender);
+ }
+ }
+ }
+
+ }
+
+ /**
* Quit shell.
*
* @return the exit shell request
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/25428747/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 5c3daa8..7399d9e 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -37,6 +37,8 @@ import lombok.Getter;
public class LensClient {
private static final Log LOG = LogFactory.getLog(LensClient.class);
+
+ public static final String CLILOGGER = "cliLogger";
private static final String DEFAULT_PASSWORD = "";
private final LensClientConfig conf;
@Getter
@@ -48,6 +50,10 @@ public class LensClient {
Maps.newHashMap();
private final LensStatement statement;
+ public static Log getCliLooger() {
+ return LogFactory.getLog(CLILOGGER);
+ }
+
public LensClient() {
this(new LensClientConfig());
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/25428747/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
index cf3a97c..edb56ac 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClientConfig.java
@@ -49,7 +49,7 @@ public class LensClientConfig extends Configuration {
private static final String QUERY_POLL_INTERVAL_KEY = CLIENT_PFX + "query.poll.interval";
/** The Constant DEFAULT_QUERY_POLL_INTERVAL. */
- private static final long DEFAULT_QUERY_POLL_INTERVAL = 10 * 1000L;
+ private static final long DEFAULT_QUERY_POLL_INTERVAL = 10L;
/** The Constant USER_NAME. */
private static final String USER_NAME = CLIENT_PFX + "user.name";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/25428747/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
index e2693c9..40e2b86 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
@@ -191,10 +191,27 @@ public class LensStatement {
* @param handle the handle
*/
public void waitForQueryToComplete(QueryHandle handle) {
+ LensClient.getCliLooger().info("Query handle: " + handle);
query = getQuery(handle);
+ while (isQueryInQUEUEDSate()) {
+ query = getQuery(handle);
+ LensClient.getCliLooger().debug("Query " + handle + " status: " + query.getStatus());
+ try {
+ Thread.sleep(connection.getLensConnectionParams().getQueryPollInterval());
+ } catch (InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ LensClient.getCliLooger().info("User query: '" + query.getUserQuery()
+ + "' was submitted to " + query.getSelectedDriverClassName());
+ if (query.getDriverQuery() != null) {
+ LensClient.getCliLooger().info(" Driver query: '" + query.getDriverQuery()
+ + "' and Driver handle: " + query.getDriverOpHandle());
+ }
while (!query.getStatus().finished()
&& !(query.getStatus().toString().equals(Status.CLOSED.toString()))) {
query = getQuery(handle);
+ LensClient.getCliLooger().info(query.getStatus());
try {
Thread.sleep(connection.getLensConnectionParams().getQueryPollInterval());
} catch (InterruptedException e) {
@@ -203,6 +220,10 @@ public class LensStatement {
}
}
+ private boolean isQueryInQUEUEDSate() {
+ return query.getStatus().getStatus().equals(QueryStatus.Status.QUEUED);
+ }
+
/**
* Gets the query web target.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/25428747/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index 20785f1..353c171 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -92,6 +92,8 @@ User CLI Commands
*--+--+
|close/bye|Releases all resources of the server session and exits the shell|
*--+--+
+|debug [[--enable] To print all logs on cli for debugging purpose]|prints all class lelvel logs on cli for debugging purpose|
+*--+--+
|get [--key] \<key\>|Fetches and prints session parameter specified with name <<<key>>> from lens server|
*--+--+
|list resources [[--type] \<resource-type\>]|list all resources from session. If type is provided, lists resources of type <<<resource-type>>>. Valid values for type are jar and file.|
@@ -104,6 +106,8 @@ User CLI Commands
*--+--+
|show params|Fetches and prints all session parameter from lens server|
*--+--+
+|verbose [[--enable] Print the clilogger logs on cli]|Show cliLogger logs for command|
+*--+--+
<<Lens Connection Commands>>
===
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/25428747/tools/conf-pseudo-distr/client/log4j.properties
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/client/log4j.properties b/tools/conf-pseudo-distr/client/log4j.properties
index 5520304..3756cf7 100644
--- a/tools/conf-pseudo-distr/client/log4j.properties
+++ b/tools/conf-pseudo-distr/client/log4j.properties
@@ -19,22 +19,20 @@
# Set root logger level to DEBUG and its only appender to A1.
-log4j.rootLogger=INFO, R
-
-# A1 is set to be a ConsoleAppender.
-log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.rootLogger=INFO, root
log4j.category.org.springframework=OFF
-# A1 uses PatternLayout.
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
-
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${lens.log.dir}/lensclient.log
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
-
-log4j.appender.R.MaxFileSize=100000KB
+log4j.appender.root=org.apache.log4j.RollingFileAppender
+log4j.appender.root.File=${lens.log.dir}/lensclient.log
+log4j.appender.root.layout=org.apache.log4j.PatternLayout
+log4j.appender.root.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.root.MaxFileSize=100000KB
# Keep 20 backup files
-log4j.appender.R.MaxBackupIndex=20
+log4j.appender.root.MaxBackupIndex=20
+
+log4j.category.cliLogger=DEBUG, verboseLogger
+log4j.appender.verboseLogger=org.apache.log4j.ConsoleAppender
+log4j.appender.verboseLogger.layout=org.apache.log4j.PatternLayout
+log4j.appender.verboseLogger.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.additivity.cliLogger=false
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/25428747/tools/conf/client/log4j.properties
----------------------------------------------------------------------
diff --git a/tools/conf/client/log4j.properties b/tools/conf/client/log4j.properties
index 5520304..3756cf7 100644
--- a/tools/conf/client/log4j.properties
+++ b/tools/conf/client/log4j.properties
@@ -19,22 +19,20 @@
# Set root logger level to DEBUG and its only appender to A1.
-log4j.rootLogger=INFO, R
-
-# A1 is set to be a ConsoleAppender.
-log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.rootLogger=INFO, root
log4j.category.org.springframework=OFF
-# A1 uses PatternLayout.
-log4j.appender.A1.layout=org.apache.log4j.PatternLayout
-log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
-
-log4j.appender.R=org.apache.log4j.RollingFileAppender
-log4j.appender.R.File=${lens.log.dir}/lensclient.log
-log4j.appender.R.layout=org.apache.log4j.PatternLayout
-log4j.appender.R.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
-
-log4j.appender.R.MaxFileSize=100000KB
+log4j.appender.root=org.apache.log4j.RollingFileAppender
+log4j.appender.root.File=${lens.log.dir}/lensclient.log
+log4j.appender.root.layout=org.apache.log4j.PatternLayout
+log4j.appender.root.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.root.MaxFileSize=100000KB
# Keep 20 backup files
-log4j.appender.R.MaxBackupIndex=20
+log4j.appender.root.MaxBackupIndex=20
+
+log4j.category.cliLogger=DEBUG, verboseLogger
+log4j.appender.verboseLogger=org.apache.log4j.ConsoleAppender
+log4j.appender.verboseLogger.layout=org.apache.log4j.PatternLayout
+log4j.appender.verboseLogger.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.additivity.cliLogger=false
[24/51] [abbrv] incubator-lens git commit: LENS-531 : Add cli command
to show all queryable fields of cube or dimension
Posted by jd...@apache.org.
LENS-531 : Add cli command to show all queryable fields of cube or dimension
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ff31ad96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ff31ad96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ff31ad96
Branch: refs/heads/current-release-line
Commit: ff31ad96b928850f4d549ebcde7e79f260b025e7
Parents: 7a7bdec
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jun 12 11:31:19 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jun 12 11:31:19 2015 +0530
----------------------------------------------------------------------
lens-api/pom.xml | 5 +
lens-api/src/main/resources/cube-0.1.xsd | 611 ++++++++++---------
.../LensCRUDStoragePartitionCommand.java | 114 ----
.../lens/cli/commands/LensCubeCommands.java | 21 +-
.../cli/commands/LensDimensionCommands.java | 19 +-
.../commands/LensDimensionTableCommands.java | 2 +-
.../lens/cli/commands/LensFactCommands.java | 2 +-
.../lens/cli/commands/LensQueryCommands.java | 3 +-
.../cli/commands/LogicalTableCrudCommand.java | 31 +
.../cli/commands/PhysicalTableCrudCommand.java | 114 ++++
.../org/apache/lens/cli/skel/LensBanner.java | 5 +
.../lens/cli/skel/LensHistoryFileProvider.java | 2 +-
.../lens/cli/skel/LensPromptProvider.java | 2 +-
.../apache/lens/cli/table/CollectionTable.java | 59 ++
.../lens/cli/table/CollectionTableFactory.java | 158 +++++
.../lens/cli/table/XFlattenedColumnTable.java | 97 +++
.../apache/lens/cli/table/XJoinChainTable.java | 77 +++
.../apache/lens/cli/TestLensCubeCommands.java | 40 +-
.../lens/cli/TestLensDimensionCommands.java | 26 +-
.../apache/lens/cli/TestLensQueryCommands.java | 15 +-
.../java/org/apache/lens/client/LensClient.java | 7 +
.../apache/lens/client/LensMetadataClient.java | 35 +-
.../api/metastore/CubeMetastoreService.java | 6 +-
.../metastore/CubeMetastoreServiceImpl.java | 80 ++-
.../server/metastore/MetastoreResource.java | 36 +-
pom.xml | 7 +-
src/site/apt/user/cli.apt | 10 +-
27 files changed, 1109 insertions(+), 475 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index adc464a..184f8a8 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -66,6 +66,10 @@
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.jvnet.jaxb2_commons</groupId>
+ <artifactId>jaxb2-basics-runtime</artifactId>
+ </dependency>
</dependencies>
<build>
<plugins>
@@ -85,6 +89,7 @@
<args>
<arg>-extension</arg>
<arg>-Xinheritance</arg>
+ <arg>-XtoString</arg>
</args>
<plugins>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 719d10d..2ae606f 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -18,18 +18,53 @@
under the License.
-->
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" attributeFormDefault="unqualified" elementFormDefault="qualified"
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xjc="http://java.sun.com/xml/ns/jaxb/xjc"
+ attributeFormDefault="unqualified" elementFormDefault="qualified"
targetNamespace="uri:lens:cube:0.1" xmlns="uri:lens:cube:0.1" xmlns:jaxb="http://java.sun.com/xml/ns/jaxb"
jaxb:version="2.1">
+
<xs:annotation>
<xs:appinfo>
<jaxb:schemaBindings>
- <jaxb:package name="org.apache.lens.api.metastore" />
+ <jaxb:package name="org.apache.lens.api.metastore"/>
</jaxb:schemaBindings>
</xs:appinfo>
</xs:annotation>
- <xs:element name="x_cube" abstract="true" type="x_cube" />
+ <xs:element name="x_field" abstract="true" type="x_field"/>
+
+ <xs:complexType name="x_field" abstract="true">
+ <xs:annotation>
+ <xs:documentation>
+ some documentation
+ </xs:documentation>
+ </xs:annotation>
+ <xs:attribute type="xs:string" name="name" use="required"/>
+ <xs:attribute type="xs:string" name="display_string">
+ <xs:annotation>
+ <xs:documentation>
+ Display string that should be shown to end users. Can be space separated. It gives UI systems
+ built on top to use this value to show the field to end user.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:string" name="description"/>
+ </xs:complexType>
+
+ <xs:element name="x_fields" type="x_fields"/>
+
+ <xs:complexType name="x_fields">
+ <xs:annotation>
+ <xs:documentation>
+ Set of fields.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:sequence>
+ <xs:element type="x_field" name="fields" maxOccurs="unbounded" minOccurs="1"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:element name="x_cube" abstract="true" type="x_cube"/>
<xs:complexType name="x_cube" abstract="true">
<xs:annotation>
@@ -65,44 +100,44 @@
Additional properties:
1. cube.timedim.relation.{time_dim1}: time_dim2+[timediff1,timediff2]. It's assumed that
- timediff1 is smaller than timediff2. Means that time_dim1 can be expected to be between
- [time_dim2+timediff1 and time_dim2+timediff2]. One use case would be the following:
+ timediff1 is smaller than timediff2. Means that time_dim1 can be expected to be between
+ [time_dim2+timediff1 and time_dim2+timediff2]. One use case would be the following:
- 1.1. if a query is on range of time_dim1 and
- time_dim1 has no partitioning column in some fact, then partitions for time_dim2 can be looked at.
- Let's say time_dim2's part col is part_col2, then for a query asking for time_dim1 between [a,b) can
- be answered by looking at partitions of part_col2 in range [a-timediff2, b-timediff1).
+ 1.1. if a query is on range of time_dim1 and
+ time_dim1 has no partitioning column in some fact, then partitions for time_dim2 can be looked at.
+ Let's say time_dim2's part col is part_col2, then for a query asking for time_dim1 between [a,b) can
+ be answered by looking at partitions of part_col2 in range [a-timediff2, b-timediff1).
- This property is first looked into fact properties, then cube properties and if that's a derived cube,
- then next in base cube properties. Wherever found first, that will be considered as the final relation
- between time dimensions.
+ This property is first looked into fact properties, then cube properties and if that's a derived cube,
+ then next in base cube properties. Wherever found first, that will be considered as the final relation
+ between time dimensions.
- Time dimension relations are transitive, but not reversible. i.e.
- cube.timedim.relation.time_dim1 = time_dim2 + [a, b]
- cube.timedim.relation.time_dim2 = time_dim3 + [c, d]
+ Time dimension relations are transitive, but not reversible. i.e.
+ cube.timedim.relation.time_dim1 = time_dim2 + [a, b]
+ cube.timedim.relation.time_dim2 = time_dim3 + [c, d]
- implies:
+ implies:
- cube.timedim.relation.time_dim1 = time_dim3 + [a+c, b+d]
+ cube.timedim.relation.time_dim1 = time_dim3 + [a+c, b+d]
- but not:
+ but not:
- cube.timedim.relation.time_dim2 = time_dim1 + [-b, -a]
+ cube.timedim.relation.time_dim2 = time_dim1 + [-b, -a]
- Reverse relations have to be defined explicitly.
+ Reverse relations have to be defined explicitly.
- Timediff syntax is sign, quantity and unit. Spaces in between can be present. e.g. -4 days, +4days, +4 day
- etc all are valid.
+ Timediff syntax is sign, quantity and unit. Spaces in between can be present. e.g. -4 days, +4days, +4 day
+ etc all are valid.
</xs:documentation>
</xs:annotation>
</xs:element>
</xs:sequence>
- <xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="xs:string" name="description" />
+ <xs:attribute type="xs:string" name="name" use="required"/>
+ <xs:attribute type="xs:string" name="description"/>
</xs:complexType>
- <xs:element name="x_base_cube" type="x_base_cube" />
+ <xs:element name="x_base_cube" type="x_base_cube"/>
<xs:complexType name="x_base_cube">
<xs:annotation>
<xs:documentation>
@@ -113,16 +148,16 @@
<xs:complexContent>
<xs:extension base="x_cube">
<xs:sequence>
- <xs:element type="x_measures" name="measures" maxOccurs="1" minOccurs="1" />
- <xs:element type="x_dim_attributes" name="dim_attributes" maxOccurs="1" minOccurs="0" />
- <xs:element type="x_expressions" name="expressions" maxOccurs="1" minOccurs="0" />
- <xs:element type="x_join_chains" name="join_chains" maxOccurs="1" minOccurs="0" />
+ <xs:element type="x_measures" name="measures" maxOccurs="1" minOccurs="1"/>
+ <xs:element type="x_dim_attributes" name="dim_attributes" maxOccurs="1" minOccurs="0"/>
+ <xs:element type="x_expressions" name="expressions" maxOccurs="1" minOccurs="0"/>
+ <xs:element type="x_join_chains" name="join_chains" maxOccurs="1" minOccurs="0"/>
</xs:sequence>
</xs:extension>
</xs:complexContent>
</xs:complexType>
- <xs:element name="x_derived_cube" type="x_derived_cube" />
+ <xs:element name="x_derived_cube" type="x_derived_cube"/>
<xs:complexType name="x_derived_cube">
<xs:annotation>
<xs:documentation>
@@ -134,10 +169,10 @@
<xs:complexContent>
<xs:extension base="x_cube">
<xs:sequence>
- <xs:element type="x_measure_names" name="measure_names" maxOccurs="1" minOccurs="0" />
- <xs:element type="x_dim_attr_names" name="dim_attr_names" maxOccurs="1" minOccurs="0" />
+ <xs:element type="x_measure_names" name="measure_names" maxOccurs="1" minOccurs="0"/>
+ <xs:element type="x_dim_attr_names" name="dim_attr_names" maxOccurs="1" minOccurs="0"/>
</xs:sequence>
- <xs:attribute type="xs:string" name="parent" use="required" />
+ <xs:attribute type="xs:string" name="parent" use="required"/>
</xs:extension>
</xs:complexContent>
</xs:complexType>
@@ -151,13 +186,13 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element type="x_dim_attributes" name="attributes" maxOccurs="1" minOccurs="0" />
- <xs:element type="x_expressions" name="expressions" maxOccurs="1" minOccurs="0" />
- <xs:element type="x_join_chains" name="join_chains" maxOccurs="1" minOccurs="0" />
- <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0" />
+ <xs:element type="x_dim_attributes" name="attributes" maxOccurs="1" minOccurs="0"/>
+ <xs:element type="x_expressions" name="expressions" maxOccurs="1" minOccurs="0"/>
+ <xs:element type="x_join_chains" name="join_chains" maxOccurs="1" minOccurs="0"/>
+ <xs:element type="x_properties" name="properties" maxOccurs="1" minOccurs="0"/>
</xs:sequence>
- <xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="xs:string" name="description" />
+ <xs:attribute type="xs:string" name="name" use="required"/>
+ <xs:attribute type="xs:string" name="description"/>
</xs:complexType>
<xs:complexType name="x_property">
@@ -166,8 +201,8 @@
A key-value pair for storing property's name and its value.
</xs:documentation>
</xs:annotation>
- <xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="xs:string" name="value" use="required" />
+ <xs:attribute type="xs:string" name="name" use="required"/>
+ <xs:attribute type="xs:string" name="value" use="required"/>
</xs:complexType>
<xs:complexType name="x_properties">
@@ -187,72 +222,68 @@
A cube measure.
</xs:documentation>
</xs:annotation>
- <xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="x_measure_type" name="type" use="required" />
- <xs:attribute type="xs:string" name="description" />
- <xs:attribute type="xs:string" name="display_string">
- <xs:annotation>
- <xs:documentation>
- Display string that should be shown to end users. Can be comma separated. It gives UI systems
- built on top to use this value to show the field to end user.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:string" name="default_aggr">
- <xs:annotation>
- <xs:documentation>
- The aggregate for the measure that is used to rollup in aggregated facts.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:string" name="format_string">
- <xs:annotation>
- <xs:documentation>
- The string format that can should be used to format the number shown to user. For example:
- formatting with respect to precision upto 3 decimals, can be specified as
- format_number(%s,"##################.###")
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:dateTime" name="start_time">
- <xs:annotation>
- <xs:documentation>
- The start time from when the measure is available for querying
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:dateTime" name="end_time">
- <xs:annotation>
- <xs:documentation>
- The end time till when the measure is available for querying
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:string" name="unit">
- <xs:annotation>
- <xs:documentation>
- Specify the unit of the measure. For example, currency in dollars or rupees.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:double" name="min">
- <xs:annotation>
- <xs:documentation>
- The minimum value the measure can take. This is only indicative value for user to know what
- vaues it can take. Lens does not do any validation based on this value.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:double" name="max">
- <xs:annotation>
- <xs:documentation>
- The maximum value the measure can take. This is only indicative value for user to know what
- vaues it can take. Lens does not do any validation based on this value.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
+ <xs:complexContent>
+ <xs:extension base="x_field">
+ <xs:attribute type="x_measure_type" name="type" use="required"/>
+ <xs:attribute type="xs:string" name="default_aggr">
+ <xs:annotation>
+ <xs:documentation>
+ The aggregate for the measure that is used to rollup in aggregated facts.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:string" name="format_string">
+ <xs:annotation>
+ <xs:documentation>
+ The string format that can should be used to format the number shown to user. For example:
+ formatting with respect to precision upto 3 decimals, can be specified as
+ format_number(%s,"##################.###")
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:dateTime" name="start_time">
+ <xs:annotation>
+ <xs:documentation>
+ The start time from when the measure is available for querying
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:dateTime" name="end_time">
+ <xs:annotation>
+ <xs:documentation>
+ The end time till when the measure is available for querying
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:string" name="unit">
+ <xs:annotation>
+ <xs:documentation>
+ Specify the unit of the measure. For example, currency in dollars or rupees.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:double" name="min">
+ <xs:annotation>
+ <xs:documentation>
+ The minimum value the measure can take. This is only indicative value for user to know what
+ vaues it can take. Lens does not do any validation based on this value.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:double" name="max">
+ <xs:annotation>
+ <xs:documentation>
+ The maximum value the measure can take. This is only indicative value for user to know what
+ vaues it can take. Lens does not do any validation based on this value.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ </xs:extension>
+ </xs:complexContent>
</xs:complexType>
+ <xs:element name="x_measures" type="x_measures"/>
+
<xs:complexType name="x_measures">
<xs:annotation>
<xs:documentation>
@@ -281,35 +312,29 @@
An expression column
</xs:documentation>
</xs:annotation>
- <xs:sequence>
- <xs:annotation>
- <xs:documentation>
- All the expressions associated with expression column.
- </xs:documentation>
- </xs:annotation>
- <xs:element type="x_expr_spec" name="expr_spec" maxOccurs="unbounded" minOccurs="1"/>
- </xs:sequence>
- <xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="xs:string" name="type" use="required">
- <xs:annotation>
- <xs:documentation>
- The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN, TINYINT,
- SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY, MAP,
- STRUCT, UNION
- See hive represenation for specifying complex types -
- https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:string" name="description" />
- <xs:attribute type="xs:string" name="display_string">
- <xs:annotation>
- <xs:documentation>
- Display string that should be shown to end users. Can be comma separated. It gives UI systems
- built on top to use this value to show the field to end user.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
+ <xs:complexContent>
+ <xs:extension base="x_field">
+ <xs:sequence>
+ <xs:annotation>
+ <xs:documentation>
+ All the expressions associated with expression column.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:element type="x_expr_spec" name="expr_spec" maxOccurs="unbounded" minOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute type="xs:string" name="type" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN, TINYINT,
+ SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY, MAP,
+ STRUCT, UNION
+ See hive represenation for specifying complex types -
+ https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ </xs:extension>
+ </xs:complexContent>
</xs:complexType>
<xs:complexType name="x_expr_spec">
@@ -345,6 +370,8 @@
</xs:attribute>
</xs:complexType>
+ <xs:element name="x_expressions" type="x_expressions"/>
+
<xs:complexType name="x_expressions">
<xs:annotation>
<xs:documentation>
@@ -362,81 +389,78 @@
A dim attribute.
</xs:documentation>
</xs:annotation>
- <xs:sequence>
- <xs:element name="ref_spec" maxOccurs="1" minOccurs="0">
- <xs:annotation>
- <xs:documentation>
- Reference specifiction needs to be specified if the attribute is a reference attribute. It
- can either be table reference or a chained column
-
- ref_spec can be specified as a list of table references to
- which the attribute is refering to.
- For ex : userid refers user.id, xuser.id, yuser.id, zuser.id.
-
- Alternately, ref_spec could be a chained column specifed with chain name and column name.
- </xs:documentation>
- </xs:annotation>
- <xs:complexType>
- <xs:choice maxOccurs="1" minOccurs="1">
- <xs:element type="x_table_references" name="table_references" maxOccurs="1" minOccurs="1" />
- <xs:element type="x_chain_column" name="chain_ref_column" maxOccurs="1" minOccurs="1" />
- </xs:choice>
- </xs:complexType>
- </xs:element>
- </xs:sequence>
- <xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="xs:string" name="type" use="required">
- <xs:annotation>
- <xs:documentation>
- The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN,TINYINT,
- SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY, MAP,
- STRUCT, UNION
- See hive represenation for specifying complex types -
- https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:string" name="description" />
- <xs:attribute type="xs:string" name="display_string">
- <xs:annotation>
- <xs:documentation>
- Display string that should be shown to end users. Can be comma separated. It gives UI systems
- built on top to use this value to show the field to end user.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:dateTime" name="start_time">
- <xs:annotation>
- <xs:documentation>
- The start time from when the attribute is available for querying.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:dateTime" name="end_time">
- <xs:annotation>
- <xs:documentation>
- The end time till when the attribute is available for querying.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:long" name="num_distinct_values" use="optional">
- <xs:annotation>
- <xs:documentation>
- Specifies an indicative value of how many distinct values the dim attribute can take.
- This would give an idea of how big the grouping will be when an attribute is chosen for groupby expressions.
- This is just an approximate value.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
- <xs:attribute type="xs:boolean" name="join_key" default="false">
- <xs:annotation>
- <xs:documentation>
- This flag will tell whether the attribute can be used as a join key or not
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
+ <xs:complexContent>
+ <xs:extension base="x_field">
+ <xs:sequence>
+ <xs:element name="ref_spec" maxOccurs="1" minOccurs="0">
+ <xs:annotation>
+ <xs:documentation>
+ Reference specifiction needs to be specified if the attribute is a reference attribute. It
+ can either be table reference or a chained column
+
+ ref_spec can be specified as a list of table references to
+ which the attribute is refering to.
+ For ex : userid refers user.id, xuser.id, yuser.id, zuser.id.
+
+ Alternately, ref_spec could be a chained column specifed with chain name and column name.
+ </xs:documentation>
+ </xs:annotation>
+ <xs:complexType>
+ <xs:choice maxOccurs="1" minOccurs="1">
+ <xs:element type="x_table_references" name="table_references" maxOccurs="1" minOccurs="1"/>
+ <xs:element type="x_chain_column" name="chain_ref_column" maxOccurs="1" minOccurs="1"/>
+ </xs:choice>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ <xs:attribute type="xs:string" name="type" use="required">
+ <xs:annotation>
+ <xs:documentation>
+ The type indicating what the evaluation of expression will produce. Allowed types are BOOLEAN,TINYINT,
+ SMALLINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, STRING, CHAR, VARCHAR, DATE, TIMESTAMP, BINARY, ARRAY, MAP,
+ STRUCT, UNION
+ See hive represenation for specifying complex types -
+ https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:dateTime" name="start_time">
+ <xs:annotation>
+ <xs:documentation>
+ The start time from when the attribute is available for querying.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:dateTime" name="end_time">
+ <xs:annotation>
+ <xs:documentation>
+ The end time till when the attribute is available for querying.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:long" name="num_distinct_values" use="optional">
+ <xs:annotation>
+ <xs:documentation>
+ Specifies an indicative value of how many distinct values the dim attribute can take.
+ This would give an idea of how big the grouping will be when an attribute is chosen for groupby
+ expressions.
+ This is just an approximate value.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ <xs:attribute type="xs:boolean" name="join_key" default="false">
+ <xs:annotation>
+ <xs:documentation>
+ This flag will tell whether the attribute can be used as a join key or not
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ </xs:extension>
+ </xs:complexContent>
</xs:complexType>
+ <xs:element name="x_dim_attributes" type="x_dim_attributes"/>
+
<xs:complexType name="x_dim_attributes">
<xs:annotation>
<xs:documentation>
@@ -460,8 +484,8 @@
</xs:complexType>
<xs:complexType name="x_chain_column">
- <xs:attribute type="xs:string" name="chain_name" use="required" />
- <xs:attribute type="xs:string" name="ref_col" use="required" />
+ <xs:attribute type="xs:string" name="chain_name" use="required"/>
+ <xs:attribute type="xs:string" name="ref_col" use="required"/>
<xs:attribute type="xs:string" name="dest_table">
<xs:annotation>
<xs:documentation>
@@ -472,8 +496,8 @@
</xs:complexType>
<xs:complexType name="x_table_reference">
- <xs:attribute type="xs:string" name="table" use="required" />
- <xs:attribute type="xs:string" name="column" use="required" />
+ <xs:attribute type="xs:string" name="table" use="required"/>
+ <xs:attribute type="xs:string" name="column" use="required"/>
</xs:complexType>
<xs:complexType name="x_table_references">
@@ -487,6 +511,8 @@
</xs:sequence>
</xs:complexType>
+ <xs:element name="x_join_chains" type="x_join_chains"/>
+
<xs:complexType name="x_join_chains">
<xs:annotation>
<xs:documentation>
@@ -513,19 +539,20 @@
SalesCube.productionStateid -> State.id, State.countryid ->Country.id will be named differently.
</xs:documentation>
</xs:annotation>
- <xs:sequence>
- <xs:element type="x_join_paths" name="paths" maxOccurs="1" minOccurs="1" />
- </xs:sequence>
- <xs:attribute type="xs:string" name="name" use="required" />
- <xs:attribute type="xs:string" name="description" />
- <xs:attribute type="xs:string" name="display_string" />
- <xs:attribute type="xs:string" name="dest_table">
- <xs:annotation>
- <xs:documentation>
- This will be the destination table of the chain. LENS will set and return, need not be set by end-user.
- </xs:documentation>
- </xs:annotation>
- </xs:attribute>
+ <xs:complexContent>
+ <xs:extension base="x_field">
+ <xs:sequence>
+ <xs:element type="x_join_paths" name="paths" maxOccurs="1" minOccurs="1"/>
+ </xs:sequence>
+ <xs:attribute type="xs:string" name="dest_table">
+ <xs:annotation>
+ <xs:documentation>
+ This will be the destination table of the chain. LENS will set and return, need not be set by end-user.
+ </xs:documentation>
+ </xs:annotation>
+ </xs:attribute>
+ </xs:extension>
+ </xs:complexContent>
</xs:complexType>
<xs:complexType name="x_join_path">
@@ -535,7 +562,7 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element type="x_join_edges" name="edges" maxOccurs="1" minOccurs="1" />
+ <xs:element type="x_join_edges" name="edges" maxOccurs="1" minOccurs="1"/>
</xs:sequence>
</xs:complexType>
@@ -557,8 +584,8 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element type="x_table_reference" name="from" maxOccurs="1" minOccurs="1" />
- <xs:element type="x_table_reference" name="to" maxOccurs="1" minOccurs="1" />
+ <xs:element type="x_table_reference" name="from" maxOccurs="1" minOccurs="1"/>
+ <xs:element type="x_table_reference" name="to" maxOccurs="1" minOccurs="1"/>
</xs:sequence>
</xs:complexType>
@@ -596,10 +623,10 @@
<xs:annotation>
<xs:documentation>
Dimension table properties. The properties that should be set are:
- 1. dimtable.{dim_table_name}.part.cols = comma separated list of partition columns of this dimtable.
- This would basically be union of all partition columns of all storage tables of the dimtable.
- Setting this makes that partition column queryable.
- Time part columns can be skipped as they will generally not be queried.
+ 1. dimtable.{dim_table_name}.part.cols = comma separated list of partition columns of this dimtable.
+ This would basically be union of all partition columns of all storage tables of the dimtable.
+ Setting this makes that partition column queryable.
+ Time part columns can be skipped as they will generally not be queried.
</xs:documentation>
</xs:annotation>
</xs:element>
@@ -659,7 +686,7 @@
Column in table.
</xs:documentation>
</xs:annotation>
- <xs:attribute name="name" type="xs:string" use="required" />
+ <xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute type="xs:string" name="type" use="required">
<xs:annotation>
<xs:documentation>
@@ -670,8 +697,8 @@
https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-ComplexTypes
</xs:documentation>
</xs:annotation>
- </xs:attribute>
- <xs:attribute name="comment" type="xs:string" />
+ </xs:attribute>
+ <xs:attribute name="comment" type="xs:string"/>
</xs:complexType>
<xs:simpleType name="x_measure_type">
@@ -681,13 +708,13 @@
</xs:documentation>
</xs:annotation>
<xs:restriction base="xs:string">
- <xs:enumeration value="TINYINT" />
- <xs:enumeration value="SMALLINT" />
- <xs:enumeration value="INT" />
- <xs:enumeration value="BIGINT" />
- <xs:enumeration value="FLOAT" />
- <xs:enumeration value="DOUBLE" />
- <xs:enumeration value="DECIMAL" />
+ <xs:enumeration value="TINYINT"/>
+ <xs:enumeration value="SMALLINT"/>
+ <xs:enumeration value="INT"/>
+ <xs:enumeration value="BIGINT"/>
+ <xs:enumeration value="FLOAT"/>
+ <xs:enumeration value="DOUBLE"/>
+ <xs:enumeration value="DECIMAL"/>
</xs:restriction>
</xs:simpleType>
@@ -698,18 +725,18 @@
</xs:documentation>
</xs:annotation>
<xs:restriction base="xs:string">
- <xs:enumeration value="SECONDLY" />
- <xs:enumeration value="MINUTELY" />
- <xs:enumeration value="HOURLY" />
- <xs:enumeration value="DAILY" />
- <xs:enumeration value="WEEKLY" />
- <xs:enumeration value="MONTHLY" />
- <xs:enumeration value="QUARTERLY" />
- <xs:enumeration value="YEARLY" />
+ <xs:enumeration value="SECONDLY"/>
+ <xs:enumeration value="MINUTELY"/>
+ <xs:enumeration value="HOURLY"/>
+ <xs:enumeration value="DAILY"/>
+ <xs:enumeration value="WEEKLY"/>
+ <xs:enumeration value="MONTHLY"/>
+ <xs:enumeration value="QUARTERLY"/>
+ <xs:enumeration value="YEARLY"/>
</xs:restriction>
</xs:simpleType>
- <xs:element name="x_storage" type="x_storage" />
+ <xs:element name="x_storage" type="x_storage"/>
<xs:complexType name="x_storage">
<xs:annotation>
<xs:documentation>
@@ -725,7 +752,7 @@
</xs:annotation>
</xs:element>
</xs:sequence>
- <xs:attribute name="name" type="xs:string" use="required" />
+ <xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="classname" type="xs:string" use="required">
<xs:annotation>
<xs:documentation>
@@ -738,7 +765,7 @@
</xs:attribute>
</xs:complexType>
- <xs:element name="x_storage_table_desc" type="x_storage_table_desc" />
+ <xs:element name="x_storage_table_desc" type="x_storage_table_desc"/>
<xs:complexType name="x_storage_table_desc">
<xs:annotation>
<xs:documentation>
@@ -907,9 +934,9 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="col_names" type="xs:string" maxOccurs="unbounded" minOccurs="0" />
- <xs:element name="col_values" type="x_skew_col_list" maxOccurs="unbounded" minOccurs="0" />
- <xs:element name="value_location_map" type="x_skewed_value_location" maxOccurs="unbounded" minOccurs="0" />
+ <xs:element name="col_names" type="xs:string" maxOccurs="unbounded" minOccurs="0"/>
+ <xs:element name="col_values" type="x_skew_col_list" maxOccurs="unbounded" minOccurs="0"/>
+ <xs:element name="value_location_map" type="x_skewed_value_location" maxOccurs="unbounded" minOccurs="0"/>
</xs:sequence>
</xs:complexType>
@@ -920,7 +947,7 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="elements" type="xs:string" maxOccurs="unbounded" minOccurs="0" />
+ <xs:element name="elements" type="xs:string" maxOccurs="unbounded" minOccurs="0"/>
</xs:sequence>
</xs:complexType>
@@ -931,12 +958,12 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="value" type="x_skew_col_list" minOccurs="1" maxOccurs="1" />
+ <xs:element name="value" type="x_skew_col_list" minOccurs="1" maxOccurs="1"/>
</xs:sequence>
- <xs:attribute type="xs:string" name="location" use="required" />
+ <xs:attribute type="xs:string" name="location" use="required"/>
</xs:complexType>
- <xs:element name="x_storage_table_element" type="x_storage_table_element" />
+ <xs:element name="x_storage_table_element" type="x_storage_table_element"/>
<xs:complexType name="x_storage_table_element">
<xs:annotation>
@@ -945,9 +972,9 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="update_periods" type="x_update_periods" maxOccurs="1" minOccurs="0" />
- <xs:element name="storage_name" type="xs:string" />
- <xs:element type="x_storage_table_desc" name="table_desc" />
+ <xs:element name="update_periods" type="x_update_periods" maxOccurs="1" minOccurs="0"/>
+ <xs:element name="storage_name" type="xs:string"/>
+ <xs:element type="x_storage_table_desc" name="table_desc"/>
</xs:sequence>
</xs:complexType>
@@ -958,11 +985,11 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="storage_table" minOccurs="1" maxOccurs="unbounded" type="x_storage_table_element" />
+ <xs:element name="storage_table" minOccurs="1" maxOccurs="unbounded" type="x_storage_table_element"/>
</xs:sequence>
</xs:complexType>
- <xs:element name="x_partition_list" type="x_partition_list" />
+ <xs:element name="x_partition_list" type="x_partition_list"/>
<xs:complexType name="x_partition_list">
<xs:annotation>
<xs:documentation>
@@ -970,11 +997,11 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="partition" type="x_partition" maxOccurs="unbounded" minOccurs="0" />
+ <xs:element name="partition" type="x_partition" maxOccurs="unbounded" minOccurs="0"/>
</xs:sequence>
</xs:complexType>
- <xs:element name="x_partition" type="x_partition" />
+ <xs:element name="x_partition" type="x_partition"/>
<xs:complexType name="x_partition">
<xs:annotation>
@@ -983,21 +1010,21 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="non_time_partition_spec" maxOccurs="1" minOccurs="0" type ="x_part_spec" >
+ <xs:element name="non_time_partition_spec" maxOccurs="1" minOccurs="0" type="x_part_spec">
<xs:annotation>
<xs:documentation>
Non time partition specification with partition columns and values.
</xs:documentation>
</xs:annotation>
</xs:element>
- <xs:element name="time_partition_spec" maxOccurs="1" minOccurs="0" type = "x_time_part_spec" >
+ <xs:element name="time_partition_spec" maxOccurs="1" minOccurs="0" type="x_time_part_spec">
<xs:annotation>
<xs:documentation>
Time partition specification with partition columns and date values.
</xs:documentation>
</xs:annotation>
</xs:element>
- <xs:element name="full_partition_spec" maxOccurs="1" minOccurs="0" type ="x_part_spec" >
+ <xs:element name="full_partition_spec" maxOccurs="1" minOccurs="0" type="x_part_spec">
<xs:annotation>
<xs:documentation>
LENS would use this type to retun the specification elements. Should not be used directly by end-user.
@@ -1078,8 +1105,8 @@
Partition column name and its value.
</xs:documentation>
</xs:annotation>
- <xs:attribute type="xs:string" name="key" use="required" />
- <xs:attribute type="xs:string" name="value" use="required" />
+ <xs:attribute type="xs:string" name="key" use="required"/>
+ <xs:attribute type="xs:string" name="value" use="required"/>
</xs:complexType>
<xs:complexType name="x_part_spec">
@@ -1094,8 +1121,8 @@
Time partition column name and its value as date-time.
</xs:documentation>
</xs:annotation>
- <xs:attribute type="xs:string" name="key" use="required" />
- <xs:attribute type="xs:dateTime" name="value" use="required" />
+ <xs:attribute type="xs:string" name="key" use="required"/>
+ <xs:attribute type="xs:dateTime" name="value" use="required"/>
</xs:complexType>
<xs:complexType name="x_time_part_spec">
@@ -1104,7 +1131,7 @@
</xs:sequence>
</xs:complexType>
- <xs:element name="x_fact_table" type="x_fact_table" />
+ <xs:element name="x_fact_table" type="x_fact_table"/>
<xs:complexType name="x_fact_table">
<xs:annotation>
@@ -1127,26 +1154,26 @@
Properties that can be set for a fact are
1. cube.fact.{fact-name}.valid.columns.size : This should be set to number of
- cube.fact.{fact-name}.valid.columns strings.
+ cube.fact.{fact-name}.valid.columns strings.
2. cube.fact.{fact-name}.valid.columns{i} : This should be set to comma separated column names which are
- valid in the fact, such that 0 <= i < cube.fact.{fact-name}.valid.columns.size and each of
- cube.fact.{fact-name}.valid.columns{i} has maximum length of 3999.
+ valid in the fact, such that 0 <= i < cube.fact.{fact-name}.valid.columns.size and each of
+ cube.fact.{fact-name}.valid.columns{i} has maximum length of 3999.
3. cube.fact.is.aggregated : Defaults to true. If the fact is a raw fact, this should be set to false,
- otherwise true.
+ otherwise true.
4. cube.timedim.relation.{time_dim1}: See the same property in cube. Fact tables can override the property.
5. cube.fact.absolute.start.time: start time of the fact. For queries that ask for time before this,
- this fact is not a candidate. Time format can be as you would specify in the time_range_in clause.
- i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]]
+ this fact is not a candidate. Time format can be as you would specify in the time_range_in clause.
+ i.e. yyyy[-mm[-dd[-hh[:MM[:ss[,SSS]]]]]]
6. cube.fact.relative.start.time: Here you can specify fact's relative validity relative to current time.
- Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just
- a time difference e.g. "-90 days". Or can be specified in relative syntax.
- e.g. now.year or now.day - 6 hour etc.
+ Useful if you want to specify e.g. this fact is valid for today - 90 days. Can be specified as just
+ a time difference e.g. "-90 days". Or can be specified in relative syntax.
+ e.g. now.year or now.day - 6 hour etc.
7. cube.fact.absolute.end.time: If you're deprecating this fact, put the final date till which the data of
- the fact will be valid here. Format same as absolute start time.
+ the fact will be valid here. Format same as absolute start time.
</xs:documentation>
</xs:annotation>
</xs:element>
- <xs:element name="storage_tables" type = "x_storage_tables" maxOccurs="1" minOccurs="0"/>
+ <xs:element name="storage_tables" type="x_storage_tables" maxOccurs="1" minOccurs="0"/>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required">
<xs:annotation>
@@ -1177,7 +1204,7 @@
</xs:attribute>
</xs:complexType>
- <xs:element name="x_native_table" type="x_native_table" />
+ <xs:element name="x_native_table" type="x_native_table"/>
<xs:complexType name="x_native_table">
<xs:annotation>
<xs:documentation>
@@ -1246,7 +1273,7 @@
</xs:attribute>
</xs:complexType>
- <xs:element name="x_flattened_columns" type="x_flattened_columns" />
+ <xs:element name="x_flattened_columns" type="x_flattened_columns"/>
<xs:complexType name="x_flattened_columns">
<xs:annotation>
<xs:documentation>
@@ -1255,16 +1282,16 @@
</xs:documentation>
</xs:annotation>
<xs:sequence>
- <xs:element name="flattened_column" type="x_flattened_column" maxOccurs="unbounded" minOccurs="0" />
+ <xs:element name="flattened_column" type="x_flattened_column" maxOccurs="unbounded" minOccurs="0"/>
</xs:sequence>
</xs:complexType>
<xs:complexType name="x_flattened_column">
<xs:choice maxOccurs="1" minOccurs="1">
- <xs:element name="measure" type="x_measure" />
- <xs:element name="expression" type="x_expr_column" />
- <xs:element name="dim_attribute" type="x_dim_attribute" />
+ <xs:element name="measure" type="x_measure"/>
+ <xs:element name="expression" type="x_expr_column"/>
+ <xs:element name="dim_attribute" type="x_dim_attribute"/>
</xs:choice>
- <xs:attribute name="table_name" type="xs:string" use="required" />
+ <xs:attribute name="table_name" type="xs:string" use="required"/>
<xs:attribute name="chain_name" type="xs:string">
<xs:annotation>
<xs:documentation>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java
deleted file mode 100644
index 208081d..0000000
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.cli.commands;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.lens.api.APIResult;
-import org.apache.lens.api.metastore.XPartition;
-import org.apache.lens.api.metastore.XStorageTableElement;
-
-import com.google.common.base.Joiner;
-
-public abstract class LensCRUDStoragePartitionCommand<T> extends LensCRUDCommand<T> {
- public String showAll(String filter) {
- List<String> all = getAll(filter);
- if (all == null || all.isEmpty()) {
- return "No " + getSingleObjectName() + " found" + (filter == null ? "" : " for " + filter);
- }
- return Joiner.on("\n").join(all);
- }
-
- public String showAllStorages(String tableName) {
- String sep = "";
- StringBuilder sb = new StringBuilder();
- List<String> storages = getAllStorages(tableName);
- if (storages != null) {
- for (String storage : storages) {
- if (!storage.isEmpty()) {
- sb.append(sep).append(storage);
- sep = "\n";
- }
- }
- }
- String ret = sb.toString();
- return ret.isEmpty() ? "No storage found for " + tableName : ret;
- }
-
- public String addStorage(String tableName, String path) {
- return doAddStorage(tableName, getValidPath(path, false, true)).toString().toLowerCase();
- }
-
- public String getStorage(String tableName, String storage) {
- try {
- return formatJson(mapper.writer(pp).writeValueAsString(readStorage(tableName, storage)));
- } catch (IOException e) {
- throw new IllegalArgumentException(e);
- }
- }
-
- public String dropStorage(String tableName, String storageName) {
- return doDropStorage(tableName, storageName).toString().toLowerCase();
- }
-
- public String dropAllStorages(String tableName) {
- return doDropAllStorages(tableName).toString();
- }
-
- public String getAllPartitions(String tableName, String storageName, String filter) {
- try {
- return formatJson(mapper.writer(pp).writeValueAsString(readAllPartitions(tableName, storageName, filter)));
- } catch (IOException e) {
- throw new IllegalArgumentException(e);
- }
- }
-
- public String addPartition(String tableName, String storageName, String path) {
- return doAddPartition(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
- }
-
- public String addPartitions(String tableName, String storageName, String path) {
- return doAddPartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
- }
-
- public String dropPartitions(String tableName, String storageName, String filter) {
- return doDropPartitions(tableName, storageName, filter).toString().toLowerCase();
- }
-
- protected abstract List<String> getAll(String filter);
-
- public abstract List<String> getAllStorages(String name);
-
- public abstract APIResult doAddStorage(String name, String path);
-
- protected abstract XStorageTableElement readStorage(String tableName, String storage);
-
- public abstract APIResult doDropStorage(String tableName, String storageName);
-
- public abstract APIResult doDropAllStorages(String name);
-
- protected abstract List<XPartition> readAllPartitions(String tableName, String storageName, String filter);
-
- protected abstract APIResult doAddPartition(String tableName, String storageName, String path);
-
- protected abstract APIResult doAddPartitions(String tableName, String storageName, String path);
-
- protected abstract APIResult doDropPartitions(String tableName, String storageName, String filter);
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCubeCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCubeCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCubeCommands.java
index d05d7a5..6ba702f 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCubeCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCubeCommands.java
@@ -35,7 +35,7 @@ import org.springframework.stereotype.Component;
@Component
@UserDocumentation(title = "OLAP Data cube metadata management",
description = "These commands provide CRUD for cubes")
-public class LensCubeCommands extends LensCRUDCommand<XCube> {
+public class LensCubeCommands extends LogicalTableCrudCommand<XCube> {
/**
* Show cubes.
@@ -106,12 +106,29 @@ public class LensCubeCommands extends LensCRUDCommand<XCube> {
help = "get latest date of data available in cube <cube_name> for time dimension <time_dimension_name>. "
+ " Instead of time dimension, partition column can be directly passed as <time_dimension>")
public String getLatest(
- @CliOption(key = {"", "cube"}, mandatory = true, help = "<cube_name>") String cube,
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<cube_name>") String cube,
@CliOption(key = {"", "time_dimension"}, mandatory = true, help = "<time_dimension>") String timeDim) {
Date dt = getClient().getLatestDateOfCube(cube, timeDim);
return dt == null ? "No Data Available" : formatDate(dt);
}
+ @CliCommand(value = "cube show fields",
+ help = "Show queryable fields of the given cube <cube_name>. "
+ + "Optionally specify <flattened> to include chained fields")
+ public String showQueryableFields(
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<cube_name>") String table,
+ @CliOption(key = {"flattened"}, mandatory = false, unspecifiedDefaultValue = "false",
+ specifiedDefaultValue = "true", help = "<flattened>") boolean flattened) {
+ return getAllFields(table, flattened);
+ }
+
+ @CliCommand(value = "cube show joinchains",
+ help = "Show joinchains of the given cube <cube_name>. ")
+ public String showJoinChains(
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<cube_name>") String table) {
+ return getAllJoinChains(table);
+ }
+
@Override
public List<String> getAll() {
return getClient().getAllCubes();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionCommands.java
index 84ae6c3..de022c1 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionCommands.java
@@ -34,7 +34,7 @@ import org.springframework.stereotype.Component;
*/
@Component
@UserDocumentation(title = "Dimension Management", description = "These commands provide CRUD for Dimensions")
-public class LensDimensionCommands extends LensCRUDCommand<XDimension> {
+public class LensDimensionCommands extends LogicalTableCrudCommand<XDimension> {
/**
* Show dimensions.
@@ -103,6 +103,23 @@ public class LensDimensionCommands extends LensCRUDCommand<XDimension> {
return drop(name, false);
}
+ @CliCommand(value = "dimension show fields",
+ help = "Show queryable fields of the given dimension <dimension_name>. "
+ + "Optionally specify <flattened> to include chained fields")
+ public String showQueryableFields(
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<dimension_name>") String table,
+ @CliOption(key = {"flattened"}, mandatory = false, unspecifiedDefaultValue = "false",
+ specifiedDefaultValue = "true", help = "<flattened>") boolean flattened) {
+ return getAllFields(table, flattened);
+ }
+
+ @CliCommand(value = "dimension show joinchains",
+ help = "Show joinchains of the given dimension <dimension_name>. ")
+ public String showJoinChains(
+ @CliOption(key = {"", "name"}, mandatory = true, help = "<dimension_name>") String table) {
+ return getAllJoinChains(table);
+ }
+
@Override
public List<String> getAll() {
return getClient().getAllDimensions();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
index 40380b7..6a93393 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
@@ -37,7 +37,7 @@ import org.springframework.stereotype.Component;
@Component
@UserDocumentation(title = "Commands for Dimension Tables",
description = "These commands provide CRUD for dimension tables, associated storages, and fact partitions")
-public class LensDimensionTableCommands extends LensCRUDStoragePartitionCommand<XDimensionTable>
+public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimensionTable>
implements CommandMarker {
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
index 24992b9..bdb9c38 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
@@ -36,7 +36,7 @@ import org.springframework.stereotype.Component;
@Component
@UserDocumentation(title = "Management of Facts",
description = "These command provide CRUD for facts, associated storages, and fact partitions")
-public class LensFactCommands extends LensCRUDStoragePartitionCommand<XFactTable> {
+public class LensFactCommands extends PhysicalTableCrudCommand<XFactTable> {
/**
* Show facts.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 1eb7ed6..928531e 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -271,7 +271,7 @@ public class LensQueryCommands extends BaseLensCommand {
@CliOption(key = {"", "query_handle"}, mandatory = true, help = "<query_handle>") String qh,
@CliOption(key = {"save_location"}, mandatory = false, help = "<save_location>") String location,
@CliOption(key = {"async"}, mandatory = false, unspecifiedDefaultValue = "true",
- help = "<async>") boolean async) {
+ help = "<async>") boolean async) {
QueryHandle queryHandle = new QueryHandle(UUID.fromString(qh));
LensClient.LensClientResultSetWithStats results;
try {
@@ -459,5 +459,4 @@ public class LensQueryCommands extends BaseLensCommand {
planStr.append("\n").append("Prepare handle:").append(plan.getPrepareHandle());
return planStr.toString();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
new file mode 100644
index 0000000..3c78e43
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LogicalTableCrudCommand.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.commands;
+
+import org.apache.lens.cli.table.XFlattenedColumnTable;
+import org.apache.lens.cli.table.XJoinChainTable;
+
+public abstract class LogicalTableCrudCommand<T> extends LensCRUDCommand<T> {
+ public String getAllFields(String table, boolean flattened) {
+ return new XFlattenedColumnTable(getClient().getQueryableFields(table, flattened), table).toString();
+ }
+ public String getAllJoinChains(String table) {
+ return new XJoinChainTable(getClient().getJoinChains(table)).toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
new file mode 100644
index 0000000..a479c14
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.commands;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.lens.api.APIResult;
+import org.apache.lens.api.metastore.XPartition;
+import org.apache.lens.api.metastore.XStorageTableElement;
+
+import com.google.common.base.Joiner;
+
+public abstract class PhysicalTableCrudCommand<T> extends LensCRUDCommand<T> {
+ public String showAll(String filter) {
+ List<String> all = getAll(filter);
+ if (all == null || all.isEmpty()) {
+ return "No " + getSingleObjectName() + " found" + (filter == null ? "" : " for " + filter);
+ }
+ return Joiner.on("\n").join(all);
+ }
+
+ public String showAllStorages(String tableName) {
+ String sep = "";
+ StringBuilder sb = new StringBuilder();
+ List<String> storages = getAllStorages(tableName);
+ if (storages != null) {
+ for (String storage : storages) {
+ if (!storage.isEmpty()) {
+ sb.append(sep).append(storage);
+ sep = "\n";
+ }
+ }
+ }
+ String ret = sb.toString();
+ return ret.isEmpty() ? "No storage found for " + tableName : ret;
+ }
+
+ public String addStorage(String tableName, String path) {
+ return doAddStorage(tableName, getValidPath(path, false, true)).toString().toLowerCase();
+ }
+
+ public String getStorage(String tableName, String storage) {
+ try {
+ return formatJson(mapper.writer(pp).writeValueAsString(readStorage(tableName, storage)));
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ public String dropStorage(String tableName, String storageName) {
+ return doDropStorage(tableName, storageName).toString().toLowerCase();
+ }
+
+ public String dropAllStorages(String tableName) {
+ return doDropAllStorages(tableName).toString();
+ }
+
+ public String getAllPartitions(String tableName, String storageName, String filter) {
+ try {
+ return formatJson(mapper.writer(pp).writeValueAsString(readAllPartitions(tableName, storageName, filter)));
+ } catch (IOException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
+ public String addPartition(String tableName, String storageName, String path) {
+ return doAddPartition(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
+ }
+
+ public String addPartitions(String tableName, String storageName, String path) {
+ return doAddPartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
+ }
+
+ public String dropPartitions(String tableName, String storageName, String filter) {
+ return doDropPartitions(tableName, storageName, filter).toString().toLowerCase();
+ }
+
+ protected abstract List<String> getAll(String filter);
+
+ public abstract List<String> getAllStorages(String name);
+
+ public abstract APIResult doAddStorage(String name, String path);
+
+ protected abstract XStorageTableElement readStorage(String tableName, String storage);
+
+ public abstract APIResult doDropStorage(String tableName, String storageName);
+
+ public abstract APIResult doDropAllStorages(String name);
+
+ protected abstract List<XPartition> readAllPartitions(String tableName, String storageName, String filter);
+
+ protected abstract APIResult doAddPartition(String tableName, String storageName, String path);
+
+ protected abstract APIResult doAddPartitions(String tableName, String storageName, String path);
+
+ protected abstract APIResult doDropPartitions(String tableName, String storageName, String filter);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
index 86099d3..f577509 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensBanner.java
@@ -47,4 +47,9 @@ public class LensBanner extends DefaultBannerProvider {
public String getWelcomeMessage() {
return "Welcome to Lens Client";
}
+
+ @Override
+ public String getVersion() {
+ return getClass().getPackage().getImplementationVersion() + " built with spring shell " + super.getVersion();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/skel/LensHistoryFileProvider.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensHistoryFileProvider.java b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensHistoryFileProvider.java
index b599d6e..905a019 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensHistoryFileProvider.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensHistoryFileProvider.java
@@ -40,7 +40,7 @@ public class LensHistoryFileProvider extends DefaultHistoryFileNameProvider {
* @see org.springframework.shell.plugin.support.DefaultHistoryFileNameProvider#name()
*/
@Override
- public String name() {
+ public String getProviderName() {
return "lens client history provider";
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/skel/LensPromptProvider.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensPromptProvider.java b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensPromptProvider.java
index e5524bd..3729bc7 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/skel/LensPromptProvider.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/skel/LensPromptProvider.java
@@ -41,7 +41,7 @@ public class LensPromptProvider extends DefaultPromptProvider {
* @see org.springframework.shell.plugin.support.DefaultPromptProvider#name()
*/
@Override
- public String name() {
+ public String getProviderName() {
return "lens prompt provider";
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTable.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTable.java b/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTable.java
new file mode 100644
index 0000000..2f46c3b
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTable.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.table;
+
+
+import java.util.Collection;
+
+import org.springframework.shell.support.table.Table;
+import org.springframework.shell.support.table.TableHeader;
+import org.springframework.shell.support.table.TableRenderer;
+
+import lombok.Data;
+
+@Data
+public class CollectionTable<T> {
+ private final Collection<T> collection;
+ private final RowProvider<T> provider;
+ private final String[] header;
+
+ public CollectionTable(Collection<T> collection, RowProvider<T> provider, String... header) {
+ this.collection = collection;
+ this.provider = provider;
+ this.header = header;
+ }
+
+ interface RowProvider<T> {
+ String[][] getRows(T element);
+ }
+
+ @Override
+ public String toString() {
+ Table table = new Table();
+ for (int i = 0; i < header.length; i++) {
+ table.addHeader(i + 1, new TableHeader(header[i]));
+ }
+ for (T element : collection) {
+ for (String[] row : provider.getRows(element)) {
+ table.addRow(row);
+ }
+ }
+ return TableRenderer.renderTextTable(table);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTableFactory.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTableFactory.java b/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTableFactory.java
new file mode 100644
index 0000000..bcc44a6
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/table/CollectionTableFactory.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.table;
+
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.lens.api.metastore.*;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+public class CollectionTableFactory {
+ private CollectionTableFactory() {}
+
+ public static CollectionTable<XFlattenedColumn> getCollectionTable(Class<? extends XField> claz, final String table) {
+ if (claz == XExprColumn.class) {
+ return new CollectionTable<>(Sets.newTreeSet(new Comparator<XFlattenedColumn>() {
+ @Override
+ public int compare(XFlattenedColumn o1, XFlattenedColumn o2) {
+ return o1.getExpression().getName().compareTo(o2.getExpression().getName());
+ }
+ }),
+ new CollectionTable.RowProvider<XFlattenedColumn>() {
+ @Override
+ public String[][] getRows(XFlattenedColumn element) {
+ return new String[][]{
+ {
+ nulltoBlank(element.getExpression().getName()),
+ nulltoBlank(element.getExpression().getDisplayString()),
+ nulltoBlank(element.getExpression().getDescription()),
+ expressionsAsString(element.getExpression().getExprSpec()),
+ },
+ };
+ }
+
+ private String expressionsAsString(List<XExprSpec> exprSpec) {
+ StringBuilder sb = new StringBuilder();
+ String sep = "";
+ for (XExprSpec spec : exprSpec) {
+ sb.append(sep);
+ sep = ", ";
+ List<String> clauses = Lists.newArrayList();
+ if (spec.getStartTime() != null) {
+ clauses.add("after " + spec.getStartTime());
+ }
+ if (spec.getEndTime() != null) {
+ clauses.add("before " + spec.getEndTime());
+ }
+ String sep1 = "";
+ if (clauses.isEmpty()) {
+ clauses.add("always valid");
+ }
+ for (String clause : clauses) {
+ sb.append(sep1).append(clause);
+ sep1 = " and ";
+ }
+ sb.append(": ");
+ sb.append(spec.getExpr());
+ }
+ return sb.toString();
+ }
+ }, "Name", "Display String", "Description", "Expr Specs");
+ } else if (claz == XDimAttribute.class) {
+ return new CollectionTable<>(Sets.newTreeSet(new Comparator<XFlattenedColumn>() {
+ @Override
+ public int compare(XFlattenedColumn o1, XFlattenedColumn o2) {
+ if (o1 == null || o1.getDimAttribute() == null) {
+ return -1;
+ } else if (o2 == null || o2.getDimAttribute() == null) {
+ return 1;
+ } else if (table.equals(o1.getTableName()) && !table.equals(o2.getTableName())) {
+ return -1;
+ } else if (table.equals(o2.getTableName()) && !table.equals(o1.getTableName())) {
+ return 1;
+ } else {
+ if (o1.getTableName() == null) {
+ o1.setTableName("");
+ }
+ if (o2.getTableName() == null) {
+ o2.setTableName("");
+ }
+ if (o1.getChainName() == null) {
+ o1.setChainName("");
+ }
+ if (o2.getChainName() == null) {
+ o2.setChainName("");
+ }
+ int cmp = o1.getTableName().compareTo(o2.getTableName());
+ if (cmp != 0) {
+ return cmp;
+ }
+ cmp = o1.getChainName().compareTo(o2.getChainName());
+ if (cmp != 0) {
+ return cmp;
+ }
+ return o1.getDimAttribute().getName().compareTo(o2.getDimAttribute().getName());
+ }
+ }
+ }),
+ new CollectionTable.RowProvider<XFlattenedColumn>() {
+ @Override
+ public String[][] getRows(XFlattenedColumn element) {
+ String prefix = XFlattenedColumnTable.firstNonNull(element.getChainName(), element.getTableName());
+ return new String[][]{
+ {
+ (prefix == null || prefix.isEmpty() || prefix.equalsIgnoreCase(table) ? "" : (prefix + "."))
+ + nulltoBlank(element.getDimAttribute().getName()),
+ nulltoBlank(element.getDimAttribute().getDisplayString()),
+ nulltoBlank(element.getDimAttribute().getDescription()),
+ },
+ };
+ }
+ }, "Name", "Display String", "Description");
+ } else if (claz == XMeasure.class) {
+ return new CollectionTable<>(Sets.newTreeSet(new Comparator<XFlattenedColumn>() {
+ @Override
+ public int compare(XFlattenedColumn o1, XFlattenedColumn o2) {
+ return o1.getMeasure().getName().compareTo(o2.getMeasure().getName());
+ }
+ }),
+ new CollectionTable.RowProvider<XFlattenedColumn>() {
+ @Override
+ public String[][] getRows(XFlattenedColumn element) {
+ return new String[][]{
+ {
+ nulltoBlank(element.getMeasure().getName()),
+ nulltoBlank(element.getMeasure().getDisplayString()),
+ nulltoBlank(element.getMeasure().getDescription()),
+ },
+ };
+ }
+ }, "Name", "Display String", "Description");
+ } else {
+ return null;
+ }
+ }
+
+ public static String nulltoBlank(String s) {
+ return s == null ? "" : s;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/table/XFlattenedColumnTable.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/table/XFlattenedColumnTable.java b/lens-cli/src/main/java/org/apache/lens/cli/table/XFlattenedColumnTable.java
new file mode 100644
index 0000000..d4208c1
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/table/XFlattenedColumnTable.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.table;
+
+
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.bind.annotation.XmlType;
+
+import org.apache.lens.api.metastore.XField;
+import org.apache.lens.api.metastore.XFlattenedColumn;
+import org.apache.lens.api.metastore.XFlattenedColumns;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+public class XFlattenedColumnTable {
+ private final String table;
+ Map<Class<? extends XField>, CollectionTable<XFlattenedColumn>> tables = Maps.newLinkedHashMap();
+ List<String> chainNames = Lists.newArrayList();
+ List<String> tableNames = Lists.newArrayList();
+
+ public XFlattenedColumnTable(XFlattenedColumns flattenedColumns, String table) {
+ this.table = table;
+ for (XFlattenedColumn column : flattenedColumns.getFlattenedColumn()) {
+ XField field = firstNonNull(column.getDimAttribute(), column.getMeasure(), column.getExpression());
+ if (field != null) {
+ if (!tables.containsKey(field.getClass())) {
+ tables.put(field.getClass(), CollectionTableFactory.getCollectionTable(field.getClass(), table));
+ }
+ tables.get(field.getClass()).getCollection().add(column);
+ } else {
+ if (column.getChainName() != null) {
+ chainNames.add(column.getChainName());
+ }
+ if (column.getTableName() != null) {
+ tableNames.add(column.getTableName());
+ }
+ }
+ }
+ }
+
+ public static <T> T firstNonNull(T... args) {
+ for (T arg : args) {
+ if (arg != null) {
+ return arg;
+ }
+ }
+ return null;
+ }
+
+
+ @Override
+ public String toString() {
+ String sep = "=============================";
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<Class<? extends XField>, CollectionTable<XFlattenedColumn>> entry : tables.entrySet()) {
+ String title =
+ entry.getKey().getAnnotation(XmlType.class).name().replaceAll("^x_", "").replaceAll("_", " ") + "s";
+ sb.append(title).append("\n").append(sep).append("\n").append(entry.getValue()).append("\n");
+ }
+ String sep1 = "";
+ if (!chainNames.isEmpty()) {
+ sb.append("Accessible Join Chains\n").append(sep).append("\n");
+ for (String chain : chainNames) {
+ sb.append(sep1).append(chain);
+ sep1 = "\n";
+ }
+ }
+ sep1 = "";
+ if (!tableNames.isEmpty()) {
+ sb.append("Accessible Tables\n").append(sep).append("\n");
+ for (String table : tableNames) {
+ sb.append(sep1).append(table);
+ sep1 = "\n";
+ }
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/main/java/org/apache/lens/cli/table/XJoinChainTable.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/table/XJoinChainTable.java b/lens-cli/src/main/java/org/apache/lens/cli/table/XJoinChainTable.java
new file mode 100644
index 0000000..8082215
--- /dev/null
+++ b/lens-cli/src/main/java/org/apache/lens/cli/table/XJoinChainTable.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cli.table;
+
+
+import static org.apache.lens.cli.table.CollectionTableFactory.nulltoBlank;
+
+import org.apache.lens.api.metastore.*;
+
+public class XJoinChainTable {
+
+ private XJoinChains xJoinChains;
+
+ public XJoinChainTable(XJoinChains xJoinChains) {
+ this.xJoinChains = xJoinChains;
+ }
+
+ @Override
+ public String toString() {
+ return new CollectionTable<>(xJoinChains.getJoinChain(), new CollectionTable.RowProvider<XJoinChain>() {
+ @Override
+ public String[][] getRows(XJoinChain element) {
+ int size = element.getPaths().getPath().size();
+ String[][] ret = new String[size][5];
+ for (int i = 0; i < size; i++) {
+ if (i == 0) {
+ ret[i][0] = nulltoBlank(element.getName());
+ ret[i][1] = nulltoBlank(element.getDisplayString());
+ ret[i][2] = nulltoBlank(element.getDescription());
+ ret[i][3] = nulltoBlank(element.getDestTable());
+ } else {
+ ret[i][0] = "";
+ ret[i][1] = "";
+ ret[i][2] = "";
+ ret[i][3] = "";
+ }
+ ret[i][4] = pathAsString(element.getPaths().getPath().get(i));
+ }
+ return ret;
+ }
+
+ private String pathAsString(XJoinPath path) {
+ StringBuilder sb = new StringBuilder();
+ String sep1 = "";
+ for (XJoinEdge edge : path.getEdges().getEdge()) {
+ sb.append(sep1)
+ .append(edge.getFrom().getTable()).append(".").append(edge.getFrom().getColumn())
+ .append("=")
+ .append(edge.getTo().getTable()).append(".").append(edge.getTo().getColumn());
+ sep1 = "->";
+ }
+ return sb.toString();
+ }
+ }
+
+ , "Name", "Display String", "Description", "Destination Table", "Path").
+
+ toString();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ff31ad96/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
index ae39a2a..73661e1 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensCubeCommands.java
@@ -18,15 +18,19 @@
*/
package org.apache.lens.cli;
+import static org.testng.Assert.*;
+
import java.io.*;
import java.net.URL;
+import java.util.Arrays;
+import org.apache.lens.api.metastore.XJoinChains;
import org.apache.lens.cli.commands.LensCubeCommands;
+import org.apache.lens.cli.table.XJoinChainTable;
import org.apache.lens.client.LensClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testng.Assert;
import org.testng.annotations.Test;
/**
@@ -51,22 +55,38 @@ public class TestLensCubeCommands extends LensCliApplicationTest {
LOG.debug("Starting to test cube commands");
URL cubeSpec = TestLensCubeCommands.class.getClassLoader().getResource("sample-cube.xml");
String cubeList = command.showCubes();
- Assert.assertFalse(cubeList.contains("sample_cube"));
+ assertFalse(cubeList.contains("sample_cube"));
command.createCube(new File(cubeSpec.toURI()).getAbsolutePath());
cubeList = command.showCubes();
- Assert.assertEquals(command.getLatest("sample_cube", "dt"), "No Data Available");
- Assert.assertTrue(cubeList.contains("sample_cube"));
-
+ assertEquals(command.getLatest("sample_cube", "dt"), "No Data Available");
+ assertTrue(cubeList.contains("sample_cube"));
+ testJoinChains(command);
+ testFields(command);
testUpdateCommand(new File(cubeSpec.toURI()), command);
command.dropCube("sample_cube");
try {
command.getLatest("sample_cube", "dt");
- Assert.fail("should have failed as cube doesn't exist");
+ fail("should have failed as cube doesn't exist");
} catch (Exception e) {
//pass
}
cubeList = command.showCubes();
- Assert.assertFalse(cubeList.contains("sample_cube"));
+ assertFalse(cubeList.contains("sample_cube"));
+ }
+
+ private void testJoinChains(LensCubeCommands command) {
+ String joinChains = command.showJoinChains("sample_cube");
+ assertEquals(joinChains, new XJoinChainTable(new XJoinChains()).toString());
+ }
+
+ private void testFields(LensCubeCommands command) {
+ String fields = command.showQueryableFields("sample_cube", true);
+ for (String field : Arrays
+ .asList("dim1", "dim2", "dim3", "measure1", "measure2", "measure3", "measure4", "expr_msr5")) {
+ assertTrue(fields.contains(field));
+ }
+ assertTrue(fields.contains("measure3 + measure4 + 0.01"));
+ assertTrue(fields.replace("measure3 + measure4 + 0.01", "blah").contains("measure3 + measure4"));
}
/**
@@ -105,13 +125,13 @@ public class TestLensCubeCommands extends LensCliApplicationTest {
String propString = "name : sample_cube.prop value : sample";
String propString1 = "name : sample_cube.prop1 value : sample1";
- Assert.assertTrue(desc.contains(propString));
+ assertTrue(desc.contains(propString));
command.updateCube("sample_cube", "/tmp/sample_cube1.xml");
desc = command.describeCube("sample_cube");
LOG.debug(desc);
- Assert.assertTrue(desc.contains(propString));
- Assert.assertTrue(desc.contains(propString1));
+ assertTrue(desc.contains(propString));
+ assertTrue(desc.contains(propString1));
} finally {
newFile.delete();
}
[36/51] [abbrv] incubator-lens git commit: LENS-617:lens-config.sh is
ignoring ${LENSCPPATH}
Posted by jd...@apache.org.
LENS-617:lens-config.sh is ignoring ${LENSCPPATH}
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/1fde486a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/1fde486a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/1fde486a
Branch: refs/heads/current-release-line
Commit: 1fde486a57e4fa0a85fe7a32a07e711a4a60b0bc
Parents: ead42f8
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jun 18 15:14:06 2015 +0530
Committer: arshad-matin <ar...@apache.org>
Committed: Thu Jun 18 15:14:06 2015 +0530
----------------------------------------------------------------------
tools/scripts/lens-config.sh | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/1fde486a/tools/scripts/lens-config.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-config.sh b/tools/scripts/lens-config.sh
index 4470132..b901fbb 100644
--- a/tools/scripts/lens-config.sh
+++ b/tools/scripts/lens-config.sh
@@ -85,9 +85,9 @@ case $type in
fi
LENS_OPTS="$LENS_OPTS $LENS_CLIENT_OPTS $LENS_CLIENT_HEAP"
LENS_LOG_DIR="${LENS_LOG_DIR:-$BASEDIR/logs}"
- export LENS_LOG_DIR
+ export LENS_LOG_DIR
LENS_HOME_DIR="${LENS_HOME_DIR:-$BASEDIR}"
- export LENS_HOME_DIR
+ export LENS_HOME_DIR
;;
server)
if test -z "$LENS_SERVER_HEAP"
@@ -95,7 +95,7 @@ case $type in
LENS_SERVER_HEAP=$DEFAULT_JAVA_HEAP_MAX
fi
LENS_OPTS="$LENS_OPTS $LENS_SERVER_OPTS $LENS_SERVER_HEAP"
- LENSCPPATH="$LENS_CONF"
+ LENSCPPATH="${LENS_CONF}:${LENSCPPATH}"
LENS_EXPANDED_WEBAPP_DIR=${LENS_EXPANDED_WEBAPP_DIR:-${BASEDIR}/webapp}
export LENS_EXPANDED_WEBAPP_DIR
# set the server classpath
@@ -111,7 +111,7 @@ case $type in
HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${LENSCPPATH}
export HADOOP_CLASSPATH
-
+
# log and pid dirs for applications
LENS_LOG_DIR="${LENS_LOG_DIR:-$BASEDIR/logs}"
export LENS_LOG_DIR
[05/51] [abbrv] incubator-lens git commit: LENS-514: Addition of an
identifier in every log line for segregating logs
Posted by jd...@apache.org.
LENS-514: Addition of an identifier in every log line for segregating logs
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/4498c448
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/4498c448
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/4498c448
Branch: refs/heads/current-release-line
Commit: 4498c4488021afce6ddaf9cdd5e16fca27b2d708
Parents: 2e90c29
Author: Himanshu Gahlaut <hi...@apache.org>
Authored: Thu May 28 12:31:50 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu May 28 12:31:50 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/api/query/QueryHandle.java | 11 +++-
.../lens/api/query/QueryPrepareHandle.java | 4 ++
.../apache/lens/api/response/LensResponse.java | 14 +++--
.../apache/lens/driver/hive/TestHiveDriver.java | 3 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 32 ++++++++---
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 8 +--
.../server/api/query/ExplainQueryContext.java | 11 +++-
.../server/api/query/PreparedQueryContext.java | 6 +-
.../lens/server/api/query/QueryContext.java | 12 ++--
.../server/api/query/QueryExecutionService.java | 8 ++-
.../server/model/LogSegregationContext.java | 35 ++++++++++++
.../MappedDiagnosticLogSegregationContext.java | 36 ++++++++++++
.../lens/server/AuthenticationFilter.java | 3 -
.../org/apache/lens/server/LensApplication.java | 4 ++
.../server/LensRequestContextInitFilter.java | 59 ++++++++++++++++++++
.../java/org/apache/lens/server/LensServer.java | 10 ++--
.../org/apache/lens/server/LensServices.java | 14 ++++-
.../org/apache/lens/server/query/QueryApp.java | 2 +
.../lens/server/query/QueryEndNotifier.java | 14 +++--
.../server/query/QueryExecutionServiceImpl.java | 36 +++++++++---
.../lens/server/query/QueryServiceResource.java | 16 ++++--
.../lens/server/query/ResultFormatter.java | 9 ++-
.../stats/store/log/LogStatisticsStore.java | 7 ++-
.../store/log/StatisticsLogFileScannerTask.java | 12 ++++
.../store/log/StatisticsLogRollupHandler.java | 10 +++-
.../org/apache/lens/server/LensJerseyTest.java | 6 +-
.../lens/server/TestStartupOnMetastoreDown.java | 7 ++-
.../server/query/QueryAPIErrorResponseTest.java | 5 +-
.../stats/TestStatisticsLogFileScannerTask.java | 6 +-
lens-server/src/test/resources/log4j.properties | 4 +-
tools/conf-pseudo-distr/server/log4j.properties | 10 ++--
tools/conf/server/log4j.properties | 10 ++--
32 files changed, 350 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
index 7b615e6..9f20184 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandle.java
@@ -27,6 +27,8 @@ import java.util.UUID;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.commons.lang.StringUtils;
+
import lombok.*;
/**
@@ -74,6 +76,13 @@ public class QueryHandle extends QuerySubmitResult implements Serializable {
return new QueryHandle(UUID.fromString(handle));
}
+ public String getHandleIdString() {
+ if (handleId == null) {
+ return StringUtils.EMPTY;
+ }
+ return handleId.toString();
+ }
+
/*
* (non-Javadoc)
*
@@ -81,6 +90,6 @@ public class QueryHandle extends QuerySubmitResult implements Serializable {
*/
@Override
public String toString() {
- return handleId.toString();
+ return getHandleIdString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
index efa04f1..756287e 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryPrepareHandle.java
@@ -75,6 +75,10 @@ public class QueryPrepareHandle extends QuerySubmitResult {
*/
@Override
public String toString() {
+ return getQueryHandleString();
+ }
+
+ public String getQueryHandleString() {
return prepareHandleId.toString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java b/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
index f6c3593..818ae40 100644
--- a/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
+++ b/lens-api/src/main/java/org/apache/lens/api/response/LensResponse.java
@@ -18,11 +18,15 @@
*/
package org.apache.lens.api.response;
+import static com.google.common.base.Preconditions.checkArgument;
+
import javax.ws.rs.core.Response.Status;
import javax.xml.bind.annotation.*;
import org.apache.lens.api.query.QuerySubmitResult;
+import org.apache.commons.lang.StringUtils;
+
import lombok.*;
/**
@@ -78,12 +82,12 @@ public class LensResponse<DATA, PAYLOAD> {
private LensResponse(final String apiVersion, final String id, final DATA data,
final LensErrorTO lensErrorTO, @NonNull final Status httpStatusCode) {
- /* The checks commented below should be enabled in future, once story of apiVersion and id to be used for log
- tracing is clear. Right now there could be REST APIs throwing LensException without initializing apiVersion
- and id.
+ /* The check commented below should be enabled in future, once story of apiVersion is clear. Right now there could
+ be REST APIs throwing LensException without initializing apiVersion
+
+ checkArgument(StringUtils.isNotBlank(apiVersion)); */
- checkArgument(StringUtils.isNotBlank(apiVersion));
- checkArgument(StringUtils.isNotBlank(id)); */
+ checkArgument(StringUtils.isNotBlank(id));
this.apiVersion = apiVersion;
this.id = id;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 71fbc74..b21258a 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -160,7 +160,8 @@ public class TestHiveDriver {
}
protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
- ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers);
+ ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf,
+ drivers);
ectx.setLensSessionIdentifier(sessionid);
return ectx;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 87e4638..70deaa4 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -47,6 +47,8 @@ import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryRewriter;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -56,6 +58,7 @@ import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.log4j.Logger;
import lombok.Getter;
+import lombok.NonNull;
import lombok.Setter;
/**
@@ -89,6 +92,8 @@ public class JDBCDriver implements LensDriver {
/** Estimate connection provider */
private ConnectionProvider estimateConnectionProvider;
+ private LogSegregationContext logSegregationContext;
+
/**
* Data related to a query submitted to JDBCDriver.
*/
@@ -142,12 +147,15 @@ public class JDBCDriver implements LensDriver {
@Setter
private long endTime;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new jdbc query context.
*
* @param context the context
*/
- public JdbcQueryContext(QueryContext context) {
+ public JdbcQueryContext(QueryContext context, @NonNull final LogSegregationContext logSegregationContext) {
+ this.logSegregationContext = logSegregationContext;
this.lensContext = context;
}
@@ -182,6 +190,10 @@ public class JDBCDriver implements LensDriver {
}
isClosed = true;
}
+
+ public String getQueryHandleString() {
+ return this.lensContext.getQueryHandleString();
+ }
}
/**
@@ -261,14 +273,16 @@ public class JDBCDriver implements LensDriver {
/** The query context. */
private final JdbcQueryContext queryContext;
+ private final LogSegregationContext logSegregationContext;
/**
* Instantiates a new query callable.
*
* @param queryContext the query context
*/
- public QueryCallable(JdbcQueryContext queryContext) {
+ public QueryCallable(JdbcQueryContext queryContext, @NonNull LogSegregationContext logSegregationContext) {
this.queryContext = queryContext;
+ this.logSegregationContext = logSegregationContext;
queryContext.setStartTime(System.currentTimeMillis());
}
@@ -279,6 +293,9 @@ public class JDBCDriver implements LensDriver {
*/
@Override
public QueryResult call() {
+
+ logSegregationContext.set(this.queryContext.getQueryHandleString());
+
Statement stmt = null;
Connection conn = null;
QueryResult result = new QueryResult();
@@ -432,6 +449,7 @@ public class JDBCDriver implements LensDriver {
LOG.error("Error initializing connection provider: " + e.getMessage(), e);
throw new LensException(e);
}
+ this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
}
/**
@@ -494,8 +512,6 @@ public class JDBCDriver implements LensDriver {
/**
* Rewrite query.
*
- * @param query the query
- * @param conf the conf
* @return the string
* @throws LensException the lens exception
*/
@@ -850,10 +866,10 @@ public class JDBCDriver implements LensDriver {
*/
private QueryResult executeInternal(QueryContext context, String rewrittenQuery) throws LensException {
- JdbcQueryContext queryContext = new JdbcQueryContext(context);
+ JdbcQueryContext queryContext = new JdbcQueryContext(context, logSegregationContext);
queryContext.setPrepared(false);
queryContext.setRewrittenQuery(rewrittenQuery);
- QueryResult result = new QueryCallable(queryContext).call();
+ QueryResult result = new QueryCallable(queryContext, logSegregationContext).call();
return result;
// LOG.info("Execute " + context.getQueryHandle());
}
@@ -870,10 +886,10 @@ public class JDBCDriver implements LensDriver {
// Always use the driver rewritten query not user query. Since the
// conf we are passing here is query context conf, we need to add jdbc xml in resource path
String rewrittenQuery = rewriteQuery(context);
- JdbcQueryContext jdbcCtx = new JdbcQueryContext(context);
+ JdbcQueryContext jdbcCtx = new JdbcQueryContext(context, logSegregationContext);
jdbcCtx.setRewrittenQuery(rewrittenQuery);
try {
- Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx));
+ Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext));
jdbcCtx.setResultFuture(future);
} catch (RejectedExecutionException e) {
LOG.error("Query execution rejected: " + context.getQueryHandle() + " reason:" + e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 4f03f23..2e3dcc7 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -24,10 +24,7 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -131,7 +128,8 @@ public class TestJdbcDriver {
}
protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
- ExplainQueryContext ectx = new ExplainQueryContext(query, "testuser", null, conf, drivers);
+ ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf,
+ drivers);
return ectx;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
index da44955..c7b9887 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/ExplainQueryContext.java
@@ -31,14 +31,21 @@ import org.apache.hadoop.conf.Configuration;
public class ExplainQueryContext extends AbstractQueryContext {
private static final long serialVersionUID = 1L;
+ private final String requestId;
/**
* Constructor. Only needs user query and conf.
*
* @param query
* @param qconf
*/
- public ExplainQueryContext(String query, final String user, LensConf conf, Configuration qconf,
- Collection<LensDriver> drivers) {
+ public ExplainQueryContext(final String requestId, String query, final String user, LensConf conf,
+ Configuration qconf, Collection<LensDriver> drivers) {
super(query, user, conf, qconf, drivers, true);
+ this.requestId = requestId;
+ }
+
+ @Override
+ public String getLogHandle() {
+ return this.requestId;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
index 1ce89ac..b6f669b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
@@ -153,12 +153,16 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye
lensConf);
}
+ public String getQueryHandleString() {
+ return prepareHandle.getQueryHandleString();
+ }
+
/**
* Get prepared query handle string
* @return
*/
@Override
public String getLogHandle() {
- return prepareHandle.getPrepareHandleId().toString();
+ return getQueryHandleString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 169ac8d..3e0c26c 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -368,12 +368,12 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
public boolean getCompressOutput() {
return conf.getBoolean(LensConfConstants.QUERY_OUTPUT_ENABLE_COMPRESSION,
- LensConfConstants.DEFAULT_OUTPUT_ENABLE_COMPRESSION);
+ LensConfConstants.DEFAULT_OUTPUT_ENABLE_COMPRESSION);
}
public long getMaxResultSplitRows() {
return conf.getLong(LensConfConstants.RESULT_SPLIT_MULTIPLE_MAX_ROWS,
- LensConfConstants.DEFAULT_RESULT_SPLIT_MULTIPLE_MAX_ROWS);
+ LensConfConstants.DEFAULT_RESULT_SPLIT_MULTIPLE_MAX_ROWS);
}
/**
@@ -383,7 +383,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
*/
public boolean splitResultIntoMultipleFiles() {
return conf.getBoolean(LensConfConstants.RESULT_SPLIT_INTO_MULTIPLE,
- LensConfConstants.DEFAULT_RESULT_SPLIT_INTO_MULTIPLE);
+ LensConfConstants.DEFAULT_RESULT_SPLIT_INTO_MULTIPLE);
}
public String getClusterUser() {
@@ -397,6 +397,10 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
*/
@Override
public String getLogHandle() {
- return queryHandle.getHandleId().toString();
+ return getQueryHandleString();
+ }
+
+ public String getQueryHandleString() {
+ return queryHandle.getHandleIdString();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
index 8b6d131..3c0f787 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
@@ -40,6 +40,7 @@ public interface QueryExecutionService {
/**
* Estimate the cost of given query.
*
+ * @param requestId the request Id of request used to start estimate operation
* @param sessionHandle the session handle
* @param query The query should be in HiveQL(SQL like)
* @param conf The query configuration
@@ -48,18 +49,21 @@ public interface QueryExecutionService {
*
* @throws LensException thrown in case of failure
*/
- QueryCost estimate(LensSessionHandle sessionHandle, String query, LensConf conf) throws LensException;
+ QueryCost estimate(final String requestId, LensSessionHandle sessionHandle, String query, LensConf conf)
+ throws LensException;
/**
* Explain the given query.
*
+ * @param requestId the request Id of request used to start explain operation
* @param sessionHandle the session handle
* @param query The query should be in HiveQL(SQL like)
* @param conf The query configuration
* @return The query plan;
* @throws LensException the lens exception
*/
- QueryPlan explain(LensSessionHandle sessionHandle, String query, LensConf conf) throws LensException;
+ QueryPlan explain(final String requestId, LensSessionHandle sessionHandle, String query, LensConf conf)
+ throws LensException;
/**
* Prepare the query.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
new file mode 100644
index 0000000..8691a05
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.model;
+
+public interface LogSegregationContext {
+
+ /**
+ * Sets an id to be used by current thread in every log line for log segregation
+ *
+ * @param id the id to be added to every log line of current thread
+ */
+ void set(final String id);
+
+ /**
+ *
+ * @return the id being used by the current thread for log segregation
+ */
+ String get();
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
new file mode 100644
index 0000000..989cb9d
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.model;
+
+import org.slf4j.MDC;
+
+public class MappedDiagnosticLogSegregationContext implements LogSegregationContext {
+
+ private static final String LOG_SEGREGATION_ID = "logSegregationId";
+
+ @Override
+ public void set(String id) {
+ MDC.put(LOG_SEGREGATION_ID, id);
+ }
+
+ @Override
+ public String get() {
+ return MDC.get(LOG_SEGREGATION_ID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
index b64d822..e39c7a2 100644
--- a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
@@ -19,7 +19,6 @@
package org.apache.lens.server;
import java.io.IOException;
-import java.util.UUID;
import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
@@ -45,9 +44,7 @@ public class AuthenticationFilter implements ContainerRequestFilter {
public void filter(ContainerRequestContext requestContext) throws IOException {
final SecurityContext securityContext = requestContext.getSecurityContext();
- String requestId = UUID.randomUUID().toString();
String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : null;
- requestContext.getHeaders().add("requestId", requestId);
LOG.info("Request from user: " + user + ", path=" + requestContext.getUriInfo().getPath());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
index cb452e8..a3079cc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
@@ -73,6 +73,10 @@ public class LensApplication extends Application {
classes.add(wsFilterClass);
LOG.info("Added filter " + wsFilterClass);
}
+
+ LOG.debug("LensRequestContextInitFilter added...");
+ classes.add(LensRequestContextInitFilter.class);
+
return classes;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
new file mode 100644
index 0000000..c9658fc
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * LensRequestContextInitFilter is expected to be called before all other request filters.
+ * Priority value of 1 is to ensure the same.
+ *
+ */
+@Slf4j
+@Priority(1)
+public class LensRequestContextInitFilter implements ContainerRequestFilter {
+
+ private static final String REQUEST_ID = "requestId";
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws IOException {
+
+ log.debug("Entering {}", getClass().getName());
+
+ /* Create a unique identifier for request */
+ String uniqueRequesId = UUID.randomUUID().toString();
+
+ /* Add request id for appearing in every log line */
+ new MappedDiagnosticLogSegregationContext().set(uniqueRequesId);
+
+ /* Add request id to headers */
+ requestContext.getHeaders().add(REQUEST_ID, uniqueRequesId);
+
+ log.debug("Leaving {}", getClass().getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensServer.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
index c6d7ea1..e200bdf 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
@@ -20,10 +20,7 @@ package org.apache.lens.server;
import java.io.IOException;
import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
import java.util.logging.Logger;
import javax.ws.rs.core.UriBuilder;
@@ -33,6 +30,7 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.error.LensExceptionMapper;
import org.apache.lens.server.metrics.MetricsServiceImpl;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.ui.UIApp;
import org.apache.commons.logging.Log;
@@ -113,6 +111,7 @@ public class LensServer {
}
private ResourceConfig getApp() {
+
ResourceConfig app = ResourceConfig.forApplicationClass(LensApplication.class);
app.register(new LoggingFilter(Logger.getLogger(LensServer.class.getName() + ".request"), true));
app.register(LensExceptionMapper.class);
@@ -185,6 +184,9 @@ public class LensServer {
*/
public static void main(String[] args) throws Exception {
+ final String runId = UUID.randomUUID().toString();
+ new MappedDiagnosticLogSegregationContext().set(runId);
+
printStartupMessage();
try {
final LensServer thisServer = LensServer.createLensServer(LensServerConf.getHiveConf());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index f769619..ab197ab 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -33,6 +33,8 @@ import org.apache.lens.server.api.ServiceProvider;
import org.apache.lens.server.api.events.LensEventService;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.metrics.MetricsServiceImpl;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.user.UserConfigLoaderFactory;
@@ -49,6 +51,7 @@ import org.apache.hive.service.Service;
import org.apache.hive.service.cli.CLIService;
import lombok.Getter;
+import lombok.NonNull;
import lombok.Setter;
@@ -68,7 +71,8 @@ public class LensServices extends CompositeService implements ServiceProvider {
private static final String FS_IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
/** The instance. */
- private static LensServices instance = new LensServices(LENS_SERVICES_NAME);
+ private static LensServices instance = new LensServices(LENS_SERVICES_NAME,
+ new MappedDiagnosticLogSegregationContext());
/** The conf. */
private HiveConf conf;
@@ -108,6 +112,9 @@ public class LensServices extends CompositeService implements ServiceProvider {
@Getter
private ErrorCollection errorCollection;
+ @Getter
+ private final LogSegregationContext logSegregationContext;
+
/**
* The Enum SERVICE_MODE.
*/
@@ -129,8 +136,9 @@ public class LensServices extends CompositeService implements ServiceProvider {
*
* @param name the name
*/
- public LensServices(String name) {
+ public LensServices(String name, @NonNull final LogSegregationContext logSegregationContext) {
super(name);
+ this.logSegregationContext = logSegregationContext;
}
// This is only for test, to simulate a restart of the server
@@ -244,6 +252,8 @@ public class LensServices extends CompositeService implements ServiceProvider {
@Override
public void run() {
try {
+ final String runId = UUID.randomUUID().toString();
+ logSegregationContext.set(runId);
persistLensServiceState();
LOG.info("SnapShot of Lens Services created");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
index 0d23726..9db034a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryApp.java
@@ -25,6 +25,7 @@ import javax.ws.rs.ApplicationPath;
import javax.ws.rs.core.Application;
import org.apache.lens.server.LensApplicationListener;
+import org.apache.lens.server.LensRequestContextInitFilter;
import org.glassfish.jersey.filter.LoggingFilter;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
@@ -40,6 +41,7 @@ public class QueryApp extends Application {
// register root resource
classes.add(QueryServiceResource.class);
classes.add(MultiPartFeature.class);
+ classes.add(LensRequestContextInitFilter.class);
classes.add(LoggingFilter.class);
classes.add(LensApplicationListener.class);
return classes;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
index 1c6f186..df36a21 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
@@ -37,6 +37,7 @@ import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.QueryEnded;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
@@ -44,6 +45,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import lombok.Data;
+import lombok.NonNull;
/**
* The Class QueryEndNotifier.
@@ -77,23 +79,26 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
/** The mail smtp connection timeout. */
private final int mailSmtpConnectionTimeout;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new query end notifier.
*
* @param queryService the query service
* @param hiveConf the hive conf
*/
- public QueryEndNotifier(QueryExecutionServiceImpl queryService, HiveConf hiveConf) {
+ public QueryEndNotifier(QueryExecutionServiceImpl queryService, HiveConf hiveConf,
+ @NonNull final LogSegregationContext logSegregationContext) {
this.queryService = queryService;
this.conf = hiveConf;
from = conf.get(LensConfConstants.MAIL_FROM_ADDRESS);
host = conf.get(LensConfConstants.MAIL_HOST);
port = conf.get(LensConfConstants.MAIL_PORT);
- mailSmtpTimeout = Integer.parseInt(conf.get(LensConfConstants.MAIL_SMTP_TIMEOUT,
- LensConfConstants.MAIL_DEFAULT_SMTP_TIMEOUT));
+ mailSmtpTimeout = Integer.parseInt(
+ conf.get(LensConfConstants.MAIL_SMTP_TIMEOUT, LensConfConstants.MAIL_DEFAULT_SMTP_TIMEOUT));
mailSmtpConnectionTimeout = Integer.parseInt(conf.get(LensConfConstants.MAIL_SMTP_CONNECTIONTIMEOUT,
LensConfConstants.MAIL_DEFAULT_SMTP_CONNECTIONTIMEOUT));
-
+ this.logSegregationContext = logSegregationContext;
}
/*
@@ -112,6 +117,7 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
+ ". No email generated");
return;
}
+ this.logSegregationContext.set(queryContext.getQueryHandleString());
boolean whetherMailNotify = Boolean.parseBoolean(queryContext.getConf().get(LensConfConstants.QUERY_MAIL_NOTIFY,
LensConfConstants.WHETHER_MAIL_NOTIFY_DEFAULT));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 7c2da3a..dbc52e9 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -51,6 +51,8 @@ import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.util.UtilityMethods;
@@ -229,6 +231,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
private ExecutorService estimatePool;
+ private final LogSegregationContext logSegregationContext;
+
/**
* The driver event listener.
*/
@@ -250,8 +254,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* @param cliService the cli service
* @throws LensException the lens exception
*/
- public QueryExecutionServiceImpl(CLIService cliService) throws LensException {
+ public QueryExecutionServiceImpl(CLIService cliService)
+ throws LensException {
super(NAME, cliService);
+ this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
}
/**
@@ -280,10 +286,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("Registered query state logger");
}
// Add result formatter
- getEventService().addListenerForType(new ResultFormatter(this), QueryExecuted.class);
+ getEventService().addListenerForType(new ResultFormatter(this, this.logSegregationContext), QueryExecuted.class);
getEventService().addListenerForType(new QueryExecutionStatisticsGenerator(this, getEventService()),
QueryEnded.class);
- getEventService().addListenerForType(new QueryEndNotifier(this, getCliService().getHiveConf()), QueryEnded.class);
+ getEventService().addListenerForType(
+ new QueryEndNotifier(this, getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class);
log.info("Registered query result formatter");
}
@@ -456,6 +463,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryContext getCtx() {
return ctx;
}
+
+ public String getQueryHandleString() {
+ return ctx.getQueryHandleString();
+ }
}
/**
@@ -479,6 +490,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
while (!pausedForTest && !stopped && !querySubmitter.isInterrupted()) {
try {
QueryContext ctx = queuedQueries.take();
+
+ /* Setting log segregation id */
+ logSegregationContext.set(ctx.getQueryHandleString());
+
synchronized (ctx) {
if (ctx.getStatus().getStatus().equals(Status.QUEUED)) {
log.info("Launching query:" + ctx.getUserQuery());
@@ -553,6 +568,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (stopped || statusPoller.isInterrupted()) {
return;
}
+
+ logSegregationContext.set(ctx.getQueryHandleString());
log.info("Polling status for " + ctx.getQueryHandle());
try {
// session is not required to update status of the query
@@ -762,6 +779,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
FinishedQuery finished = null;
try {
finished = finishedQueries.take();
+ logSegregationContext.set(finished.getQueryHandleString());
} catch (InterruptedException e) {
log.info("QueryPurger has been interrupted, exiting");
return;
@@ -835,6 +853,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
while (!stopped && !prepareQueryPurger.isInterrupted()) {
try {
PreparedQueryContext prepared = preparedQueryQueue.take();
+ logSegregationContext.set(prepared.getQueryHandleString());
destroyPreparedQuery(prepared);
log.info("Purged prepared query: " + prepared.getPrepareHandle());
} catch (LensException e) {
@@ -1005,6 +1024,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
};
+ LOG.debug("starting estimate pool");
ThreadPoolExecutor estimatePool = new ThreadPoolExecutor(minPoolSize, maxPoolSize, keepAlive, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(), threadFactory);
estimatePool.allowCoreThreadTimeOut(true);
@@ -1163,6 +1183,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public void run() {
try {
+ logSegregationContext.set(ctx.getLogHandle());
acquire(ctx.getLensSessionIdentifier());
MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
REWRITE_GAUGE);
@@ -2021,13 +2042,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* java.lang.String, org.apache.lens.api.LensConf)
*/
@Override
- public QueryCost estimate(LensSessionHandle sessionHandle, String query, LensConf lensConf)
+ public QueryCost estimate(final String requestId, LensSessionHandle sessionHandle, String query, LensConf lensConf)
throws LensException {
try {
log.info("Estimate: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
- ExplainQueryContext estimateQueryContext = new ExplainQueryContext(query,
+ ExplainQueryContext estimateQueryContext = new ExplainQueryContext(requestId, query,
getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values());
estimateQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
accept(query, qconf, SubmitOp.ESTIMATE);
@@ -2045,12 +2066,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* java.lang.String, org.apache.lens.api.LensConf)
*/
@Override
- public QueryPlan explain(LensSessionHandle sessionHandle, String query, LensConf lensConf) throws LensException {
+ public QueryPlan explain(final String requestId, LensSessionHandle sessionHandle, String query, LensConf lensConf)
+ throws LensException {
try {
log.info("Explain: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
- ExplainQueryContext explainQueryContext = new ExplainQueryContext(query,
+ ExplainQueryContext explainQueryContext = new ExplainQueryContext(requestId, query,
getSession(sessionHandle).getLoggedInUser(), lensConf, qconf, drivers.values());
explainQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
accept(query, qconf, SubmitOp.EXPLAIN);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index 34e14b6..badde8c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -40,6 +40,7 @@ import org.apache.lens.server.api.annotations.MultiPurposeResource;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.lens.server.error.UnSupportedQuerySubmitOpException;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.lang.StringUtils;
@@ -61,6 +62,8 @@ public class QueryServiceResource {
private final ErrorCollection errorCollection;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Check session id.
*
@@ -123,6 +126,7 @@ public class QueryServiceResource {
public QueryServiceResource() throws LensException {
queryServer = (QueryExecutionService) LensServices.get().getService("query");
errorCollection = LensServices.get().getErrorCollection();
+ logSegregationContext = LensServices.get().getLogSegregationContext();
}
QueryExecutionService getQueryServer() {
@@ -201,8 +205,9 @@ public class QueryServiceResource {
@FormDataParam("conf") LensConf conf, @DefaultValue("30000") @FormDataParam("timeoutmillis") Long timeoutmillis,
@DefaultValue("") @FormDataParam("queryName") String queryName) throws LensException {
- try {
+ final String requestId = this.logSegregationContext.get();
+ try {
validateSessionId(sessionid);
SubmitOp sop = checkAndGetQuerySubmitOperation(operation);
validateQuery(query);
@@ -210,13 +215,13 @@ public class QueryServiceResource {
QuerySubmitResult result;
switch (sop) {
case ESTIMATE:
- result = queryServer.estimate(sessionid, query, conf);
+ result = queryServer.estimate(requestId, sessionid, query, conf);
break;
case EXECUTE:
result = queryServer.executeAsync(sessionid, query, conf, queryName);
break;
case EXPLAIN:
- result = queryServer.explain(sessionid, query, conf);
+ result = queryServer.explain(requestId, sessionid, query, conf);
break;
case EXECUTE_WITH_TIMEOUT:
result = queryServer.execute(sessionid, query, timeoutmillis, conf, queryName);
@@ -224,9 +229,10 @@ public class QueryServiceResource {
default:
throw new UnSupportedQuerySubmitOpException();
}
- return LensResponse.composedOf(null, null, result);
+
+ return LensResponse.composedOf(null, requestId, result);
} catch (LensException e) {
- e.buildLensErrorResponse(errorCollection, null, null);
+ e.buildLensErrorResponse(errorCollection, null, requestId);
throw e;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index 7f5ed65..de7e120 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -28,6 +28,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,6 +36,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ReflectionUtils;
+import lombok.NonNull;
+
/**
* The Class ResultFormatter.
*/
@@ -46,13 +49,16 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
/** The query service. */
QueryExecutionServiceImpl queryService;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new result formatter.
*
* @param queryService the query service
*/
- public ResultFormatter(QueryExecutionServiceImpl queryService) {
+ public ResultFormatter(QueryExecutionServiceImpl queryService, @NonNull LogSegregationContext logSegregationContext) {
this.queryService = queryService;
+ this.logSegregationContext = logSegregationContext;
}
/*
@@ -73,6 +79,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
private void formatOutput(QueryExecuted event) {
QueryHandle queryHandle = event.getQueryHandle();
QueryContext ctx = queryService.getQueryContext(queryHandle);
+ this.logSegregationContext.set(ctx.getQueryHandleString());
try {
if (!ctx.isPersistent()) {
LOG.info("No result formatting required for query " + queryHandle);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
index 131a375..81ad260 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/LogStatisticsStore.java
@@ -21,6 +21,8 @@ package org.apache.lens.server.stats.store.log;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.events.LensEventService;
import org.apache.lens.server.api.metrics.MetricsService;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.stats.event.LoggableLensStatistics;
import org.apache.lens.server.stats.store.StatisticsStore;
@@ -53,11 +55,14 @@ public class LogStatisticsStore extends StatisticsStore<LoggableLensStatistics>
/** The rollup handler. */
private StatisticsLogRollupHandler rollupHandler;
+ private final LogSegregationContext logSegregationContext;
+
/**
* Instantiates a new log statistics store.
*/
public LogStatisticsStore() {
this.mapper = new ObjectMapper();
+ this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
}
@@ -71,7 +76,7 @@ public class LogStatisticsStore extends StatisticsStore<LoggableLensStatistics>
handler = new StatisticsLogPartitionHandler();
handler.initialize(conf);
LOG.info("Creating new rollup handler");
- rollupHandler = new StatisticsLogRollupHandler();
+ rollupHandler = new StatisticsLogRollupHandler(this.logSegregationContext);
rollupHandler.initialize(conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
index 254632b..7e9c74d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
@@ -23,18 +23,21 @@ import java.io.FilenameFilter;
import java.util.HashMap;
import java.util.Map;
import java.util.TimerTask;
+import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventService;
import org.apache.lens.server.api.metrics.MetricsService;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Logger;
import org.slf4j.LoggerFactory;
+import lombok.NonNull;
import lombok.Setter;
/**
@@ -58,6 +61,11 @@ public class StatisticsLogFileScannerTask extends TimerTask {
/** The class set. */
private Map<String, String> classSet = new ConcurrentHashMap<String, String>();
+ private final LogSegregationContext logSegregationContext;
+
+ public StatisticsLogFileScannerTask(@NonNull final LogSegregationContext logSegregationContext) {
+ this.logSegregationContext = logSegregationContext;
+ }
/*
* (non-Javadoc)
*
@@ -66,6 +74,10 @@ public class StatisticsLogFileScannerTask extends TimerTask {
@Override
public void run() {
try {
+
+ final String runId = UUID.randomUUID().toString();
+ this.logSegregationContext.set(runId);
+
for (Map.Entry<String, String> entry : scanSet.entrySet()) {
File f = new File(entry.getValue()).getAbsoluteFile();
String fileName = f.getAbsolutePath();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
index 78e4518..9ff878c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogRollupHandler.java
@@ -22,11 +22,14 @@ import java.util.Timer;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.events.LensEventService;
+import org.apache.lens.server.model.LogSegregationContext;
import org.apache.hadoop.conf.Configuration;
import org.eclipse.jetty.util.ConcurrentHashSet;
+import lombok.NonNull;
+
/**
* Class which handles the log4j rolled file.
*/
@@ -44,13 +47,18 @@ public class StatisticsLogRollupHandler {
/** The scan set. */
private final ConcurrentHashSet<String> scanSet = new ConcurrentHashSet<String>();
+ private final LogSegregationContext logSegregationContext;
+
+ public StatisticsLogRollupHandler(@NonNull final LogSegregationContext logSegregationContext) {
+ this.logSegregationContext = logSegregationContext;
+ }
/**
* Initalize the handler.
*
* @param conf configuration to be used while initialization.
*/
public void initialize(Configuration conf) {
- task = new StatisticsLogFileScannerTask();
+ task = new StatisticsLogFileScannerTask(this.logSegregationContext);
timer = new Timer();
rate = conf.getLong(LensConfConstants.STATS_ROLLUP_SCAN_RATE, LensConfConstants.DEFAULT_STATS_ROLLUP_SCAN_RATE);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
index 7913845..fb3e67b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
@@ -30,6 +30,8 @@ import org.apache.lens.driver.hive.TestRemoteHiveDriver;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.LensMetricsUtil;
import org.apache.lens.server.api.metrics.MetricsService;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -50,6 +52,8 @@ public abstract class LensJerseyTest extends JerseyTest {
private int port = -1;
+ private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
+
protected URI getUri() {
return UriBuilder.fromUri("http://localhost/").port(getTestPort()).build();
}
@@ -195,7 +199,7 @@ public abstract class LensJerseyTest extends JerseyTest {
LensServices.get().stop();
LensMetricsUtil.clearRegistry();
System.out.println("Lens services stopped!");
- LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME));
+ LensServices.setInstance(new LensServices(LensServices.LENS_SERVICES_NAME, this.logSegregationContext));
LensServices.get().init(conf);
LensServices.get().start();
System.out.println("Lens services restarted!");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
index decd6ef..3fb123e 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.server;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -28,9 +31,11 @@ import org.testng.Assert;
public class TestStartupOnMetastoreDown {
private static final Log LOG = LogFactory.getLog(TestStartupOnMetastoreDown.class);
+ private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
+
// @Test
public void testServicesStartOnMetastoreDown() throws Exception {
- LensServices services = new LensServices(LensServices.LENS_SERVICES_NAME);
+ LensServices services = new LensServices(LensServices.LENS_SERVICES_NAME, logSegregationContext);
HiveConf hiveConf = new HiveConf();
// Set metastore uri to an invalid location
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 5e135be..4c2a7a4 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -45,6 +45,7 @@ import org.apache.lens.api.response.LensErrorTO;
import org.apache.lens.api.response.LensJAXBContextResolver;
import org.apache.lens.cube.error.ColUnAvailableInTimeRange;
import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.LensRequestContextInitFilter;
import org.apache.lens.server.common.ErrorResponseExpectedData;
import org.apache.lens.server.error.LensExceptionMapper;
import org.apache.lens.server.metastore.MetastoreResource;
@@ -89,8 +90,8 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
enable(TestProperties.LOG_TRAFFIC);
enable(TestProperties.DUMP_ENTITY);
- return new ResourceConfig(SessionResource.class, MetastoreResource.class, QueryServiceResource.class,
- MultiPartFeature.class, LensExceptionMapper.class, LensJAXBContextResolver.class);
+ return new ResourceConfig(LensRequestContextInitFilter.class, SessionResource.class, MetastoreResource.class,
+ QueryServiceResource.class, MultiPartFeature.class, LensExceptionMapper.class, LensJAXBContextResolver.class);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
index d5e60c4..819c62a 100644
--- a/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
+++ b/lens-server/src/test/java/org/apache/lens/server/stats/TestStatisticsLogFileScannerTask.java
@@ -23,6 +23,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lens.server.api.events.LensEventService;
+import org.apache.lens.server.model.LogSegregationContext;
+import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.stats.store.log.PartitionEvent;
import org.apache.lens.server.stats.store.log.StatisticsLogFileScannerTask;
@@ -52,6 +54,8 @@ public class TestStatisticsLogFileScannerTask {
/** The hidden. */
private File hidden;
+ private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
+
/**
* Creates the test log file.
*
@@ -90,7 +94,7 @@ public class TestStatisticsLogFileScannerTask {
appender.setName(TestStatisticsLogFileScannerTask.class.getSimpleName());
l.addAppender(appender);
- StatisticsLogFileScannerTask task = new StatisticsLogFileScannerTask();
+ StatisticsLogFileScannerTask task = new StatisticsLogFileScannerTask(this.logSegregationContext);
task.addLogFile(TestStatisticsLogFileScannerTask.class.getName());
LensEventService service = Mockito.mock(LensEventService.class);
final List<PartitionEvent> events = new ArrayList<PartitionEvent>();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/lens-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/log4j.properties b/lens-server/src/test/resources/log4j.properties
index 65d065f..f3ccea2 100644
--- a/lens-server/src/test/resources/log4j.properties
+++ b/lens-server/src/test/resources/log4j.properties
@@ -31,11 +31,11 @@ log4j.additivity.org.apache.lens.server.stats.TestLogStatisticsStore$MyLoggableL
# Configuration for appenders
log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender
log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout
-log4j.appender.STDOUT.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.STDOUT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.STDOUT.Threshold=ERROR
log4j.appender.TEST_LOG_FILE=org.apache.log4j.RollingFileAppender
log4j.appender.TEST_LOG_FILE.File=lens-server/target/test.log
log4j.appender.TEST_LOG_FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %X{logSegregationId} [%t] %-5p %c %x - %m%n
log4j.appender.TEST_LOG_FILE.Threshold=ERROR
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/tools/conf-pseudo-distr/server/log4j.properties
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/log4j.properties b/tools/conf-pseudo-distr/server/log4j.properties
index afadc2f..2b04ea5 100644
--- a/tools/conf-pseudo-distr/server/log4j.properties
+++ b/tools/conf-pseudo-distr/server/log4j.properties
@@ -38,12 +38,12 @@ log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT=org.apache.log4j.RollingFileAppender
log4j.appender.ROOT.File=${lens.log.dir}/lensserver.log
log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -53,7 +53,7 @@ log4j.appender.ROOT.MaxBackupIndex=20
log4j.appender.AUDIT=org.apache.log4j.RollingFileAppender
log4j.appender.AUDIT.File=${lens.log.dir}/lensserver-audit.log
log4j.appender.AUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.AUDIT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -62,7 +62,7 @@ log4j.appender.AUDIT.MaxBackupIndex=20
log4j.appender.REQUEST=org.apache.log4j.RollingFileAppender
log4j.appender.REQUEST.File=${lens.log.dir}/lensserver-requests.log
log4j.appender.REQUEST.layout=org.apache.log4j.PatternLayout
-log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.REQUEST.MaxFileSize=100000KB
# Keep 20 backup files
@@ -71,7 +71,7 @@ log4j.appender.REQUEST.MaxBackupIndex=20
log4j.appender.STATUS=org.apache.log4j.RollingFileAppender
log4j.appender.STATUS.File=${lens.log.dir}/lensserver-query-status.log
log4j.appender.STATUS.layout=org.apache.log4j.PatternLayout
-log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.STATUS.MaxFileSize=100000KB
# Keep 20 backup files
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4498c448/tools/conf/server/log4j.properties
----------------------------------------------------------------------
diff --git a/tools/conf/server/log4j.properties b/tools/conf/server/log4j.properties
index afadc2f..2b04ea5 100644
--- a/tools/conf/server/log4j.properties
+++ b/tools/conf/server/log4j.properties
@@ -38,12 +38,12 @@ log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
# CONSOLE uses PatternLayout.
log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT=org.apache.log4j.RollingFileAppender
log4j.appender.ROOT.File=${lens.log.dir}/lensserver.log
log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.ROOT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -53,7 +53,7 @@ log4j.appender.ROOT.MaxBackupIndex=20
log4j.appender.AUDIT=org.apache.log4j.RollingFileAppender
log4j.appender.AUDIT.File=${lens.log.dir}/lensserver-audit.log
log4j.appender.AUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.AUDIT.MaxFileSize=100000KB
# Keep 20 backup files
@@ -62,7 +62,7 @@ log4j.appender.AUDIT.MaxBackupIndex=20
log4j.appender.REQUEST=org.apache.log4j.RollingFileAppender
log4j.appender.REQUEST.File=${lens.log.dir}/lensserver-requests.log
log4j.appender.REQUEST.layout=org.apache.log4j.PatternLayout
-log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.REQUEST.MaxFileSize=100000KB
# Keep 20 backup files
@@ -71,7 +71,7 @@ log4j.appender.REQUEST.MaxBackupIndex=20
log4j.appender.STATUS=org.apache.log4j.RollingFileAppender
log4j.appender.STATUS.File=${lens.log.dir}/lensserver-query-status.log
log4j.appender.STATUS.layout=org.apache.log4j.PatternLayout
-log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
+log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
log4j.appender.STATUS.MaxFileSize=100000KB
# Keep 20 backup files
[11/51] [abbrv] incubator-lens git commit: LENS-479 : Add server
state persisting error counter (Yash Sharma via amareshwari)
Posted by jd...@apache.org.
LENS-479 : Add server state persisting error counter (Yash Sharma via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/fa83c481
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/fa83c481
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/fa83c481
Branch: refs/heads/current-release-line
Commit: fa83c481e5b20dfc798e7487380baa8f77a55b4f
Parents: 85da8c8
Author: Yash Sharma <ya...@gmail.com>
Authored: Wed Jun 3 15:46:13 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jun 3 15:46:13 2015 +0530
----------------------------------------------------------------------
.../lens/server/api/metrics/MetricsService.java | 2 +
.../org/apache/lens/server/LensServices.java | 42 ++++++++++++++++++++
.../lens/server/metrics/MetricsServiceImpl.java | 19 +++++++--
3 files changed, 59 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/fa83c481/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
index 9cfb65d..ffde912 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
@@ -149,6 +149,8 @@ public interface MetricsService {
long getTotalFailedQueries();
+ long getTotalServerStatePersistenceErrors();
+
/**
* Publish report.
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/fa83c481/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index ab197ab..c27717f 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -98,6 +98,16 @@ public class LensServices extends CompositeService implements ServiceProvider {
/** The snap shot interval. */
private long snapShotInterval;
+ /**
+ * The metrics service.
+ */
+ private MetricsService metricsService;
+
+ /**
+ * The Constant SERVER_STATE_PERSISTENCE_ERRORS.
+ */
+ public static final String SERVER_STATE_PERSISTENCE_ERRORS = "total-server-state-persistence-errors";
+
/** The service mode. */
@Getter
@Setter
@@ -116,6 +126,24 @@ public class LensServices extends CompositeService implements ServiceProvider {
private final LogSegregationContext logSegregationContext;
/**
+ * Incr counter.
+ *
+ * @param counter the counter
+ */
+ private void incrCounter(String counter) {
+ getMetricService().incrCounter(LensServices.class, counter);
+ }
+
+ /**
+ * Gets counter value.
+ *
+ * @param counter the counter
+ */
+ private long getCounter(String counter) {
+ return getMetricService().getCounter(LensServices.class, counter);
+ }
+
+ /**
* The Enum SERVICE_MODE.
*/
public enum SERVICE_MODE {
@@ -257,6 +285,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
persistLensServiceState();
LOG.info("SnapShot of Lens Services created");
} catch (IOException e) {
+ incrCounter(SERVER_STATE_PERSISTENCE_ERRORS);
LOG.warn("Unable to persist lens server state", e);
}
}
@@ -326,6 +355,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
}
}
if (!persistenceFS.rename(serviceWritePath, servicePath)) {
+ incrCounter(SERVER_STATE_PERSISTENCE_ERRORS);
LOG.error("Failed to persist " + service.getName() + " to [" + servicePath + "]");
} else {
LOG.info("Persisted service " + service.getName() + " to [" + servicePath + "]");
@@ -372,6 +402,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
persistenceFS.close();
LOG.info("Persistence File system object close complete");
} catch (IOException e) {
+ incrCounter(SERVER_STATE_PERSISTENCE_ERRORS);
LOG.error("Could not persist server state", e);
throw new IllegalStateException(e);
} finally {
@@ -419,4 +450,15 @@ public class LensServices extends CompositeService implements ServiceProvider {
throw new RuntimeException("Could not create error collection.", e);
}
}
+
+
+ private MetricsService getMetricService() {
+ if (metricsService == null) {
+ metricsService = (MetricsService) LensServices.get().getService(MetricsService.NAME);
+ if (metricsService == null) {
+ throw new NullPointerException("Could not get metrics service");
+ }
+ }
+ return metricsService;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/fa83c481/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index 6e8b3a6..66beeed 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -96,6 +96,9 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
/** The total expired sessions*/
private Counter totalExpiredSessions;
+ /** The total errors while persisting server state */
+ private Counter totalServerStatePersistenceErrors;
+
/** The total accepted queries. */
private Counter totalAcceptedQueries;
@@ -355,6 +358,9 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
totalExpiredSessions = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-"
+ EXPIRED_SESSIONS));
+ totalServerStatePersistenceErrors = metricRegistry.counter(MetricRegistry.name(LensServices.class,
+ LensServices.SERVER_STATE_PERSISTENCE_ERRORS));
+
metricRegistry.register("gc", new GarbageCollectorMetricSet());
metricRegistry.register("memory", new MemoryUsageGaugeSet());
metricRegistry.register("threads", new ThreadStatesGaugeSet());
@@ -534,11 +540,16 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
return totalExpiredSessions.getCount();
}
+ @Override
+ public long getTotalServerStatePersistenceErrors() {
+ return totalServerStatePersistenceErrors.getCount();
+ }
+
/*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.metrics.MetricsService#publishReport()
- */
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.metrics.MetricsService#publishReport()
+ */
@Override
public void publishReport() {
if (reporters != null) {
[51/51] [abbrv] incubator-lens git commit: LENS-513: Add jar regex -
Testcase bugfix
Posted by jd...@apache.org.
LENS-513: Add jar regex - Testcase bugfix
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b0edfb96
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b0edfb96
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b0edfb96
Branch: refs/heads/current-release-line
Commit: b0edfb964382bd8e189291cabfc312d6d0d6a1b4
Parents: 4dcff2c
Author: Yash Sharma <ya...@gmail.com>
Authored: Tue Jun 23 15:11:07 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 23 15:11:07 2015 +0530
----------------------------------------------------------------------
.../lens/server/util/TestScannedPaths.java | 177 ++++++++++---------
1 file changed, 98 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0edfb96/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java b/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
index ac9faf2..9092420 100644
--- a/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
+++ b/lens-server/src/test/java/org/apache/lens/server/util/TestScannedPaths.java
@@ -22,7 +22,11 @@ package org.apache.lens.server.util;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
import org.apache.commons.io.FileUtils;
@@ -30,7 +34,6 @@ import org.testng.Assert;
import org.testng.annotations.Test;
import com.google.common.base.Joiner;
-import com.google.common.collect.Iterators;
import lombok.extern.slf4j.Slf4j;
@@ -39,25 +42,20 @@ import lombok.extern.slf4j.Slf4j;
public class TestScannedPaths {
public void testScannedPaths() throws Exception {
- File fileA = null, fileB = null;
String filenameA, filenameB, fileRegex;
String tempPath = "target/tempfiles/";
ScannedPaths sc;
- Iterator<String> iter = null;
try {
filenameA = "tempdata_a";
filenameB = "tempdata_b";
fileRegex = tempPath + "tempdata_*";
- fileA = createNewPath(tempPath + filenameA);
- fileB = createNewPath(tempPath + filenameB);
+ createNewPath(tempPath + filenameA);
+ createNewPath(tempPath + filenameB);
sc = new ScannedPaths(fileRegex, "file");
- Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains(filenameA));
- Assert.assertTrue(iter.next().contains(filenameB));
+ assertUnOrdered(sc, filenameA, filenameB);
/**
* Testing negative Scenario
@@ -74,8 +72,7 @@ public class TestScannedPaths {
}
public void testScannedPathsJarGlobOrder() throws Exception {
- File fileA = null, fileB = null, fileC = null;
- File jarFile = null, globFile = null;
+ File jarFile, globFile;
String filenameA, filenameB, filenameC, fileRegex;
String tempPath = "target/tempfiles/";
ScannedPaths sc;
@@ -88,9 +85,9 @@ public class TestScannedPaths {
filenameC = "tempdata_c.data";
fileRegex = tempPath;
- fileA = createNewPath(tempPath + filenameA);
- fileB = createNewPath(tempPath + filenameB);
- fileC = createNewPath(tempPath + filenameC);
+ createNewPath(tempPath + filenameA);
+ createNewPath(tempPath + filenameB);
+ createNewPath(tempPath + filenameC);
/** Test jar_order **/
jarFile = createNewPath(tempPath + "jar_order");
@@ -99,10 +96,8 @@ public class TestScannedPaths {
writer.println(filenameA);
writer.close();
sc = new ScannedPaths(fileRegex, "jar");
- Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains(filenameB));
- Assert.assertTrue(iter.next().contains(filenameA));
+
+ assertOrdered(sc, filenameB, filenameA);
/** Test glob_order **/
if (jarFile != null) {
@@ -115,11 +110,9 @@ public class TestScannedPaths {
writer.println(filenameC);
writer.close();
sc = new ScannedPaths(fileRegex, "file");
- Assert.assertEquals(Iterators.size(sc.iterator()), 3, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains(filenameB));
- Assert.assertTrue(iter.next().contains(filenameA));
- Assert.assertTrue(iter.next().contains(filenameC));
+
+ assertOrdered(sc, filenameB, filenameA, filenameC);
+
} catch (Exception e) {
Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
} finally {
@@ -135,7 +128,7 @@ public class TestScannedPaths {
* sourceDirPath/tempfiles/tempdata_a.jar
* sourceDirPath/tempfiles/tempdata_c.jar
* sourceDirPath/tempfiles/dir1/tempdata_a.jar
- * sourceDirPath/tempfiles/dir1/tempdata_b.jar
+ * sourceDirPath/tempfiles + "-duplicate"/dir1/tempdata_b.jar
* sourceDirPath/tempfiles/dir1/tempdata_c.data
* sourceDirPath/tempfiles/dir2/tempdata_a.data
* sourceDirPath/tempfiles/dir2/tempdata_b.data
@@ -152,7 +145,7 @@ public class TestScannedPaths {
* @param sourceDirPath
*/
private void createTempDirStructure1(String sourceDirPath) {
- File jarFile = null, globFile = null;
+ File jarFile, globFile;
String filenameA, filenameB, filenameC;
String tempPath = sourceDirPath + "/tempfiles/";
FileUtils.deleteQuietly(new File(tempPath));
@@ -247,8 +240,7 @@ public class TestScannedPaths {
* @param sourceDirPath
*/
private void createTempDirStructure2(String sourceDirPath) {
- File orderFile = null;
- String filenameA, filenameB, filenameC;
+ File orderFile;
String tempPath = sourceDirPath + "/parent_dir/";
FileUtils.deleteQuietly(new File(tempPath));
@@ -290,12 +282,16 @@ public class TestScannedPaths {
public void testScannedPathsMultipleJarGlobOrder() throws Exception {
ScannedPaths sc;
- Iterator<String> iter = null;
String tempPath = "target/test/";
- String filenameA = "tempdata_a";
- String filenameB = "tempdata_b";
- String filenameC = "tempdata_c";
+ String filenameAJar = "tempdata_a.jar";
+ String filenameBJar = "tempdata_b.jar";
+ String filenameCJar = "tempdata_c.jar";
+ String filenameCDupJar = "tempdata_c-duplicate.jar";
+ String filenameCDupJar2 = "tempdata_c-duplicate-2.jar";
+ String filenameAData = "tempdata_a.data";
+ String filenameBData = "tempdata_b.data";
+ String filenameCData = "tempdata_c.data";
String fileRegex1 = tempPath + "*";
String fileRegex2 = tempPath + "*/*";
@@ -304,21 +300,19 @@ public class TestScannedPaths {
createTempDirStructure1(tempPath);
sc = new ScannedPaths(fileRegex1, "jar");
- Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains(filenameC));
- Assert.assertTrue(iter.next().contains(filenameA));
+ assertUnOrdered(sc, filenameCJar, filenameAJar);
sc = new ScannedPaths(fileRegex2, "jar");
- Assert.assertEquals(Iterators.size(sc.iterator()), 7, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains(filenameB));
- Assert.assertTrue(iter.next().contains(filenameA));
- Assert.assertTrue(iter.next().contains(filenameC)); // Direct matched tempdata_c.jar
- Assert.assertTrue(iter.next().contains(filenameC));
- Assert.assertTrue(iter.next().contains(filenameC + "-duplicate"));
- Assert.assertTrue(iter.next().contains(filenameA));
- Assert.assertTrue(iter.next().contains(filenameA)); // Direct matched tempdata_a.jar
+
+ assertUnOrdered(sc,
+ filenameBJar,
+ filenameAJar,
+ filenameCJar, // Direct matched tempdata_c.jar
+ filenameCJar,
+ filenameCDupJar,
+ filenameAJar,
+ filenameAJar // Direct matched tempdata_a.jar
+ );
/** Remove jar_order files from temp dir **/
FileUtils.deleteQuietly(new File(tempPath + "tempfiles/dir1/jar_order"));
@@ -326,26 +320,24 @@ public class TestScannedPaths {
FileUtils.deleteQuietly(new File(tempPath + "tempfiles/dir3/jar_order"));
sc = new ScannedPaths(fileRegex1, "file");
- Assert.assertEquals(Iterators.size(sc.iterator()), 2, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains(filenameC));
- Assert.assertTrue(iter.next().contains(filenameA));
+ assertUnOrdered(sc, filenameCJar, filenameAJar);
sc = new ScannedPaths(fileRegex2, "file");
- Assert.assertEquals(Iterators.size(sc.iterator()), 12, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains(filenameC));
- Assert.assertTrue(iter.next().contains(filenameB));
- Assert.assertTrue(iter.next().contains(filenameA));
- Assert.assertTrue(iter.next().contains(filenameC)); // Direct matched tempdata_c.jar
- Assert.assertTrue(iter.next().contains(filenameC));
- Assert.assertTrue(iter.next().contains(filenameB));
- Assert.assertTrue(iter.next().contains(filenameA));
- Assert.assertTrue(iter.next().contains(filenameC));
- Assert.assertTrue(iter.next().contains(filenameC + "-duplicate"));
- Assert.assertTrue(iter.next().contains(filenameC + "-duplicate-2")); // Inner dir regex match
- Assert.assertTrue(iter.next().contains(filenameC + "-duplicate")); // Inner dir regex match
- Assert.assertTrue(iter.next().contains(filenameA)); // Direct matched tempdata_a.jar
+
+ assertUnOrdered(sc,
+ filenameCData, // dir1
+ filenameBJar, // dir1
+ filenameAJar, // dir1
+ filenameCJar, // Direct matched tempdata_c.jar
+ filenameCJar, // dir2
+ filenameBData, // dir2
+ filenameAData, // dir2
+ filenameCJar, // dir3
+ filenameCDupJar, //dir3
+ filenameCDupJar2, // dir3/inner
+ filenameCDupJar, // dir3/inner
+ filenameAJar // Direct matched tempdata_a.jar
+ );
} catch (Exception e) {
Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
@@ -357,7 +349,6 @@ public class TestScannedPaths {
public void testScannedPathsRecursive() throws Exception {
ScannedPaths sc;
- Iterator<String> iter = null;
String tempPath = "target/test/";
String fileRegex = tempPath + "parent_*";
@@ -366,14 +357,14 @@ public class TestScannedPaths {
createTempDirStructure2(tempPath);
sc = new ScannedPaths(fileRegex, "file");
- Assert.assertEquals(Iterators.size(sc.iterator()), 6, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains("jar_1"));
- Assert.assertTrue(iter.next().contains("jar_2"));
- Assert.assertTrue(iter.next().contains("jar_4"));
- Assert.assertTrue(iter.next().contains("jar_3"));
- Assert.assertTrue(iter.next().contains("jar_6"));
- Assert.assertTrue(iter.next().contains("jar_5"));
+
+ assertUnOrdered(sc,
+ "jar_1",
+ "jar_2",
+ "jar_4",
+ "jar_3",
+ "jar_6",
+ "jar_5");
/** Now also enforce order for c_dir **/
File orderFile = createNewPath(tempPath, "parent_dir/c_dir/glob_order");
@@ -383,14 +374,14 @@ public class TestScannedPaths {
writer.close();
sc = new ScannedPaths(fileRegex, "file");
- Assert.assertEquals(Iterators.size(sc.iterator()), 6, "Incorrect number of matches found");
- iter = sc.iterator();
- Assert.assertTrue(iter.next().contains("jar_1"));
- Assert.assertTrue(iter.next().contains("jar_2"));
- Assert.assertTrue(iter.next().contains("jar_4"));
- Assert.assertTrue(iter.next().contains("jar_3"));
- Assert.assertTrue(iter.next().contains("jar_5"));
- Assert.assertTrue(iter.next().contains("jar_6"));
+
+ assertUnOrdered(sc,
+ "jar_1",
+ "jar_2",
+ "jar_4",
+ "jar_3",
+ "jar_5",
+ "jar_6");
} catch (Exception e) {
Assert.fail("Exception while testing ScannedPaths : " + e.getMessage());
@@ -437,4 +428,32 @@ public class TestScannedPaths {
}
return f;
}
+
+ private void assertOrdered(ScannedPaths sc, String ... expectedPaths) {
+ List<String> actual = new ArrayList<>();
+ for (String path : sc) {
+ actual.add(path.substring(path.lastIndexOf("/") + 1, path.length()));
+ }
+
+ List<String> expected = new ArrayList<>();
+ for (String path : expectedPaths) {
+ expected.add(path);
+ }
+
+ Assert.assertEquals(actual, expected);
+ }
+
+ private void assertUnOrdered(ScannedPaths sc, String ... expectedPaths) {
+ Set<String> actual = new HashSet<>();
+ for (String path : sc) {
+ actual.add(path.substring(path.lastIndexOf("/") + 1, path.length()));
+ }
+
+ Set<String> expected = new HashSet<>();
+ for (String path : expectedPaths) {
+ expected.add(path);
+ }
+
+ Assert.assertEquals(actual, expected);
+ }
}
[43/51] [abbrv] incubator-lens git commit: LENS-487: Implementation
for consistent error response display for Lens Client
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
index 6b8bdda..8957aef 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
@@ -29,7 +29,7 @@ import javax.xml.bind.JAXBException;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.query.*;
-import org.apache.lens.api.response.LensResponse;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.regression.core.constants.QueryURL;
import org.apache.lens.regression.core.type.FormBuilder;
import org.apache.lens.regression.core.type.MapBuilder;
@@ -83,7 +83,7 @@ public class QueryHelper extends ServiceManagerHelper {
AssertUtil.assertSucceededResponse(response);
String queryHandleString = response.readEntity(String.class);
logger.info(queryHandleString);
- LensResponse successResponse = (LensResponse) Util.getObject(queryHandleString, LensResponse.class);
+ LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryHandleString, LensAPIResult.class);
QueryHandle queryHandle = (QueryHandle) successResponse.getData();
if (queryHandle == null) {
throw new LensException("Query Execute Failed");
@@ -138,7 +138,7 @@ public class QueryHelper extends ServiceManagerHelper {
AssertUtil.assertSucceededResponse(response);
String queryHandleString = response.readEntity(String.class);
logger.info(queryHandleString);
- LensResponse successResponse = (LensResponse) Util.getObject(queryHandleString, LensResponse.class);
+ LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryHandleString, LensAPIResult.class);
QueryHandleWithResultSet queryHandleWithResultSet = (QueryHandleWithResultSet) successResponse.getData();
if (queryHandleWithResultSet==null) {
throw new LensException("Query Execute Failed");
@@ -200,7 +200,7 @@ public class QueryHelper extends ServiceManagerHelper {
AssertUtil.assertSucceededResponse(response);
String queryHandleString = response.readEntity(String.class);
logger.info(queryHandleString);
- LensResponse successResponse = (LensResponse) Util.getObject(queryHandleString, LensResponse.class);
+ LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryHandleString, LensAPIResult.class);
QueryHandle queryHandle = (QueryHandle) successResponse.getData();
return queryHandle;
}
@@ -228,7 +228,7 @@ public class QueryHelper extends ServiceManagerHelper {
AssertUtil.assertSucceededResponse(response);
String queryPlanString = response.readEntity(String.class);
logger.info(queryPlanString);
- LensResponse successResponse = (LensResponse) Util.getObject(queryPlanString, LensResponse.class);
+ LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryPlanString, LensAPIResult.class);
QueryPlan queryPlan = (QueryPlan) successResponse.getData();
return queryPlan;
}
@@ -265,7 +265,7 @@ public class QueryHelper extends ServiceManagerHelper {
AssertUtil.assertSucceededResponse(response);
String queryCostString = response.readEntity(String.class);
logger.info(queryCostString);
- LensResponse successResponse = (LensResponse) Util.getObject(queryCostString, LensResponse.class);
+ LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryCostString, LensAPIResult.class);
QueryCost queryCost = (QueryCost) successResponse.getData();
if (queryCost == null) {
throw new LensException("Estimate Failed");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
index 2c35276..f78b7c3 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
@@ -167,7 +167,7 @@ public class DriverQueryStatus implements Serializable {
break;
}
- return new QueryStatus(progress, qstate, statusMessage, isResultSetAvailable, progressMessage, errorMessage);
+ return new QueryStatus(progress, qstate, statusMessage, isResultSetAvailable, progressMessage, errorMessage, null);
}
/**
@@ -179,7 +179,7 @@ public class DriverQueryStatus implements Serializable {
*/
public static QueryStatus createQueryStatus(QueryStatus.Status state, DriverQueryStatus dstatus) {
return new QueryStatus(dstatus.progress, state, dstatus.statusMessage, dstatus.isResultSetAvailable,
- dstatus.progressMessage, dstatus.errorMessage);
+ dstatus.progressMessage, dstatus.errorMessage, null);
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java b/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
index 9d8748c..49b118d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
@@ -29,8 +29,8 @@ import java.util.Arrays;
import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.error.LensError;
-import org.apache.lens.api.response.LensErrorTO;
-import org.apache.lens.api.response.LensResponse;
+import org.apache.lens.api.result.LensAPIResult;
+import org.apache.lens.api.result.LensErrorTO;
import org.apache.commons.lang.exception.ExceptionUtils;
@@ -53,7 +53,7 @@ public class LensException extends Exception {
* The lensResponse prepared by {@link #buildLensErrorResponse(ErrorCollection, String, String)}
* */
@Getter
- private LensResponse lensResponse;
+ private LensAPIResult lensAPIResult;
/**
* Constructs a new Lens Exception.
@@ -138,7 +138,7 @@ public class LensException extends Exception {
final LensError lensError = errorCollection.getLensError(errorCode);
final LensErrorTO lensErrorTO = buildLensErrorTO(errorCollection, lensError);
- lensResponse = LensResponse.composedOf(apiVersion, id, lensErrorTO, lensError.getHttpStatusCode());
+ lensAPIResult = LensAPIResult.composedOf(apiVersion, id, lensErrorTO, lensError.getHttpStatusCode());
}
public final LensErrorTO buildLensErrorTO(final ErrorCollection errorCollection) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensMultiCauseException.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensMultiCauseException.java b/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensMultiCauseException.java
index eb0df4d..af9e33a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensMultiCauseException.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensMultiCauseException.java
@@ -26,7 +26,7 @@ import java.util.LinkedList;
import java.util.List;
import org.apache.lens.api.error.ErrorCollection;
-import org.apache.lens.api.response.LensErrorTO;
+import org.apache.lens.api.result.LensErrorTO;
import com.google.common.collect.ImmutableList;
import lombok.AccessLevel;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index 9ba1843..6e7ab5d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -191,7 +191,7 @@ public class FinishedLensQuery {
qctx.setQueryHandle(QueryHandle.fromString(handle));
qctx.setEndTime(getEndTime());
qctx.setStatusSkippingTransitionTest(new QueryStatus(0.0, QueryStatus.Status.valueOf(getStatus()),
- getErrorMessage() == null ? "" : getErrorMessage(), getResult() != null, null, null));
+ getErrorMessage() == null ? "" : getErrorMessage(), getResult() != null, null, null, null));
qctx.getDriverStatus().setDriverStartTime(getDriverStartTime());
qctx.getDriverStatus().setDriverFinishTime(getDriverEndTime());
qctx.setResultSetPath(getResult());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 3e0c26c..2d3a19a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -213,7 +213,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
super(userQuery, user, qconf, conf, drivers, mergeDriverConf);
this.submissionTime = submissionTime;
this.queryHandle = new QueryHandle(UUID.randomUUID());
- this.status = new QueryStatus(0.0f, Status.NEW, "Query just got created", false, null, null);
+ this.status = new QueryStatus(0.0f, Status.NEW, "Query just got created", false, null, null, null);
this.priority = Priority.NORMAL;
this.lensConf = qconf;
this.conf = conf;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/main/java/org/apache/lens/server/LensServer.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
index e200bdf..d3dfad4 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
@@ -25,7 +25,7 @@ import java.util.logging.Logger;
import javax.ws.rs.core.UriBuilder;
-import org.apache.lens.api.response.LensJAXBContextResolver;
+import org.apache.lens.api.result.LensJAXBContextResolver;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.error.LensExceptionMapper;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/main/java/org/apache/lens/server/error/LensExceptionMapper.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/error/LensExceptionMapper.java b/lens-server/src/main/java/org/apache/lens/server/error/LensExceptionMapper.java
index 7240a64..1bb65ca 100644
--- a/lens-server/src/main/java/org/apache/lens/server/error/LensExceptionMapper.java
+++ b/lens-server/src/main/java/org/apache/lens/server/error/LensExceptionMapper.java
@@ -22,7 +22,7 @@ import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import javax.ws.rs.ext.Provider;
-import org.apache.lens.api.response.LensResponse;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.api.error.LensException;
@Provider
@@ -31,7 +31,7 @@ public class LensExceptionMapper implements ExceptionMapper<LensException> {
@Override
public Response toResponse(LensException exception) {
- final LensResponse lensResponse = exception.getLensResponse();
- return Response.status(lensResponse.getHttpStatusCode()).entity(lensResponse).build();
+ final LensAPIResult lensAPIResult = exception.getLensAPIResult();
+ return Response.status(lensAPIResult.getHttpStatusCode()).entity(lensAPIResult).build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/main/java/org/apache/lens/server/error/UnSupportedQuerySubmitOpException.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/error/UnSupportedQuerySubmitOpException.java b/lens-server/src/main/java/org/apache/lens/server/error/UnSupportedQuerySubmitOpException.java
index f0a0ab0..4ab26ec 100644
--- a/lens-server/src/main/java/org/apache/lens/server/error/UnSupportedQuerySubmitOpException.java
+++ b/lens-server/src/main/java/org/apache/lens/server/error/UnSupportedQuerySubmitOpException.java
@@ -22,9 +22,9 @@ import static org.apache.lens.server.error.LensServerErrorCode.UNSUPPORTED_QUERY
import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.error.LensError;
-import org.apache.lens.api.response.LensErrorTO;
+import org.apache.lens.api.query.SupportedQuerySubmitOperations;
+import org.apache.lens.api.result.LensErrorTO;
import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.query.SupportedQuerySubmitOperations;
public class UnSupportedQuerySubmitOpException extends LensException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index e41a0d2..8a1b7c8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -35,8 +35,10 @@ import javax.ws.rs.core.StreamingOutput;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
+import org.apache.lens.api.result.LensErrorTO;
import org.apache.lens.driver.cube.RewriteUtil;
import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.server.LensServerConf;
@@ -78,6 +80,8 @@ import org.codehaus.jackson.map.module.SimpleModule;
import com.google.common.collect.ImmutableList;
import lombok.Getter;
+import lombok.NonNull;
+
import lombok.extern.slf4j.Slf4j;
/**
@@ -165,7 +169,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* The query submitter runnable.
*/
- private final QuerySubmitter querySubmitterRunnable = new QuerySubmitter();
+ private final QuerySubmitter querySubmitterRunnable = new QuerySubmitter(LensServices.get().getErrorCollection());
/**
* The query submitter.
@@ -234,6 +238,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private final LogSegregationContext logSegregationContext;
+ private final ErrorCollection errorCollection = LensServices.get().getErrorCollection();
+
/**
* The driver event listener.
*/
@@ -480,6 +486,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
private boolean pausedForTest = false;
+ private final ErrorCollection errorCollection;
+
+ public QuerySubmitter(@NonNull final ErrorCollection errorCollection) {
+ this.errorCollection = errorCollection;
+ }
/*
* (non-Javadoc)
*
@@ -510,10 +521,17 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
// Check if we need to pass session's effective resources to selected driver
addSessionResourcesToDriver(ctx);
ctx.getSelectedDriver().executeAsync(ctx);
+ } catch (LensException e) {
+
+ LOG.error("Error launching query " + ctx.getQueryHandle(), e);
+ String reason = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
+ setFailedStatus(ctx, "Launching query failed", reason, e.buildLensErrorTO(this.errorCollection));
+ continue;
+
} catch (Exception e) {
log.error("Error launching query " + ctx.getQueryHandle(), e);
String reason = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
- setFailedStatus(ctx, "Launching query failed", reason);
+ setFailedStatus(ctx, "Launching query failed", reason, null);
continue;
} finally {
release(ctx.getLensSessionIdentifier());
@@ -601,9 +619,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* @param reason the reason
* @throws LensException the lens exception
*/
- void setFailedStatus(QueryContext ctx, String statusMsg, String reason) throws LensException {
+ void setFailedStatus(QueryContext ctx, String statusMsg, String reason, final LensErrorTO lensErrorTO)
+ throws LensException {
+
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0f, QueryStatus.Status.FAILED, statusMsg, false, null, reason));
+ ctx.setStatus(new QueryStatus(0.0f, QueryStatus.Status.FAILED, statusMsg, false, null, reason, lensErrorTO));
updateFinishedQuery(ctx, before);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
}
@@ -611,7 +631,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private void setLaunchedStatus(QueryContext ctx) throws LensException {
QueryStatus before = ctx.getStatus();
ctx.setStatus(new QueryStatus(ctx.getStatus().getProgress(), QueryStatus.Status.LAUNCHED, "launched on the driver",
- false, null, null));
+ false, null, null, null));
launchedQueries.add(ctx);
ctx.setLaunchTime(System.currentTimeMillis());
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
@@ -627,7 +647,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
private void setCancelledStatus(QueryContext ctx, String statusMsg) throws LensException {
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0f, QueryStatus.Status.CANCELED, statusMsg, false, null, null));
+ ctx.setStatus(new QueryStatus(0.0f, QueryStatus.Status.CANCELED, statusMsg, false, null, null, null));
updateFinishedQuery(ctx, before);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
}
@@ -654,7 +674,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
void setSuccessState(QueryContext ctx) throws LensException {
QueryStatus before = ctx.getStatus();
ctx.setStatus(new QueryStatus(1.0f, QueryStatus.Status.SUCCESSFUL, "Query is successful!", ctx
- .isResultAvailableInDriver(), null, null));
+ .isResultAvailableInDriver(), null, null, null));
updateFinishedQuery(ctx, before);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
}
@@ -678,8 +698,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
ctx.setStatus(ctx.getDriverStatus().toQueryStatus());
} catch (LensException exc) {
// Driver gave exception while updating status
- setFailedStatus(ctx, "Status update failed", exc.getMessage());
+
+ setFailedStatus(ctx, "Status update failed", exc.getMessage(), exc.buildLensErrorTO(this.errorCollection));
log.error("Status update failed for " + handle, exc);
+
}
// query is successfully executed by driver and
// if query result need not be persisted or there is no result available in driver, move the query to
@@ -831,8 +853,9 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
resultSets.remove(finished.getCtx().getQueryHandle());
}
fireStatusChangeEvent(finished.getCtx(),
- new QueryStatus(1f, Status.CLOSED, "Query purged", false, null, null), finished.getCtx().getStatus());
+ new QueryStatus(1f, Status.CLOSED, "Query purged", false, null, null, null), finished.getCtx().getStatus());
log.info("Query purged: " + finished.getCtx().getQueryHandle());
+
} catch (LensException e) {
incrCounter(QUERY_PURGER_COUNTER);
log.error("Error closing query ", e);
@@ -1546,7 +1569,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private QueryHandle executeAsyncInternal(LensSessionHandle sessionHandle, QueryContext ctx) throws LensException {
ctx.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0, QueryStatus.Status.QUEUED, "Query is queued", false, null, null));
+ ctx.setStatus(new QueryStatus(0.0, QueryStatus.Status.QUEUED, "Query is queued", false, null, null, null));
queuedQueries.add(ctx);
allQueries.put(ctx.getQueryHandle(), ctx);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index badde8c..15015d3 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -33,8 +33,7 @@ import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.query.*;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.annotations.MultiPurposeResource;
import org.apache.lens.server.api.error.LensException;
@@ -190,7 +189,7 @@ public class QueryServiceResource {
* @param timeoutmillis The timeout for the query, honored only in case of value {@value
* SubmitOp#EXECUTE_WITH_TIMEOUT} operation
* @param queryName human readable query name set by user (optional parameter)
- * @return {@link LensResponse} with DATA as {@link QueryHandle} in case of {@value SubmitOp#EXECUTE} operation.
+ * @return {@link LensAPIResult} with DATA as {@link QueryHandle} in case of {@value SubmitOp#EXECUTE} operation.
* {@link QueryPlan} in case of {@value SubmitOp#EXPLAIN} operation. {@link QueryHandleWithResultSet} in case
* {@value SubmitOp#EXECUTE_WITH_TIMEOUT} operation. {@link QueryCost} in case of
* {@value SubmitOp#ESTIMATE} operation.
@@ -200,7 +199,7 @@ public class QueryServiceResource {
@Consumes({MediaType.MULTIPART_FORM_DATA})
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
@MultiPurposeResource(formParamName = "operation")
- public LensResponse<QuerySubmitResult, NoErrorPayload> query(@FormDataParam("sessionid") LensSessionHandle sessionid,
+ public LensAPIResult<QuerySubmitResult> query(@FormDataParam("sessionid") LensSessionHandle sessionid,
@FormDataParam("query") String query, @FormDataParam("operation") String operation,
@FormDataParam("conf") LensConf conf, @DefaultValue("30000") @FormDataParam("timeoutmillis") Long timeoutmillis,
@DefaultValue("") @FormDataParam("queryName") String queryName) throws LensException {
@@ -230,7 +229,7 @@ public class QueryServiceResource {
throw new UnSupportedQuerySubmitOpException();
}
- return LensResponse.composedOf(null, requestId, result);
+ return LensAPIResult.composedOf(null, requestId, result);
} catch (LensException e) {
e.buildLensErrorResponse(errorCollection, null, requestId);
throw e;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index de7e120..c526c02 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -141,7 +141,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
metricsService.incrCounter(ResultFormatter.class, "formatting-errors");
LOG.warn("Exception while formatting result for " + queryHandle, e);
try {
- queryService.setFailedStatus(ctx, "Result formatting failed!", e.getMessage());
+ queryService.setFailedStatus(ctx, "Result formatting failed!", e.getMessage(), null);
} catch (LensException e1) {
LOG.error("Exception while setting failure for " + queryHandle, e1);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/main/java/org/apache/lens/server/query/SupportedQuerySubmitOperations.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/SupportedQuerySubmitOperations.java b/lens-server/src/main/java/org/apache/lens/server/query/SupportedQuerySubmitOperations.java
deleted file mode 100644
index 50858a6..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/query/SupportedQuerySubmitOperations.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.query;
-
-import static org.apache.lens.api.query.SubmitOp.*;
-
-import java.util.LinkedList;
-import java.util.List;
-
-import javax.xml.bind.annotation.*;
-
-import org.apache.lens.api.query.SubmitOp;
-
-import org.apache.commons.lang.StringUtils;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
-import lombok.EqualsAndHashCode;
-import lombok.ToString;
-
-@XmlRootElement
-@EqualsAndHashCode
-@ToString
-@XmlAccessorType(XmlAccessType.FIELD)
-public class SupportedQuerySubmitOperations {
-
- private static final String SEP = ", ";
-
- @XmlElementWrapper(name = "supportedOperations")
- @XmlElement(name = "operation")
- private List<String> supportedOps = new LinkedList<String>();
-
- public SupportedQuerySubmitOperations() {
- ImmutableSet<SubmitOp> supportedOps = Sets.immutableEnumSet(ESTIMATE, EXECUTE, EXPLAIN, EXECUTE_WITH_TIMEOUT);
-
- for (SubmitOp supportedOp : supportedOps) {
- this.supportedOps.add(supportedOp.toString().toLowerCase());
- }
- }
-
- public String getSupportedOperationsAsString() {
- return StringUtils.join(supportedOps, SEP);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
index c05a2e7..fbf524d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
@@ -34,8 +34,7 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.commons.io.FileUtils;
@@ -94,7 +93,7 @@ public final class LensTestUtil {
final QueryHandle handle = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
// wait till the query finishes
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
@@ -135,7 +134,7 @@ public final class LensTestUtil {
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
// wait till the query finishes
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
@@ -204,7 +203,7 @@ public final class LensTestUtil {
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
// wait till the query finishes
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
index b33b906..28ff920 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
@@ -35,8 +35,7 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.LensServices.SERVICE_MODE;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
@@ -182,7 +181,7 @@ public class TestServerMode extends LensAllApplicationJerseyTest {
QueryHandle qhandle = null;
try {
qhandle = queryTarget.request().post(Entity.entity(query, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
} catch (NotAllowedException nae) {
if (mode.equals(SERVICE_MODE.READ_ONLY)) {
// expected
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index 09f8dc7..537e750 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -41,8 +41,7 @@ import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.PersistentQueryResult;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.driver.hive.TestRemoteHiveDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
@@ -173,7 +172,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
new LensConf(), MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
@@ -259,7 +258,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
MediaType.APPLICATION_XML_TYPE));
QueryHandle handle = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
@@ -313,7 +312,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
new LensConf(), MediaType.APPLICATION_XML_TYPE));
handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
// Poll for second query, this should finish successfully
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/common/ErrorResponseExpectedData.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/common/ErrorResponseExpectedData.java b/lens-server/src/test/java/org/apache/lens/server/common/ErrorResponseExpectedData.java
index 3963a06..38b5ea0 100644
--- a/lens-server/src/test/java/org/apache/lens/server/common/ErrorResponseExpectedData.java
+++ b/lens-server/src/test/java/org/apache/lens/server/common/ErrorResponseExpectedData.java
@@ -24,8 +24,9 @@ import static org.testng.Assert.assertTrue;
import javax.ws.rs.core.Response;
-import org.apache.lens.api.response.LensErrorTO;
-import org.apache.lens.api.response.LensResponse;
+import org.apache.lens.api.result.LensAPIResult;
+import org.apache.lens.api.result.LensErrorTO;
+
public class ErrorResponseExpectedData {
@@ -44,14 +45,14 @@ public class ErrorResponseExpectedData {
/* Assert Equal Http Status Code */
assertEquals(response.getStatus(), expectedStatus.getStatusCode());
- LensResponse lensResponse = response.readEntity(LensResponse.class);
+ LensAPIResult lensAPIResult = response.readEntity(LensAPIResult.class);
/* Assert Equal LensErrorTO (stack trace gets excluded in equality check) */
- final LensErrorTO actualLensErrorTO = lensResponse.getLensErrorTO();
+ final LensErrorTO actualLensErrorTO = lensAPIResult.getLensErrorTO();
assertEquals(actualLensErrorTO, expectedLensErrorTO);
/* Assert receipt of valid stacktraces */
- assertTrue(lensResponse.areValidStackTracesPresent(), "Received Lens Response:" + lensResponse);
+ assertTrue(lensAPIResult.areValidStackTracesPresent(), "Received Lens Response:" + lensAPIResult);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
index a59252c..0400519 100644
--- a/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
+++ b/lens-server/src/test/java/org/apache/lens/server/common/TestDataUtils.java
@@ -62,21 +62,16 @@ public class TestDataUtils {
return "dimfield" + getRandomName();
}
- public static XCube createXCube(final String cubeName, final XDimAttribute... dimensions) {
-
- return createXCube(cubeName, Optional.<String>absent(), dimensions);
- }
-
- public static XCube createXCube(final String cubeName, final Optional<String> dtColName,
+ public static XCube createXCubeWithDummyMeasure(final String cubeName, final Optional<String> dtColName,
final XDimAttribute... dimensions) {
XDimAttributes xDimAttributes = createXDimAttributes(dimensions);
XMeasures xMeasures = createXMeasures(createDummyXMeasure());
- return createXCube(cubeName, dtColName, xDimAttributes, xMeasures);
+ return createXCubeWithDummyMeasure(cubeName, dtColName, xDimAttributes, xMeasures);
}
- public static XCube createXCube(final String cubeName, final Optional<String> dtColName,
+ public static XCube createXCubeWithDummyMeasure(final String cubeName, final Optional<String> dateColName,
final XDimAttributes xDimAttributes, final XMeasures xMeasures) {
XBaseCube cube = new XBaseCube();
@@ -84,11 +79,11 @@ public class TestDataUtils {
cube.setDimAttributes(xDimAttributes);
cube.setMeasures(xMeasures);
- if (dtColName.isPresent()) {
+ if (dateColName.isPresent()) {
XProperty xp = new XProperty();
xp.setName("cube."+cubeName+".timed.dimensions.list");
- xp.setValue(dtColName.get());
+ xp.setValue(dateColName.get());
XProperties xProperties = new XProperties();
xProperties.getProperty().add(xp);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java b/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
index 695403e..118970d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
@@ -38,8 +38,7 @@ import org.apache.lens.api.APIResult;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.LensAllApplicationJerseyTest;
import org.apache.lens.server.LensApplication;
import org.apache.lens.server.LensServices;
@@ -242,7 +241,7 @@ public class TestResourceMethodMetrics extends LensAllApplicationJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(),
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 720229d..91e9081 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -41,8 +41,9 @@ import org.apache.lens.api.metastore.XColumn;
import org.apache.lens.api.metastore.XCube;
import org.apache.lens.api.metastore.XDimAttribute;
import org.apache.lens.api.metastore.XFactTable;
-import org.apache.lens.api.response.LensErrorTO;
-import org.apache.lens.api.response.LensJAXBContextResolver;
+import org.apache.lens.api.query.SupportedQuerySubmitOperations;
+import org.apache.lens.api.result.LensErrorTO;
+import org.apache.lens.api.result.LensJAXBContextResolver;
import org.apache.lens.cube.error.ColUnAvailableInTimeRange;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensRequestContextInitFilter;
@@ -264,13 +265,13 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
/* Create a test cube with test dimension field having a start Date and end Date */
XDimAttribute testXDim = createXDimAttribute(testDimensionField, colStartDate, colEndDate);
- XCube xcube = createXCube(testCube, Optional.of("dt"), testXDim);
+ XCube xcube = createXCubeWithDummyMeasure(testCube, Optional.of("dt"), testXDim);
createCubeFailFast(target, sessionId, xcube);
/* Create a fact with test dimension field */
XColumn xColumn = createXColumn(testDimensionField);
XFactTable xFactTable = createXFactTableWithColumns(testFact, testCube, xColumn);
- createFact(target, sessionId, xFactTable);
+ createFactFailFast(target, sessionId, xFactTable);
/* Setup: End */
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
index d3fa02a..0b152bf 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
@@ -35,8 +35,7 @@ import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.query.QueryStatus.Status;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.LensTestUtil;
@@ -176,7 +175,7 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>(){}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>(){}).getData();
Assert.assertNotNull(handle);
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 62c24ba..655549b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -39,10 +39,9 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.error.LensCommonErrorCode;
import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
-import org.apache.lens.api.response.LensErrorTO;
-import org.apache.lens.api.response.LensJAXBContextResolver;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
+import org.apache.lens.api.result.LensErrorTO;
+import org.apache.lens.api.result.LensJAXBContextResolver;
import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.driver.hive.TestHiveDriver;
import org.apache.lens.server.LensJerseyTest;
@@ -236,7 +235,7 @@ public class TestQueryService extends LensJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {
+ new GenericType<LensAPIResult<QueryHandle>>() {
}).getData();
Assert.assertNotNull(handle);
@@ -287,7 +286,7 @@ public class TestQueryService extends LensJerseyTest {
long finishedQueries = metricsSvc.getFinishedQueries();
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {
+ new GenericType<LensAPIResult<QueryHandle>>() {
}).getData();
Assert.assertNotNull(handle);
@@ -379,7 +378,7 @@ public class TestQueryService extends LensJerseyTest {
final QueryPlan plan = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryPlan, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryPlan>>() {}).getData();
Assert.assertEquals(plan.getTablesQueried().size(), 1);
Assert.assertTrue(plan.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase()));
Assert.assertNull(plan.getPrepareHandle());
@@ -425,7 +424,7 @@ public class TestQueryService extends LensJerseyTest {
final QueryPlan plan = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryPlan, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryPlan>>() {}).getData();
Assert.assertTrue(plan.isError());
Assert.assertNotNull(plan.getErrorMsg());
Assert.assertTrue(plan.getErrorMsg()
@@ -676,7 +675,7 @@ public class TestQueryService extends LensJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(),
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
@@ -715,7 +714,7 @@ public class TestQueryService extends LensJerseyTest {
// test cancel query
final QueryHandle handle2 = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle2);
APIResult result = target.path(handle2.toString()).queryParam("sessionid", lensSessionId).request()
@@ -747,7 +746,7 @@ public class TestQueryService extends LensJerseyTest {
mp3.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle3 = target.request().post(Entity.entity(mp3, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
// Get query
ctx = target.path(handle3.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
@@ -961,7 +960,7 @@ public class TestQueryService extends LensJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
@@ -1016,7 +1015,7 @@ public class TestQueryService extends LensJerseyTest {
drop.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
final QueryHandle dropHandle = target.request().post(Entity.entity(drop, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(dropHandle);
@@ -1047,7 +1046,7 @@ public class TestQueryService extends LensJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
@@ -1075,7 +1074,7 @@ public class TestQueryService extends LensJerseyTest {
fetch.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle2 = target.request().post(Entity.entity(fetch, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle2);
@@ -1184,7 +1183,7 @@ public class TestQueryService extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
QueryHandleWithResultSet result = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandleWithResultSet, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
Assert.assertNotNull(result.getQueryHandle());
Assert.assertNotNull(result.getResult());
validatePersistentResult((PersistentQueryResult) result.getResult(), result.getQueryHandle(), true);
@@ -1203,7 +1202,7 @@ public class TestQueryService extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
result = target.request().post(Entity.entity(mp2, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandleWithResultSet, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
Assert.assertNotNull(result.getQueryHandle());
Assert.assertNotNull(result.getResult());
validateInmemoryResult((InMemoryQueryResult) result.getResult());
@@ -1230,7 +1229,7 @@ public class TestQueryService extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
QueryHandleWithResultSet result = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandleWithResultSet, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
Assert.assertNotNull(result.getQueryHandle());
Assert.assertNull(result.getResult());
@@ -1322,7 +1321,7 @@ public class TestQueryService extends LensJerseyTest {
final QueryCost result = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryCost, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryCost>>() {}).getData();
Assert.assertNotNull(result);
Assert.assertEquals(result.getEstimatedExecTimeMillis(), 1L);
Assert.assertEquals(result.getEstimatedResourceUsage(), 1.0);
@@ -1477,7 +1476,7 @@ public class TestQueryService extends LensJerseyTest {
final QueryCost queryCost = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryCost, NoErrorPayload>>() {
+ new GenericType<LensAPIResult<QueryCost>>() {
}).getData();
Assert.assertNotNull(queryCost);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
index f65805d..ea81e90 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
@@ -35,8 +35,7 @@ import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.query.QueryStatus.Status;
-import org.apache.lens.api.response.LensResponse;
-import org.apache.lens.api.response.NoErrorPayload;
+import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.LensTestUtil;
@@ -214,7 +213,7 @@ public class TestResultFormatting extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
QueryHandle handle = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d1d99708/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8744974..a896aea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,7 +81,7 @@
<!-- test -->
<testng.version>6.8</testng.version>
- <mockito.version>1.8.5</mockito.version>
+ <mockito.version>1.9.5</mockito.version>
<powermock.version>1.4.12</powermock.version>
<test.tmp.dir>${basedir}/target/tmp</test.tmp.dir>
[14/51] [abbrv] incubator-lens git commit: LENS-590: Support sync
option in "query results" command in CLI
Posted by jd...@apache.org.
LENS-590: Support sync option in "query results" command in CLI
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/41b293ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/41b293ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/41b293ca
Branch: refs/heads/current-release-line
Commit: 41b293ca143855f7e9afbdb974fdda0aa5a5fc1b
Parents: 06e1155
Author: Pranav Agarwal <pr...@gmail.com>
Authored: Wed Jun 3 16:31:31 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed Jun 3 16:31:31 2015 +0530
----------------------------------------------------------------------
.../lens/cli/commands/LensQueryCommands.java | 24 +++++++++++++++-----
.../apache/lens/cli/TestLensQueryCommands.java | 23 +++++++++++++++----
.../java/org/apache/lens/client/LensClient.java | 13 ++++++++---
.../org/apache/lens/client/LensStatement.java | 8 +++++--
src/site/apt/user/cli.apt | 2 +-
5 files changed, 53 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/41b293ca/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 6c4a1b5..1eb7ed6 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -264,12 +264,16 @@ public class LensQueryCommands extends BaseLensCommand {
* @return the query results
*/
@CliCommand(value = "query results",
- help = "get results of async query with query handle <query_handle>. Can "
- + "optionally save the results to a file by providing <save_location>")
+ help = "get results of query with query handle <query_handle>. If async is false "
+ + "then wait till the query execution is completed, it's by default true. "
+ + "Can optionally save the results to a file by providing <save_location>.")
public String getQueryResults(
@CliOption(key = {"", "query_handle"}, mandatory = true, help = "<query_handle>") String qh,
- @CliOption(key = {"save_location"}, mandatory = false, help = "<save_location>") String location) {
+ @CliOption(key = {"save_location"}, mandatory = false, help = "<save_location>") String location,
+ @CliOption(key = {"async"}, mandatory = false, unspecifiedDefaultValue = "true",
+ help = "<async>") boolean async) {
QueryHandle queryHandle = new QueryHandle(UUID.fromString(qh));
+ LensClient.LensClientResultSetWithStats results;
try {
String prefix = "";
if (StringUtils.isNotBlank(location)) {
@@ -285,8 +289,14 @@ public class LensQueryCommands extends BaseLensCommand {
}
return "Saved to " + location;
} else {
- LensClient.LensClientResultSetWithStats results = getClient().getAsyncResults(queryHandle);
- if (results.getResultSet().getResult() instanceof InMemoryQueryResult) {
+ if (async) {
+ results = getClient().getAsyncResults(queryHandle);
+ } else {
+ results = getClient().getSyncResults(queryHandle);
+ }
+ if (results.getResultSet() == null) {
+ return "Resultset not yet available";
+ } else if (results.getResultSet().getResult() instanceof InMemoryQueryResult) {
location = getValidPath(location + File.separator + qh + ".csv", false, false);
try (OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(location),
Charset.defaultCharset())) {
@@ -298,8 +308,10 @@ public class LensQueryCommands extends BaseLensCommand {
+ formatResultSet(results);
}
}
- } else {
+ } else if (async) {
return formatResultSet(getClient().getAsyncResults(queryHandle));
+ } else {
+ return formatResultSet(getClient().getSyncResults(queryHandle));
}
} catch (Throwable t) {
return t.getMessage();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/41b293ca/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index 32a89ac..d5fd999 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -83,6 +83,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
assertTrue(resDir.exists() || resDir.mkdirs());
testExecuteSyncQuery(qCom);
testExecuteAsyncQuery(qCom);
+ testSyncResults(qCom);
testExplainQuery(qCom);
testExplainFailQuery(qCom);
testPreparedQuery(qCom);
@@ -98,6 +99,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
assertEquals(result, "No prepared queries");
testExecuteSyncQuery(qCom);
testExecuteAsyncQuery(qCom);
+ testSyncResults(qCom);
testExplainQuery(qCom);
testExplainFailQuery(qCom);
testPreparedQuery(qCom);
@@ -140,11 +142,11 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
log.debug("Prepared Query Status is " + status);
assertTrue(status.contains("Status : SUCCESSFUL"));
- result = qCom.getQueryResults(handle, null);
+ result = qCom.getQueryResults(handle, null, true);
log.debug("Prepared Query Result is " + result);
assertTrue(result.contains("1\tfirst"));
// Fetch again.
- result = qCom.getQueryResults(handle, null);
+ result = qCom.getQueryResults(handle, null, true);
log.debug("Prepared Query Result is " + result);
assertTrue(result.contains("1\tfirst"));
@@ -248,7 +250,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
details = qCom.getDetails(qh);
assertTrue(details.contains("driverQuery"));
- result = qCom.getQueryResults(qh, null);
+ result = qCom.getQueryResults(qh, null, true);
assertTrue(result.contains("1\tfirst"));
downloadResult(qCom, qh, result);
@@ -298,7 +300,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
}
private void downloadResult(LensQueryCommands qCom, String qh, String expected) throws IOException {
- assertTrue(qCom.getQueryResults(qh, resDir.getAbsolutePath()).contains("Saved"));
+ assertTrue(qCom.getQueryResults(qh, resDir.getAbsolutePath(), true).contains("Saved"));
assertEquals(readFile(resDir.getAbsolutePath() + File.separator + qh + ".csv").trim(), expected.trim());
}
private String readFile(String path) throws FileNotFoundException {
@@ -371,6 +373,17 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
}
/**
+ * Test execute sync results.
+ *
+ * @param qCom the q com
+ */
+ private void testSyncResults(LensQueryCommands qCom) {
+ String sql = "cube select id,name from test_dim";
+ String qh = qCom.executeQuery(sql, true, "testQuery4");
+ String result = qCom.getQueryResults(qh, null, false);
+ assertTrue(result.contains("1\tfirst"), result);
+ }
+ /**
* Test purged finished result set.
*
* @param qCom the q com
@@ -387,7 +400,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
}
assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL"));
- String result = qCom.getQueryResults(qh, null);
+ String result = qCom.getQueryResults(qh, null, true);
System.out.println("@@ RESULT " + result);
assertNotNull(result);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/41b293ca/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index cd8bf16..7c82619 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -133,7 +133,10 @@ public class LensClient {
return new LensClientResultSetWithStats(result, statement.getQuery());
}
- private LensClientResultSetWithStats getResultsFromHandle(QueryHandle q) {
+ private LensClientResultSetWithStats getResultsFromHandle(QueryHandle q, boolean async) {
+ if (!async) {
+ statement.waitForQueryToComplete(q);
+ }
LensQuery query = statement.getQuery(q);
if (query.getStatus().getStatus()
== QueryStatus.Status.FAILED) {
@@ -148,7 +151,11 @@ public class LensClient {
}
public LensClientResultSetWithStats getAsyncResults(QueryHandle q) {
- return getResultsFromHandle(q);
+ return getResultsFromHandle(q, true);
+ }
+
+ public LensClientResultSetWithStats getSyncResults(QueryHandle q) {
+ return getResultsFromHandle(q, false);
}
public Response getHttpResults() {
@@ -524,7 +531,7 @@ public class LensClient {
public LensClientResultSetWithStats getResultsFromPrepared(QueryPrepareHandle phandle, String queryName) {
QueryHandle qh = statement.executeQuery(phandle, true, queryName);
- return getResultsFromHandle(qh);
+ return getResultsFromHandle(qh, true);
}
public QueryHandle executePrepared(QueryPrepareHandle phandle, String queryName) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/41b293ca/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
index 4472a80..e2693c9 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
@@ -30,10 +30,13 @@ import javax.ws.rs.core.Response;
import org.apache.lens.api.APIResult;
import org.apache.lens.api.query.*;
+import org.apache.lens.api.query.QueryStatus.Status;
+
import org.apache.lens.api.response.LensJAXBContextResolver;
import org.apache.lens.api.response.LensResponse;
import org.apache.lens.api.response.NoErrorPayload;
+
import org.apache.commons.lang.StringUtils;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
@@ -187,9 +190,10 @@ public class LensStatement {
*
* @param handle the handle
*/
- private void waitForQueryToComplete(QueryHandle handle) {
+ public void waitForQueryToComplete(QueryHandle handle) {
query = getQuery(handle);
- while (!query.getStatus().finished()) {
+ while (!query.getStatus().finished()
+ && !(query.getStatus().toString().equals(Status.CLOSED.toString()))) {
query = getQuery(handle);
try {
Thread.sleep(connection.getLensConnectionParams().getQueryPollInterval());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/41b293ca/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index 2016bfd..a5bc056 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -324,7 +324,7 @@ User CLI Commands
*--+--+
|query list [--state \<query-status\>] [--name \<query-name\>] [--user \<user-who-submitted-query\>] [--fromDate \<submission-time-is-after\>] [--toDate \<submission-time-is-before\>]|Get all queries. Various filter options can be provided(optionally), as can be seen from the command syntax|
*--+--+
-|query results [--query_handle] \<query_handle\> [--save_location \<save_location\>]|get results of async query with query handle <<<query_handle>>>. Can optionally save the results to a file by providing <<<save_location>>>|
+|query results [--query_handle] \<query_handle\> [--save_location \<save_location\>] [--async \<async\>]|get results of query with query handle <<<query_handle>>>. If async is false then wait till the query execution is completed, it's by default true. Can optionally save the results to a file by providing <<<save_location>>>.|
*--+--+
|query status [--query_handle] \<query_handle\>|Fetch status of executed query having query handle <<<query_handle>>>|
*--+--+
[34/51] [abbrv] incubator-lens git commit: LENS-517 : Add cascade
option to drop db from lens cli
Posted by jd...@apache.org.
LENS-517 : Add cascade option to drop db from lens cli
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/2a61a7b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/2a61a7b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/2a61a7b5
Branch: refs/heads/current-release-line
Commit: 2a61a7b54c7515dfed7de29bce8206fe34208b4a
Parents: 44a6f51
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jun 18 11:44:25 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Jun 18 11:44:25 2015 +0530
----------------------------------------------------------------------
.../lens/cli/commands/LensDatabaseCommands.java | 8 +--
.../lens/cli/TestLensDatabaseCommands.java | 51 ++++++++++++++------
.../java/org/apache/lens/client/LensClient.java | 10 ++--
.../lens/client/LensClientSingletonWrapper.java | 37 +++++++-------
.../apache/lens/client/LensMetadataClient.java | 3 +-
.../org/apache/lens/client/TestLensClient.java | 2 +-
6 files changed, 67 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2a61a7b5/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java
index 800c0b5..1de24a4 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java
@@ -86,9 +86,9 @@ public class LensDatabaseCommands extends LensCRUDCommand {
* @return the string
*/
@CliCommand(value = "drop database", help = "drop a database with specified name")
- public String dropDatabase(@CliOption(key = {"", "db"}, mandatory = true,
- help = "<database-name>") String database) {
- return drop(database, false);
+ public String dropDatabase(@CliOption(key = {"", "db"}, mandatory = true, help = "<database-name>") String database,
+ @CliOption(key = "cascade", specifiedDefaultValue = "true", unspecifiedDefaultValue = "false") boolean cascade) {
+ return drop(database, cascade);
}
@Override
@@ -116,6 +116,6 @@ public class LensDatabaseCommands extends LensCRUDCommand {
@Override
protected APIResult doDelete(String name, boolean cascade) {
- return getClient().dropDatabase(name);
+ return getClient().dropDatabase(name, cascade);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2a61a7b5/lens-cli/src/test/java/org/apache/lens/cli/TestLensDatabaseCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDatabaseCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDatabaseCommands.java
index f8958c4..b6f96e6 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDatabaseCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDatabaseCommands.java
@@ -18,12 +18,17 @@
*/
package org.apache.lens.cli;
+import static org.testng.Assert.*;
+
+import java.io.File;
+import java.net.URISyntaxException;
+
+import org.apache.lens.cli.commands.LensCubeCommands;
import org.apache.lens.cli.commands.LensDatabaseCommands;
import org.apache.lens.client.LensClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testng.Assert;
import org.testng.annotations.Test;
/**
@@ -38,28 +43,44 @@ public class TestLensDatabaseCommands extends LensCliApplicationTest {
* Test database commands.
*/
@Test
- public void testDatabaseCommands() {
+ public void testDatabaseCommands() throws URISyntaxException {
LensClient client = new LensClient();
LensDatabaseCommands command = new LensDatabaseCommands();
+ LensCubeCommands cubeCommand = new LensCubeCommands();
command.setClient(client);
+ cubeCommand.setClient(client);
+ boolean cascade = true;
+ for(int i = 0; i < 4; i++, cascade = !cascade) {
+ testDrop(command, cubeCommand, cascade);
+ }
+ }
+ private void testDrop(LensDatabaseCommands command, LensCubeCommands cubeCommand, boolean cascade)
+ throws URISyntaxException {
String myDatabase = "my_db";
- String databaseList = command.showAllDatabases();
- Assert.assertFalse(databaseList.contains(myDatabase));
+ assertFalse(command.showAllDatabases().contains(myDatabase));
+ assertFalse(cubeCommand.showCubes().contains("sample_cube"));
String result;
command.createDatabase(myDatabase, false);
-
- databaseList = command.showAllDatabases();
- Assert.assertTrue(databaseList.contains(myDatabase));
-
+ assertTrue(command.showAllDatabases().contains(myDatabase));
result = command.switchDatabase(myDatabase);
- Assert.assertEquals("Successfully switched to my_db", result);
-
+ assertEquals(result, "Successfully switched to my_db");
+ if (cascade) {
+ String createOutput = cubeCommand.createCube(
+ new File(TestLensDatabaseCommands.class.getClassLoader().getResource("sample-cube.xml").toURI())
+ .getAbsolutePath());
+ assertEquals(createOutput, "succeeded");
+ assertTrue(cubeCommand.showCubes().contains("sample_cube"));
+ }
result = command.switchDatabase("default");
- Assert.assertEquals("Successfully switched to default", result);
-
- result = command.dropDatabase(myDatabase);
- Assert.assertEquals(result, "succeeded");
+ assertEquals(result, "Successfully switched to default");
+ assertFalse(cubeCommand.showCubes().contains("sample_cube"));
+ if (cascade) {
+ assertEquals(command.dropDatabase(myDatabase, false), "failed");
+ }
+ result = command.dropDatabase(myDatabase, cascade);
+ assertEquals(result, "succeeded");
+ assertFalse(command.showAllDatabases().contains(myDatabase));
+ assertFalse(cubeCommand.showCubes().contains("sample_cube"));
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2a61a7b5/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index ca017b8..5c3daa8 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -76,6 +76,10 @@ public class LensClient {
this(cred.getUsername(), cred.getPassword());
}
+ public LensMetadataClient getMetadataClient() {
+ return mc;
+ }
+
public QueryHandle executeQueryAsynch(String sql, String queryName) {
LOG.debug("Executing query " + sql);
statement.execute(sql, false, queryName);
@@ -270,9 +274,9 @@ public class LensClient {
return result.getStatus() == APIResult.Status.SUCCEEDED;
}
- public APIResult dropDatabase(String database) {
- LOG.debug("Dropping database " + database);
- APIResult result = mc.dropDatabase(database);
+ public APIResult dropDatabase(String database, boolean cascade) {
+ LOG.debug("Dropping database " + database + ", cascade: " + cascade);
+ APIResult result = mc.dropDatabase(database, cascade);
LOG.debug("Return status of dropping " + database + " result " + result);
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2a61a7b5/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
index d9ab5a1..14a0690 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
@@ -72,25 +72,24 @@ public class LensClientSingletonWrapper {
}
public LensClient getClient() {
- if (client != null) {
- return client;
- }
- try {
- client = new LensClient();
- } catch (LensClientServerConnectionException e) {
- if (e.getErrorCode() != 401) {
- explainFailedAttempt(e);
- throw e;
- }
- // Connecting without password prompt failed.
- for (int i = 0; i < MAX_RETRIES; i++) {
- try {
- client = new LensClient(Credentials.prompt());
- break;
- } catch (LensClientServerConnectionException lensClientServerConnectionException) {
- explainFailedAttempt(lensClientServerConnectionException);
- if (i == MAX_RETRIES - 1) {
- throw lensClientServerConnectionException;
+ if (client == null) {
+ try {
+ client = new LensClient();
+ } catch (LensClientServerConnectionException e) {
+ if (e.getErrorCode() != 401) {
+ explainFailedAttempt(e);
+ throw e;
+ }
+ // Connecting without password prompt failed.
+ for (int i = 0; i < MAX_RETRIES; i++) {
+ try {
+ client = new LensClient(Credentials.prompt());
+ break;
+ } catch (LensClientServerConnectionException lensClientServerConnectionException) {
+ explainFailedAttempt(lensClientServerConnectionException);
+ if (i == MAX_RETRIES - 1) {
+ throw lensClientServerConnectionException;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2a61a7b5/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index c58f5ee..b4e0552 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -138,11 +138,10 @@ public class LensMetadataClient {
public APIResult dropDatabase(String database, boolean cascade) {
WebTarget target = getMetastoreWebTarget();
- APIResult result = target.path("databases").path(database)
+ return target.path("databases").path(database)
.queryParam("sessionid", this.connection.getSessionHandle())
.queryParam("cascade", cascade)
.request().delete(APIResult.class);
- return result;
}
public APIResult dropDatabase(String database) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2a61a7b5/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
index 24f3473..35d52db 100644
--- a/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
+++ b/lens-client/src/test/java/org/apache/lens/client/TestLensClient.java
@@ -78,7 +78,7 @@ public class TestLensClient extends LensAllApplicationJerseyTest {
"current database");
client.createDatabase("testclientdb", true);
Assert.assertTrue(client.getAllDatabases().contains("testclientdb"));
- client.dropDatabase("testclientdb");
+ client.dropDatabase("testclientdb", false);
Assert.assertFalse(client.getAllDatabases().contains("testclientdb"));
}
}
[35/51] [abbrv] incubator-lens git commit: LENS-616 : Altering table
should not remove existing properties (amareshwari)
Posted by jd...@apache.org.
LENS-616 : Altering table should not remove existing properties (amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ead42f8e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ead42f8e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ead42f8e
Branch: refs/heads/current-release-line
Commit: ead42f8e1929bba9c3e71e9c91f348a54a6f233b
Parents: 2a61a7b
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Thu Jun 18 14:22:55 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Jun 18 14:22:55 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/cube/metadata/Storage.java | 5 ++++-
.../lens/cube/metadata/TestCubeMetastoreClient.java | 15 +++++++++++++--
2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ead42f8e/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index d3c9974..c754042 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -129,7 +129,10 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
*/
public Table getStorageTable(Hive client, Table parent, StorageTableDesc crtTbl) throws HiveException {
String storageTableName = MetastoreUtil.getStorageTableName(parent.getTableName(), this.getPrefix());
- Table tbl = client.newTable(storageTableName);
+ Table tbl = client.getTable(storageTableName, false);
+ if (tbl == null) {
+ tbl = client.newTable(storageTableName);
+ }
tbl.getTTable().setSd(new StorageDescriptor(parent.getTTable().getSd()));
if (crtTbl.getTblProps() != null) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ead42f8e/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index d481f93..18b6c87 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -1209,7 +1209,6 @@ public class TestCubeMetastoreClient {
Assert.assertEquals(timeline1Temp.getClass(), EndsAndHolesPartitionTimeline.class);
Assert.assertEquals(timeline2Temp.getClass(), StoreAllPartitionTimeline.class);
-
Assert.assertEquals(client.getAllParts(c1TableName).size(), 3);
Assert.assertEquals(client.getAllParts(c2TableName).size(), 3);
@@ -1297,9 +1296,21 @@ public class TestCubeMetastoreClient {
assertTimeline(timelineEt, timelineEtC2, UpdatePeriod.HOURLY, nowMinus5, nowPlus1, nowMinus4, nowMinus3);
assertTimeline(timelineIt, timelineItC2, UpdatePeriod.HOURLY, nowMinus5, nowPlus1, nowMinus4, nowMinus3, nowMinus2);
-
assertNoPartitionNamedLatest(c1TableName, partColNames);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
+ Assert.assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get(
+ MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
+ Assert.assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get(
+ MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
+
+ // alter tables and see timeline still exists
+ client.alterCubeFactTable(factName, cubeFact, storageTables);
+ assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
+ Assert.assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get(
+ MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
+ Assert.assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get(
+ MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
+
client.dropPartition(cubeFact.getName(), c1, timeParts5, null, UpdatePeriod.HOURLY);
client.dropPartition(cubeFact.getName(), c2, timeParts5, null, UpdatePeriod.HOURLY);
[02/51] [abbrv] incubator-lens git commit: LENS-551: Cleanup
testcases of exception stack traces
Posted by jd...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
index c08a5b1..51344ce 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
@@ -42,8 +42,6 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.query.QueryServiceResource;
import org.apache.lens.server.session.SessionResource;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -59,11 +57,12 @@ import org.testng.annotations.BeforeMethod;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
-
+@Slf4j
@Test
public class TestMLResource extends LensJerseyTest {
- private static final Log LOG = LogFactory.getLog(TestMLResource.class);
+
private static final String TEST_DB = "default";
private WebTarget mlTarget;
@@ -113,7 +112,7 @@ public class TestMLResource extends LensJerseyTest {
hive.dropDatabase(TEST_DB);
} catch (Exception exc) {
// Ignore drop db exception
- exc.printStackTrace();
+ log.error("Exception while dropping database.", exc);
}
mlClient.close();
}
@@ -158,13 +157,13 @@ public class TestMLResource extends LensJerseyTest {
Assert.assertFalse(params.isEmpty());
for (String key : params.keySet()) {
- LOG.info("## Param " + key + " help = " + params.get(key));
+ log.info("## Param " + key + " help = " + params.get(key));
}
}
@Test
public void trainAndEval() throws Exception {
- LOG.info("Starting train & eval");
+ log.info("Starting train & eval");
final String algoName = MLUtils.getAlgoName(NaiveBayesAlgo.class);
HiveConf conf = new HiveConf();
String tableName = "naivebayes_training_table";
@@ -177,7 +176,7 @@ public class TestMLResource extends LensJerseyTest {
String[] features = { "feature_1", "feature_2", "feature_3" };
String outputTable = "naivebayes_eval_table";
- LOG.info("Creating training table from file "
+ log.info("Creating training table from file "
+ sampleDataFileURI.toString());
Map<String, String> tableParams = new HashMap<String, String>();
@@ -185,7 +184,7 @@ public class TestMLResource extends LensJerseyTest {
ExampleUtils.createTable(conf, TEST_DB, tableName,
sampleDataFileURI.toString(), labelColumn, tableParams, features);
} catch (HiveException exc) {
- exc.printStackTrace();
+ log.error("Hive exception encountered.", exc);
}
MLTask.Builder taskBuilder = new MLTask.Builder();
@@ -198,7 +197,7 @@ public class TestMLResource extends LensJerseyTest {
MLTask task = taskBuilder.build();
- LOG.info("Created task " + task.toString());
+ log.info("Created task " + task.toString());
task.run();
Assert.assertEquals(task.getTaskState(), MLTask.State.SUCCESSFUL);
@@ -216,7 +215,7 @@ public class TestMLResource extends LensJerseyTest {
MLTask anotherTask = taskBuilder.build();
- LOG.info("Created second task " + anotherTask.toString());
+ log.info("Created second task " + anotherTask.toString());
anotherTask.run();
String secondModelID = anotherTask.getModelID();
@@ -233,7 +232,7 @@ public class TestMLResource extends LensJerseyTest {
int i = 0;
Set<String> partReports = new HashSet<String>();
for (Partition part : partitions) {
- LOG.info("@@PART#" + i + " " + part.getSpec().toString());
+ log.info("@@PART#" + i + " " + part.getSpec().toString());
partReports.add(part.getSpec().get("part_testid"));
}
@@ -243,7 +242,7 @@ public class TestMLResource extends LensJerseyTest {
Assert.assertTrue(partReports.contains(secondReportID), secondReportID
+ " second partition not there");
- LOG.info("Completed task run");
+ log.info("Completed task run");
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/LensRequestListener.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensRequestListener.java b/lens-server/src/main/java/org/apache/lens/server/LensRequestListener.java
index 7bf5d09..cb226d5 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensRequestListener.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensRequestListener.java
@@ -27,9 +27,12 @@ import org.apache.lens.server.api.metrics.MetricsService;
import org.glassfish.jersey.server.monitoring.RequestEvent;
import org.glassfish.jersey.server.monitoring.RequestEventListener;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Event listener used for metrics in errors.
*/
+@Slf4j
public class LensRequestListener implements RequestEventListener {
/** The Constant HTTP_REQUESTS_STARTED. */
@@ -108,7 +111,7 @@ public class LensRequestListener implements RequestEventListener {
metrics.incrCounter(LensRequestListener.class, HTTP_CLIENT_ERROR);
} else {
metrics.incrCounter(LensRequestListener.class, HTTP_UNKOWN_ERROR);
- error.printStackTrace();
+ log.error("Encountered HTTP exception", error);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java b/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
index 61a0dd8..d360978 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServletContextListener.java
@@ -26,11 +26,14 @@ import org.apache.hive.service.CompositeService;
import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.PropertyConfigurator;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Initialize the webapp.
*
* @see LensServletContextEvent
*/
+@Slf4j
public class LensServletContextListener implements ServletContextListener {
/** The Constant LOG_PROPERTIES_FILE_KEY. */
@@ -58,7 +61,7 @@ public class LensServletContextListener implements ServletContextListener {
} catch (Exception exc) {
// Try basic configuration
System.err.println("WARNING - log4j property configurator gave error, falling back to basic configurator");
- exc.printStackTrace();
+ log.error("WARNING - log4j property configurator gave error, falling back to basic configurator", exc);
BasicConfigurator.configure();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 4597614..64f6cd4 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -41,14 +41,14 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hive.service.cli.CLIService;
import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class CubeMetastoreServiceImpl extends LensService implements CubeMetastoreService {
- public static final Logger LOG = LogManager.getLogger(CubeMetastoreServiceImpl.class);
public CubeMetastoreServiceImpl(CLIService cliService) {
super(NAME, cliService);
@@ -86,7 +86,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
if (!Hive.get(getSession(sessionid).getHiveConf()).databaseExists(database)) {
throw new NotFoundException("Database " + database + " does not exist");
}
- LOG.info("Set database " + database);
+ log.info("Set database " + database);
getSession(sessionid).setCurrentDatabase(database);
} catch (HiveException e) {
throw new LensException(e);
@@ -106,7 +106,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
Hive.get(getSession(sessionid).getHiveConf()).dropDatabase(database, false, true, cascade);
- LOG.info("Database dropped " + database + " cascade? " + true);
+ log.info("Database dropped " + database + " cascade? " + true);
} catch (HiveException e) {
throw new LensException(e);
} catch (NoSuchObjectException e) {
@@ -137,7 +137,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
} finally {
release(sessionid);
}
- LOG.info("Database created " + database);
+ log.info("Database created " + database);
}
/**
@@ -195,7 +195,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
Cube parent = cube instanceof XDerivedCube ? (Cube) msClient.getCube(
((XDerivedCube) cube).getParent()) : null;
msClient.createCube(JAXBUtils.hiveCubeFromXCube(cube, parent));
- LOG.info("Created cube " + cube.getName());
+ log.info("Created cube " + cube.getName());
} catch (HiveException e) {
throw new LensException(e);
} catch (ParseException e) {
@@ -237,7 +237,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).dropCube(cubeName);
- LOG.info("Dropped cube " + cubeName);
+ log.info("Dropped cube " + cubeName);
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -259,7 +259,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
Cube parent = cube instanceof XDerivedCube ? (Cube) msClient.getCube(
((XDerivedCube) cube).getParent()) : null;
msClient.alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent));
- LOG.info("Cube updated " + cube.getName());
+ log.info("Cube updated " + cube.getName());
} catch (HiveException e) {
throw new LensException(e);
} catch (ParseException e) {
@@ -288,7 +288,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
- LOG.info("# Columns: " + columns);
+ log.info("# Columns: " + columns);
getClient(sessionid).createCubeDimensionTable(xDimTable.getDimensionName(),
dimTblName,
columns,
@@ -296,7 +296,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
updatePeriodMap,
properties,
storageDesc);
- LOG.info("Dimension Table created " + xDimTable.getTableName());
+ log.info("Dimension Table created " + xDimTable.getTableName());
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -309,9 +309,9 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).dropDimensionTable(dimTblName, cascade);
- LOG.info("Dropped dimension table " + dimTblName + " cascade? " + cascade);
+ log.info("Dropped dimension table " + dimTblName + " cascade? " + cascade);
} catch (HiveException e) {
- LOG.error("@@@@ Got HiveException: >>>>>>>" + e.getMessage() + "<<<<<<<<<");
+ log.error("@@@@ Got HiveException: >>>>>>>" + e.getMessage() + "<<<<<<<<<", e);
throw new LensException(e);
} finally {
release(sessionid);
@@ -352,7 +352,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
getClient(sessionid).alterCubeDimensionTable(dimensionTable.getTableName(),
JAXBUtils.cubeDimTableFromDimTable(dimensionTable),
JAXBUtils.storageTableMapFromXStorageTables(dimensionTable.getStorageTables()));
- LOG.info("Updated dimension table " + dimensionTable.getTableName());
+ log.info("Updated dimension table " + dimensionTable.getTableName());
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -387,7 +387,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
msClient.addStorage(dimTable, storageTable.getStorageName(), period,
JAXBUtils.storageTableDescFromXStorageTableDesc(storageTable.getTableDesc()));
- LOG.info("Added storage " + storageTable.getStorageName() + " for dimension table " + dimTblName
+ log.info("Added storage " + storageTable.getStorageName() + " for dimension table " + dimTblName
+ " with update period " + period);
} catch (HiveException exc) {
throw new LensException(exc);
@@ -407,10 +407,10 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
List<String> storageNames = new ArrayList<String>(tab.getStorages());
for (String s : storageNames) {
msClient.dropStorageFromDim(dimTblName, s);
- LOG.info("Dropped storage " + s + " from dimension table " + dimTblName
+ log.info("Dropped storage " + s + " from dimension table " + dimTblName
+ " [" + ++i + "/" + total + "]");
}
- LOG.info("Dropped " + total + " storages from dimension table " + dimTblName);
+ log.info("Dropped " + total + " storages from dimension table " + dimTblName);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -451,10 +451,10 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
List<String> storageNames = new ArrayList<String>(tab.getStorages());
for (String s : storageNames) {
msClient.dropStorageFromFact(factName, s);
- LOG.info("Dropped storage " + s + " from fact table " + factName
+ log.info("Dropped storage " + s + " from fact table " + factName
+ " [" + ++i + "/" + total + "]");
}
- LOG.info("Dropped " + total + " storages from fact table " + factName);
+ log.info("Dropped " + total + " storages from fact table " + factName);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -474,7 +474,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
msClient.dropStorageFromDim(dimTblName, storage);
- LOG.info("Dropped storage " + storage + " from dimension table " + dimTblName);
+ log.info("Dropped storage " + storage + " from dimension table " + dimTblName);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -518,7 +518,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
fact.getWeight(),
JAXBUtils.mapFromXProperties(fact.getProperties()),
JAXBUtils.storageTableMapFromXStorageTables(fact.getStorageTables()));
- LOG.info("Created fact table " + fact.getName());
+ log.info("Created fact table " + fact.getName());
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -532,7 +532,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
acquire(sessionid);
getClient(sessionid).alterCubeFactTable(fact.getName(), JAXBUtils.cubeFactFromFactTable(fact),
JAXBUtils.storageTableMapFromXStorageTables(fact.getStorageTables()));
- LOG.info("Updated fact table " + fact.getName());
+ log.info("Updated fact table " + fact.getName());
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -545,7 +545,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).dropFact(fact, cascade);
- LOG.info("Dropped fact table " + fact + " cascade? " + cascade);
+ log.info("Dropped fact table " + fact + " cascade? " + cascade);
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -653,7 +653,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
msClient.addStorage(msClient.getFactTable(fact),
storageTable.getStorageName(), updatePeriods,
JAXBUtils.storageTableDescFromXStorageTableElement(storageTable));
- LOG.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact);
+ log.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -667,7 +667,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
acquire(sessionid);
checkFactStorage(sessionid, fact, storage);
getClient(sessionid).dropStorageFromFact(fact, storage);
- LOG.info("Dropped storage " + storage + " from fact " + fact);
+ log.info("Dropped storage " + storage + " from fact " + fact);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -797,7 +797,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
getClient(sessionid).addPartition(
JAXBUtils.storagePartSpecFromXPartition(partition),
storageName);
- LOG.info("Added partition for dimension: " + dimTblName + " storage: " + storageName);
+ log.info("Added partition for dimension: " + dimTblName + " storage: " + storageName);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -826,7 +826,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
List<FieldSchema> cols = client.getHiveTable(tableName).getPartCols();
String[] vals = StringUtils.split(values, ",");
if (vals.length != cols.size()) {
- LOG.error("Values for all the part columns not specified, cols:" + cols + " vals:" + vals);
+ log.error("Values for all the part columns not specified, cols:" + cols + " vals:" + vals);
throw new BadRequestException("Values for all the part columns not specified");
}
StringBuilder filter = new StringBuilder();
@@ -881,10 +881,10 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
List<Partition> partitions = msClient.getPartitionsByFilter(
tableName, filter);
if (partitions.size() > 1) {
- LOG.error("More than one partition with specified values, correspoding filter:" + filter);
+ log.error("More than one partition with specified values, correspoding filter:" + filter);
throw new BadRequestException("More than one partition with specified values");
} else if (partitions.size() == 0) {
- LOG.error("No partition exists with specified values, correspoding filter:" + filter);
+ log.error("No partition exists with specified values, correspoding filter:" + filter);
throw new NotFoundException("No partition exists with specified values");
}
Map<String, Date> timeSpec = new HashMap<String, Date>();
@@ -892,7 +892,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
UpdatePeriod updatePeriod = populatePartSpec(partitions.get(0), timeSpec, nonTimeSpec);
msClient.dropPartition(cubeTableName,
storageName, timeSpec, nonTimeSpec, updatePeriod);
- LOG.info("Dropped partition for dimension: " + cubeTableName
+ log.info("Dropped partition for dimension: " + cubeTableName
+ " storage: " + storageName + " values:" + values);
} catch (HiveException exc) {
throw new LensException(exc);
@@ -924,7 +924,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
}
}
- LOG.info("Dropped partition for cube table: " + cubeTableName
+ log.info("Dropped partition for cube table: " + cubeTableName
+ " storage: " + storageName + " by filter:" + filter);
} catch (HiveException exc) {
throw new LensException(exc);
@@ -939,7 +939,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).createStorage(JAXBUtils.storageFromXStorage(storage));
- LOG.info("Created storage " + storage.getName());
+ log.info("Created storage " + storage.getName());
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -954,7 +954,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).dropStorage(storageName);
- LOG.info("Dropped storage " + storageName);
+ log.info("Dropped storage " + storageName);
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -969,7 +969,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
acquire(sessionid);
getClient(sessionid).alterStorage(storageName,
JAXBUtils.storageFromXStorage(storage));
- LOG.info("Altered storage " + storageName);
+ log.info("Altered storage " + storageName);
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -1086,7 +1086,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).createDimension(JAXBUtils.dimensionFromXDimension(dimension));
- LOG.info("Created dimension " + dimension.getName());
+ log.info("Created dimension " + dimension.getName());
} catch (HiveException e) {
throw new LensException(e);
} catch (ParseException e) {
@@ -1115,7 +1115,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
getClient(sessionid).dropDimension(dimName);
- LOG.info("Dropped dimension " + dimName);
+ log.info("Dropped dimension " + dimName);
} catch (HiveException e) {
throw new LensException(e);
} finally {
@@ -1130,7 +1130,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
acquire(sessionid);
getClient(sessionid).alterDimension(dimName,
JAXBUtils.dimensionFromXDimension(dimension));
- LOG.info("Altered dimension " + dimName);
+ log.info("Altered dimension " + dimName);
} catch (HiveException e) {
throw new LensException(e);
} catch (ParseException e) {
@@ -1222,7 +1222,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
if (!StringUtils.isBlank(dbName)) {
tables = getTablesFromDB(sessionid, dbName, false);
} else {
- LOG.info("Getting tables from all dbs");
+ log.info("Getting tables from all dbs");
List<String> alldbs = getAllDatabases(sessionid);
tables = new ArrayList<String>();
for (String db : alldbs) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index a628c90..b6d5fec 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -38,20 +38,20 @@ import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import com.google.common.base.Optional;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Utilities for converting to and from JAXB types to hive.ql.metadata.cube types
*/
+@Slf4j
public final class JAXBUtils {
private JAXBUtils() {
}
- public static final Logger LOG = LogManager.getLogger(JAXBUtils.class);
private static final ObjectFactory XCF = new ObjectFactory();
/**
@@ -223,7 +223,7 @@ public final class JAXBUtils {
try {
return DatatypeFactory.newInstance().newXMLGregorianCalendar(c1);
} catch (DatatypeConfigurationException e) {
- LOG.warn("Error converting date " + d, e);
+ log.warn("Error converting date " + d, e);
return null;
}
}
@@ -322,7 +322,7 @@ public final class JAXBUtils {
xcc.setChainName(rd.getChainName());
xcc.setRefCol(rd.getRefColumn());
if (baseTable.getChainByName(rd.getChainName()) == null) {
- LOG.error("Missing chain definition for " + rd.getChainName());
+ log.error("Missing chain definition for " + rd.getChainName());
} else {
xcc.setDestTable(baseTable.getChainByName(rd.getChainName()).getDestTable());
}
@@ -568,7 +568,7 @@ public final class JAXBUtils {
storage.addProperties(mapFromXProperties(xs.getProperties()));
return storage;
} catch (Exception e) {
- LOG.error("Could not create storage class" + xs.getClassname() + "with name:" + xs.getName());
+ log.error("Could not create storage class" + xs.getClassname() + "with name:" + xs.getName(), e);
throw new WebApplicationException(e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index ee24e1f..6e8b3a6 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -45,7 +45,6 @@ import org.apache.lens.server.api.session.SessionService;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.AbstractService;
-import org.apache.log4j.Logger;
import org.glassfish.jersey.server.ContainerRequest;
import org.glassfish.jersey.server.model.ResourceMethod;
@@ -63,14 +62,14 @@ import info.ganglia.gmetric4j.gmetric.GMetric;
import info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class MetricsServiceImpl.
*/
+@Slf4j
public class MetricsServiceImpl extends AbstractService implements MetricsService {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(MetricsService.class);
-
/** The query status listener. */
private AsyncEventListener<StatusChange> queryStatusListener;
@@ -132,7 +131,7 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
private boolean enableResourceMethodMetering;
public void setEnableResourceMethodMetering(boolean enableResourceMethodMetering) {
- LOG.info("setEnableResourceMethodMetering: " + enableResourceMethodMetering);
+ log.info("setEnableResourceMethodMetering: " + enableResourceMethodMetering);
this.enableResourceMethodMetering = enableResourceMethodMetering;
if (!enableResourceMethodMetering) {
methodMetricsFactory.clear();
@@ -273,7 +272,7 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
reporters.add(greporter);
} catch (IOException e) {
- LOG.error("Could not start ganglia reporter", e);
+ log.error("Could not start ganglia reporter", e);
}
}
if (hiveConf.getBoolean(LensConfConstants.ENABLE_GRAPHITE_METRICS, false)) {
@@ -289,10 +288,10 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
.build(graphite);
reporters.add(reporter);
} catch (UnknownHostException e) {
- LOG.error("Couldn't get localhost. So couldn't setup graphite reporting");
+ log.error("Couldn't get localhost. So couldn't setup graphite reporting", e);
}
}
- LOG.info("Started metrics service");
+ log.info("Started metrics service");
super.init(hiveConf);
}
@@ -416,7 +415,7 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
}
}
- LOG.info("Stopped metrics service");
+ log.info("Stopped metrics service");
super.stop();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index a57888d..1904350 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -37,17 +37,14 @@ import org.apache.commons.dbutils.handlers.BeanHandler;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import lombok.extern.slf4j.Slf4j;
/**
* Top level class which logs and retrieves finished query from Database.
*/
+@Slf4j
public class LensServerDAO {
- /** The Constant LOG. */
- private static final Logger LOG = LoggerFactory.getLogger(LensServerDAO.class);
-
/** The ds. */
private DataSource ds;
@@ -83,7 +80,7 @@ public class LensServerDAO {
try {
runner.update("drop table finished_queries");
} catch (SQLException e) {
- e.printStackTrace();
+ log.error("SQL exception while dropping finished queries table.", e);
}
}
@@ -102,9 +99,9 @@ public class LensServerDAO {
try {
createTable(sql);
ds.getConnection().commit();
- LOG.info("Created finished queries table");
+ log.info("Created finished queries table");
} catch (SQLException e) {
- LOG.warn("Unable to create finished queries table", e);
+ log.warn("Unable to create finished queries table", e);
}
}
@@ -128,10 +125,10 @@ public class LensServerDAO {
query.getErrorMessage(), query.getDriverStartTime(), query.getDriverEndTime(), query.getMetadataClass(),
query.getQueryName(), query.getSubmissionTime());
} else {
- LOG.warn("Re insert happening in purge: " + Thread.currentThread().getStackTrace());
+ log.warn("Re insert happening in purge: " + Thread.currentThread().getStackTrace());
if (alreadyExisting.equals(query)) {
// This is also okay
- LOG.warn("Skipping Re-insert. Finished Query found in DB while trying to insert, handle=" + query.getHandle());
+ log.warn("Skipping Re-insert. Finished Query found in DB while trying to insert, handle=" + query.getHandle());
} else {
String msg = "Found different value pre-existing in DB while trying to insert finished query. "
+ "Old = " + alreadyExisting + "\nNew = " + query;
@@ -155,7 +152,7 @@ public class LensServerDAO {
try {
return runner.query(sql, rsh, handle);
} catch (SQLException e) {
- e.printStackTrace();
+ log.error("SQL exception while executing query.", e);
}
return null;
}
@@ -212,7 +209,7 @@ public class LensServerDAO {
try {
queryHandleList.add(QueryHandle.fromString(handle));
} catch (IllegalArgumentException exc) {
- LOG.warn("Warning invalid query handle found in DB " + handle);
+ log.warn("Warning invalid query handle found in DB " + handle);
}
}
return queryHandleList;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 4cf1fa9..7c2da3a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -73,19 +73,17 @@ import org.codehaus.jackson.map.*;
import org.codehaus.jackson.map.module.SimpleModule;
import com.google.common.collect.ImmutableList;
+
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class QueryExecutionServiceImpl.
*/
+@Slf4j
public class QueryExecutionServiceImpl extends LensService implements QueryExecutionService {
/**
- * The Constant LOG.
- */
- public static final Log LOG = LogFactory.getLog(QueryExecutionServiceImpl.class);
-
- /**
* The Constant PREPARED_QUERIES_COUNTER.
*/
public static final String PREPARED_QUERIES_COUNTER = "prepared-queries";
@@ -239,7 +237,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public void onEvent(DriverEvent event) {
// Need to restore session only in case of hive driver
if (event instanceof DriverSessionStarted) {
- LOG.info("New driver event by driver " + event.getDriver());
+ log.info("New driver event by driver " + event.getDriver());
handleDriverSessionStart(event);
}
}
@@ -266,10 +264,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
Class<?> clazz = Class.forName(acceptorClass);
QueryAcceptor acceptor = (QueryAcceptor) clazz.newInstance();
- LOG.info("initialized query acceptor: " + acceptor);
+ log.info("initialized query acceptor: " + acceptor);
queryAcceptors.add(acceptor);
} catch (Exception e) {
- LOG.warn("Could not load the acceptor:" + acceptorClass, e);
+ log.warn("Could not load the acceptor:" + acceptorClass, e);
throw new LensException("Could not load acceptor" + acceptorClass, e);
}
}
@@ -279,14 +277,14 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private void initializeListeners() {
if (conf.getBoolean(LensConfConstants.QUERY_STATE_LOGGER_ENABLED, true)) {
getEventService().addListenerForType(new QueryStatusLogger(), StatusChange.class);
- LOG.info("Registered query state logger");
+ log.info("Registered query state logger");
}
// Add result formatter
getEventService().addListenerForType(new ResultFormatter(this), QueryExecuted.class);
getEventService().addListenerForType(new QueryExecutionStatisticsGenerator(this, getEventService()),
QueryEnded.class);
getEventService().addListenerForType(new QueryEndNotifier(this, getCliService().getHiveConf()), QueryEnded.class);
- LOG.info("Registered query result formatter");
+ log.info("Registered query result formatter");
}
/**
@@ -308,9 +306,9 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
drivers.put(driverClass, driver);
- LOG.info("Driver for " + driverClass + " is loaded");
+ log.info("Driver for " + driverClass + " is loaded");
} catch (Exception e) {
- LOG.warn("Could not load the driver:" + driverClass, e);
+ log.warn("Could not load the driver:" + driverClass, e);
throw new LensException("Could not load driver " + driverClass, e);
}
}
@@ -321,7 +319,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
Class<? extends DriverSelector> driverSelectorClass = conf.getClass(LensConfConstants.DRIVER_SELECTOR_CLASS,
MinQueryCostSelector.class,
DriverSelector.class);
- LOG.info("Using driver selector class: " + driverSelectorClass.getCanonicalName());
+ log.info("Using driver selector class: " + driverSelectorClass.getCanonicalName());
driverSelector = driverSelectorClass.newInstance();
} catch (Exception e) {
throw new LensException("Couldn't instantiate driver selector class. Class name: "
@@ -371,6 +369,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* The Class QueryStatusLogger.
*/
+ @Slf4j
public static class QueryStatusLogger implements LensEventListener<StatusChange> {
/**
@@ -476,13 +475,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
@Override
public void run() {
- LOG.info("Starting QuerySubmitter thread");
+ log.info("Starting QuerySubmitter thread");
while (!pausedForTest && !stopped && !querySubmitter.isInterrupted()) {
try {
QueryContext ctx = queuedQueries.take();
synchronized (ctx) {
if (ctx.getStatus().getStatus().equals(Status.QUEUED)) {
- LOG.info("Launching query:" + ctx.getUserQuery());
+ log.info("Launching query:" + ctx.getUserQuery());
try {
// acquire session before any query operation.
acquire(ctx.getLensSessionIdentifier());
@@ -490,13 +489,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (!ctx.isDriverQueryExplicitlySet()) {
rewriteAndSelect(ctx);
} else {
- LOG.info("Submitting to already selected driver");
+ log.info("Submitting to already selected driver");
}
// Check if we need to pass session's effective resources to selected driver
addSessionResourcesToDriver(ctx);
ctx.getSelectedDriver().executeAsync(ctx);
} catch (Exception e) {
- LOG.error("Error launching query " + ctx.getQueryHandle(), e);
+ log.error("Error launching query " + ctx.getQueryHandle(), e);
String reason = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
setFailedStatus(ctx, "Launching query failed", reason);
continue;
@@ -504,18 +503,18 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
release(ctx.getLensSessionIdentifier());
}
setLaunchedStatus(ctx);
- LOG.info("Launched query " + ctx.getQueryHandle());
+ log.info("Launched query " + ctx.getQueryHandle());
}
}
} catch (InterruptedException e) {
- LOG.info("Query Submitter has been interrupted, exiting");
+ log.info("Query Submitter has been interrupted, exiting");
return;
} catch (Exception e) {
incrCounter(QUERY_SUBMITTER_COUNTER);
- LOG.error("Error in query submitter", e);
+ log.error("Error in query submitter", e);
}
}
- LOG.info("QuerySubmitter exited");
+ log.info("QuerySubmitter exited");
}
}
@@ -545,7 +544,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
@Override
public void run() {
- LOG.info("Starting Status poller thread");
+ log.info("Starting Status poller thread");
while (!stopped && !statusPoller.isInterrupted()) {
try {
List<QueryContext> launched = new ArrayList<QueryContext>();
@@ -554,25 +553,25 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (stopped || statusPoller.isInterrupted()) {
return;
}
- LOG.info("Polling status for " + ctx.getQueryHandle());
+ log.info("Polling status for " + ctx.getQueryHandle());
try {
// session is not required to update status of the query
// don't need to wrap this with acquire/release
updateStatus(ctx.getQueryHandle());
} catch (LensException e) {
- LOG.error("Error updating status ", e);
+ log.error("Error updating status ", e);
}
}
Thread.sleep(pollInterval);
} catch (InterruptedException e) {
- LOG.info("Status poller has been interrupted, exiting");
+ log.info("Status poller has been interrupted, exiting");
return;
} catch (Exception e) {
incrCounter(STATUS_UPDATE_COUNTER);
- LOG.error("Error in status poller", e);
+ log.error("Error in status poller", e);
}
}
- LOG.info("StatusPoller exited");
+ log.info("StatusPoller exited");
}
}
@@ -655,14 +654,14 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
QueryStatus before = ctx.getStatus();
if (!ctx.getStatus().getStatus().equals(QueryStatus.Status.QUEUED) && !ctx.getDriverStatus().isFinished()
&& !ctx.getStatus().finished()) {
- LOG.info("Updating status for " + ctx.getQueryHandle());
+ log.info("Updating status for " + ctx.getQueryHandle());
try {
ctx.getSelectedDriver().updateStatus(ctx);
ctx.setStatus(ctx.getDriverStatus().toQueryStatus());
} catch (LensException exc) {
// Driver gave exception while updating status
setFailedStatus(ctx, "Status update failed", exc.getMessage());
- LOG.error("Status update failed for " + handle, exc);
+ log.error("Status update failed for " + handle, exc);
}
// query is successfully executed by driver and
// if query result need not be persisted or there is no result available in driver, move the query to
@@ -712,7 +711,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
case SUCCESSFUL:
return new QuerySuccess(ctx.getEndTime(), prevState, currState, query);
default:
- LOG.warn("Query " + query + " transitioned to " + currState + " state from " + prevState + " state");
+ log.warn("Query " + query + " transitioned to " + currState + " state from " + prevState + " state");
return null;
}
}
@@ -741,7 +740,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
getEventService().notifyEvent(event);
} catch (LensException e) {
- LOG.warn("LensEventService encountered error while handling event: " + event.getEventId(), e);
+ log.warn("LensEventService encountered error while handling event: " + event.getEventId(), e);
}
}
}
@@ -758,13 +757,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
@Override
public void run() {
- LOG.info("Starting Query purger thread");
+ log.info("Starting Query purger thread");
while (!stopped && !queryPurger.isInterrupted()) {
FinishedQuery finished = null;
try {
finished = finishedQueries.take();
} catch (InterruptedException e) {
- LOG.info("QueryPurger has been interrupted, exiting");
+ log.info("QueryPurger has been interrupted, exiting");
return;
}
try {
@@ -785,9 +784,9 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
try {
lensServerDao.insertFinishedQuery(finishedQuery);
- LOG.info("Saved query " + finishedQuery.getHandle() + " to DB");
+ log.info("Saved query " + finishedQuery.getHandle() + " to DB");
} catch (Exception e) {
- LOG.warn("Exception while purging query ", e);
+ log.warn("Exception while purging query ", e);
finishedQueries.add(finished);
continue;
}
@@ -799,24 +798,24 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
finished.getCtx().getSelectedDriver().closeQuery(finished.getCtx().getQueryHandle());
}
} catch (Exception e) {
- LOG.warn("Exception while closing query with selected driver.", e);
+ log.warn("Exception while closing query with selected driver.", e);
}
- LOG.info("Purging: " + finished.getCtx().getQueryHandle());
+ log.info("Purging: " + finished.getCtx().getQueryHandle());
allQueries.remove(finished.getCtx().getQueryHandle());
resultSets.remove(finished.getCtx().getQueryHandle());
}
fireStatusChangeEvent(finished.getCtx(),
new QueryStatus(1f, Status.CLOSED, "Query purged", false, null, null), finished.getCtx().getStatus());
- LOG.info("Query purged: " + finished.getCtx().getQueryHandle());
+ log.info("Query purged: " + finished.getCtx().getQueryHandle());
} catch (LensException e) {
incrCounter(QUERY_PURGER_COUNTER);
- LOG.error("Error closing query ", e);
+ log.error("Error closing query ", e);
} catch (Exception e) {
incrCounter(QUERY_PURGER_COUNTER);
- LOG.error("Error in query purger", e);
+ log.error("Error in query purger", e);
}
}
- LOG.info("QueryPurger exited");
+ log.info("QueryPurger exited");
}
}
@@ -832,24 +831,24 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
@Override
public void run() {
- LOG.info("Starting Prepared Query purger thread");
+ log.info("Starting Prepared Query purger thread");
while (!stopped && !prepareQueryPurger.isInterrupted()) {
try {
PreparedQueryContext prepared = preparedQueryQueue.take();
destroyPreparedQuery(prepared);
- LOG.info("Purged prepared query: " + prepared.getPrepareHandle());
+ log.info("Purged prepared query: " + prepared.getPrepareHandle());
} catch (LensException e) {
incrCounter(PREPARED_QUERY_PURGER_COUNTER);
- LOG.error("Error closing prepared query ", e);
+ log.error("Error closing prepared query ", e);
} catch (InterruptedException e) {
- LOG.info("PreparedQueryPurger has been interrupted, exiting");
+ log.info("PreparedQueryPurger has been interrupted, exiting");
return;
} catch (Exception e) {
incrCounter(PREPARED_QUERY_PURGER_COUNTER);
- LOG.error("Error in prepared query purger", e);
+ log.error("Error in prepared query purger", e);
}
}
- LOG.info("PreparedQueryPurger exited");
+ log.info("PreparedQueryPurger exited");
}
}
@@ -875,7 +874,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
maxFinishedQueries = conf.getInt(LensConfConstants.MAX_NUMBER_OF_FINISHED_QUERY,
LensConfConstants.DEFAULT_FINISHED_QUERIES);
initalizeFinishedQueryStore(conf);
- LOG.info("Query execution service initialized");
+ log.info("Query execution service initialized");
}
/**
@@ -889,7 +888,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
this.lensServerDao.createFinishedQueriesTable();
} catch (Exception e) {
- LOG.warn("Unable to create finished query table, query purger will not purge queries", e);
+ log.warn("Unable to create finished query table, query purger will not purge queries", e);
}
SimpleModule module = new SimpleModule("HiveColumnModule", new Version(1, 0, 0, null));
module.addSerializer(ColumnDescriptor.class, new JsonSerializer<ColumnDescriptor>() {
@@ -940,15 +939,15 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
super.stop();
for (Thread th : new Thread[]{querySubmitter, statusPoller, queryPurger, prepareQueryPurger}) {
try {
- LOG.debug("Waiting for" + th.getName());
+ log.debug("Waiting for" + th.getName());
th.join();
} catch (InterruptedException e) {
- LOG.error("Error waiting for thread: " + th.getName(), e);
+ log.error("Error waiting for thread: " + th.getName(), e);
}
}
estimatePool.shutdownNow();
- LOG.info("Query execution service stopped");
+ log.info("Query execution service stopped");
}
/*
@@ -973,7 +972,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
}
} catch (LensException e) {
- LOG.error("Could not set query conf ", e);
+ log.error("Could not set query conf ", e);
}
}
}
@@ -1068,7 +1067,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
++inCompleteDrivers;
// Cancel the corresponding task
estimateFutures.get(i).cancel(true);
- LOG.warn("Timeout reached for estimate task for driver " + r.getDriver() + " " + debugInfo);
+ log.warn("Timeout reached for estimate task for driver " + r.getDriver() + " " + debugInfo);
}
}
@@ -1188,20 +1187,20 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (!succeeded) {
failureCause = estimateRunnable.getFailureCause();
cause = estimateRunnable.getCause();
- LOG.error("Estimate failed for driver " + driver + " cause: " + failureCause);
+ log.error("Estimate failed for driver " + driver + " cause: " + failureCause);
}
estimateGauge.markSuccess();
} else {
- LOG.error("Estimate skipped since rewrite failed for driver " + driver + " cause: " + failureCause);
+ log.error("Estimate skipped since rewrite failed for driver " + driver + " cause: " + failureCause);
}
} catch (Throwable th) {
- LOG.error("Error computing estimate for driver " + driver, th);
+ log.error("Error computing estimate for driver " + driver, th);
} finally {
completed = true;
try {
release(ctx.getLensSessionIdentifier());
} catch (LensException e) {
- LOG.error("Could not release session: " + ctx.getLensSessionIdentifier(), e);
+ log.error("Could not release session: " + ctx.getLensSessionIdentifier(), e);
} finally {
estimateCompletionLatch.countDown();
}
@@ -1368,7 +1367,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
throws LensException {
PreparedQueryContext prepared = null;
try {
- LOG.info("ExplainAndPrepare: " + sessionHandle.toString() + " query: " + query);
+ log.info("ExplainAndPrepare: " + sessionHandle.toString() + " query: " + query);
acquire(sessionHandle);
prepared = prepareQuery(sessionHandle, query, lensConf, SubmitOp.EXPLAIN_AND_PREPARE);
prepared.setQueryName(queryName);
@@ -1380,7 +1379,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (prepared != null) {
destroyPreparedQuery(prepared);
}
- LOG.error("Explain and prepare failed", e);
+ log.error("Explain and prepare failed", e);
QueryPlan plan;
if (e.getCause() != null && e.getCause().getMessage() != null) {
plan = new QueryPlan(true, e.getCause().getMessage());
@@ -1406,7 +1405,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryHandle executePrepareAsync(LensSessionHandle sessionHandle, QueryPrepareHandle prepareHandle,
LensConf conf, String queryName) throws LensException {
try {
- LOG.info("ExecutePrepareAsync: " + sessionHandle.toString() + " query:" + prepareHandle.getPrepareHandleId());
+ log.info("ExecutePrepareAsync: " + sessionHandle.toString() + " query:" + prepareHandle.getPrepareHandleId());
acquire(sessionHandle);
PreparedQueryContext pctx = getPreparedQueryContext(sessionHandle, prepareHandle);
Configuration qconf = getLensConf(sessionHandle, conf);
@@ -1434,7 +1433,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryHandleWithResultSet executePrepare(LensSessionHandle sessionHandle, QueryPrepareHandle prepareHandle,
long timeoutMillis, LensConf conf, String queryName) throws LensException {
try {
- LOG.info("ExecutePrepare: " + sessionHandle.toString() + " query:" + prepareHandle.getPrepareHandleId()
+ log.info("ExecutePrepare: " + sessionHandle.toString() + " query:" + prepareHandle.getPrepareHandleId()
+ " timeout:" + timeoutMillis);
acquire(sessionHandle);
PreparedQueryContext pctx = getPreparedQueryContext(sessionHandle, prepareHandle);
@@ -1462,7 +1461,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryHandle executeAsync(LensSessionHandle sessionHandle, String query, LensConf conf, String queryName)
throws LensException {
try {
- LOG.info("ExecuteAsync: " + sessionHandle.toString() + " query: " + query);
+ log.info("ExecuteAsync: " + sessionHandle.toString() + " query: " + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, conf);
accept(query, qconf, SubmitOp.EXECUTE);
@@ -1521,7 +1520,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
queuedQueries.add(ctx);
allQueries.put(ctx.getQueryHandle(), ctx);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
- LOG.info("Returning handle " + ctx.getQueryHandle().getHandleId());
+ log.info("Returning handle " + ctx.getQueryHandle().getHandleId());
return ctx.getQueryHandle();
}
@@ -1535,7 +1534,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public boolean updateQueryConf(LensSessionHandle sessionHandle, QueryHandle queryHandle, LensConf newconf)
throws LensException {
try {
- LOG.info("UpdateQueryConf:" + sessionHandle.toString() + " query: " + queryHandle);
+ log.info("UpdateQueryConf:" + sessionHandle.toString() + " query: " + queryHandle);
acquire(sessionHandle);
QueryContext ctx = getQueryContext(sessionHandle, queryHandle);
if (ctx != null && ctx.getStatus().getStatus() == QueryStatus.Status.QUEUED) {
@@ -1560,7 +1559,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public boolean updateQueryConf(LensSessionHandle sessionHandle, QueryPrepareHandle prepareHandle, LensConf newconf)
throws LensException {
try {
- LOG.info("UpdatePreparedQueryConf:" + sessionHandle.toString() + " query: " + prepareHandle);
+ log.info("UpdatePreparedQueryConf:" + sessionHandle.toString() + " query: " + prepareHandle);
acquire(sessionHandle);
PreparedQueryContext ctx = getPreparedQueryContext(sessionHandle, prepareHandle);
ctx.updateConf(newconf.getProperties());
@@ -1663,7 +1662,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryHandleWithResultSet execute(LensSessionHandle sessionHandle, String query, long timeoutMillis,
LensConf conf, String queryName) throws LensException {
try {
- LOG.info("Blocking execute " + sessionHandle.toString() + " query: " + query + " timeout: " + timeoutMillis);
+ log.info("Blocking execute " + sessionHandle.toString() + " query: " + query + " timeout: " + timeoutMillis);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, conf);
accept(query, qconf, SubmitOp.EXECUTE);
@@ -1694,7 +1693,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
Thread.sleep(10);
} catch (InterruptedException e) {
- e.printStackTrace();
+ log.error("Encountered Interrupted exception.", e);
}
}
QueryCompletionListener listener = new QueryCompletionListenerImpl(handle);
@@ -1708,7 +1707,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
listener.wait(timeoutMillis);
}
} catch (InterruptedException e) {
- LOG.info("Waiting thread interrupted");
+ log.info("Waiting thread interrupted");
}
if (getQueryContext(sessionHandle, handle).getStatus().finished()) {
result.setResult(getResultset(handle).toQueryResult());
@@ -1750,7 +1749,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public void onCompletion(QueryHandle handle) {
synchronized (this) {
succeeded = true;
- LOG.info("Query " + handle + " with time out succeeded");
+ log.info("Query " + handle + " with time out succeeded");
this.notify();
}
}
@@ -1765,7 +1764,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public void onError(QueryHandle handle, String error) {
synchronized (this) {
succeeded = false;
- LOG.info("Query " + handle + " with time out failed");
+ log.info("Query " + handle + " with time out failed");
this.notify();
}
}
@@ -1782,7 +1781,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryResultSetMetadata getResultSetMetadata(LensSessionHandle sessionHandle, QueryHandle queryHandle)
throws LensException {
try {
- LOG.info("GetResultSetMetadata: " + sessionHandle.toString() + " query: " + queryHandle);
+ log.info("GetResultSetMetadata: " + sessionHandle.toString() + " query: " + queryHandle);
acquire(sessionHandle);
LensResultSet resultSet = getResultset(queryHandle);
if (resultSet != null) {
@@ -1806,7 +1805,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryResult fetchResultSet(LensSessionHandle sessionHandle, QueryHandle queryHandle, long startIndex,
int fetchSize) throws LensException {
try {
- LOG.info("FetchResultSet:" + sessionHandle.toString() + " query:" + queryHandle);
+ log.info("FetchResultSet:" + sessionHandle.toString() + " query:" + queryHandle);
acquire(sessionHandle);
return getResultset(queryHandle).toQueryResult();
} finally {
@@ -1823,7 +1822,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public void closeResultSet(LensSessionHandle sessionHandle, QueryHandle queryHandle) throws LensException {
try {
- LOG.info("CloseResultSet:" + sessionHandle.toString() + " query: " + queryHandle);
+ log.info("CloseResultSet:" + sessionHandle.toString() + " query: " + queryHandle);
acquire(sessionHandle);
resultSets.remove(queryHandle);
// Ask driver to close result set
@@ -1842,7 +1841,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public boolean cancelQuery(LensSessionHandle sessionHandle, QueryHandle queryHandle) throws LensException {
try {
- LOG.info("CancelQuery: " + sessionHandle.toString() + " query:" + queryHandle);
+ log.info("CancelQuery: " + sessionHandle.toString() + " query:" + queryHandle);
acquire(sessionHandle);
QueryContext ctx = getQueryContext(sessionHandle, queryHandle);
if (ctx.getStatus().finished()) {
@@ -1914,7 +1913,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
List<QueryHandle> persistedQueries = lensServerDao.findFinishedQueries(state, userName, queryName, fromDate,
toDate);
if (persistedQueries != null && !persistedQueries.isEmpty()) {
- LOG.info("Adding persisted queries " + persistedQueries.size());
+ log.info("Adding persisted queries " + persistedQueries.size());
all.addAll(persistedQueries);
}
}
@@ -1991,7 +1990,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public boolean destroyPrepared(LensSessionHandle sessionHandle, QueryPrepareHandle prepared) throws LensException {
try {
- LOG.info("DestroyPrepared: " + sessionHandle.toString() + " query:" + prepared);
+ log.info("DestroyPrepared: " + sessionHandle.toString() + " query:" + prepared);
acquire(sessionHandle);
destroyPreparedQuery(getPreparedQueryContext(sessionHandle, prepared));
return true;
@@ -2025,7 +2024,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QueryCost estimate(LensSessionHandle sessionHandle, String query, LensConf lensConf)
throws LensException {
try {
- LOG.info("Estimate: " + sessionHandle.toString() + " query:" + query);
+ log.info("Estimate: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
ExplainQueryContext estimateQueryContext = new ExplainQueryContext(query,
@@ -2048,7 +2047,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public QueryPlan explain(LensSessionHandle sessionHandle, String query, LensConf lensConf) throws LensException {
try {
- LOG.info("Explain: " + sessionHandle.toString() + " query:" + query);
+ log.info("Explain: " + sessionHandle.toString() + " query:" + query);
acquire(sessionHandle);
Configuration qconf = getLensConf(sessionHandle, lensConf);
ExplainQueryContext explainQueryContext = new ExplainQueryContext(query,
@@ -2059,7 +2058,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
addSessionResourcesToDriver(explainQueryContext);
return explainQueryContext.getSelectedDriver().explain(explainQueryContext).toQueryPlan();
} catch (LensException e) {
- LOG.error("Error during explain :", e);
+ log.error("Error during explain :", e);
QueryPlan plan;
if (e.getCause() != null && e.getCause().getMessage() != null) {
plan = new QueryPlan(true, e.getCause().getMessage());
@@ -2135,10 +2134,10 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
driver = (LensDriver) driverCls.newInstance();
driver.configure(conf);
} catch (Exception e) {
- LOG.error("Could not instantiate driver:" + driverClsName);
+ log.error("Could not instantiate driver:" + driverClsName, e);
throw new IOException(e);
}
- LOG.info("Driver state for " + driverClsName + " will be ignored");
+ log.info("Driver state for " + driverClsName + " will be ignored");
}
driver.readExternal(in);
}
@@ -2187,7 +2186,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
allQueries.remove(ctx.getQueryHandle());
}
}
- LOG.info("Recovered " + allQueries.size() + " queries");
+ log.info("Recovered " + allQueries.size() + " queries");
}
}
@@ -2219,7 +2218,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
}
}
- LOG.info("Persisted " + allQueries.size() + " queries");
+ log.info("Persisted " + allQueries.size() + " queries");
}
/**
@@ -2256,7 +2255,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
throw new NotFoundException("Http result not available for query:" + queryHandle.toString());
}
} catch (IOException e) {
- LOG.warn("Unable to get status for Result Directory", e);
+ log.warn("Unable to get status for Result Directory", e);
throw new NotFoundException("Http result not available for query:" + queryHandle.toString());
}
String resultFSReadUrl = ctx.getConf().get(LensConfConstants.RESULT_FS_READ_URL);
@@ -2360,7 +2359,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
String lensSession = sessionStarted.getLensSessionID();
LensSessionHandle sessionHandle = getSessionHandle(lensSession);
if (sessionHandle == null) {
- LOG.warn("Lens session went away for sessionid:" + lensSession);
+ log.warn("Lens session went away for sessionid:" + lensSession);
return;
}
@@ -2371,22 +2370,22 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
List<ResourceEntry> resources = session.getLensSessionPersistInfo().getResources();
if (resources != null && !resources.isEmpty()) {
for (ResourceEntry resource : resources) {
- LOG.info("Restoring resource " + resource + " for session " + lensSession);
+ log.info("Restoring resource " + resource + " for session " + lensSession);
String command = "add " + resource.getType().toLowerCase() + " " + resource.getLocation();
try {
// Execute add resource query in blocking mode
hiveDriver.execute(createResourceQuery(command, sessionHandle, hiveDriver));
resource.restoredResource();
- LOG.info("Restored resource " + resource + " for session " + lensSession);
+ log.info("Restored resource " + resource + " for session " + lensSession);
} catch (Exception exc) {
- LOG.error("Unable to add resource " + resource + " for session " + lensSession, exc);
+ log.error("Unable to add resource " + resource + " for session " + lensSession, exc);
}
}
} else {
- LOG.info("No resources to restore for session " + lensSession);
+ log.info("No resources to restore for session " + lensSession);
}
} catch (Exception e) {
- LOG.warn(
+ log.warn(
"Lens session went away! " + lensSession + " driver session: "
+ ((DriverSessionStarted) event).getDriverSessionID(), e);
} finally {
@@ -2421,7 +2420,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
Collection<ResourceEntry> dbResources = session.getDBResources(ctx.getDatabase());
if (CollectionUtils.isNotEmpty(dbResources)) {
- LOG.info("Proceeding to add resources for DB "
+ log.info("Proceeding to add resources for DB "
+ session.getCurrentDatabase() + " for query " + ctx.getLogHandle() + " resources: " + dbResources);
List<ResourceEntry> failedDBResources = addResources(dbResources, sessionHandle, hiveDriver);
@@ -2433,7 +2432,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
}
} else {
- LOG.info("No need to add DB resources for session: " + sessionIdentifier
+ log.info("No need to add DB resources for session: " + sessionIdentifier
+ " db= " + session.getCurrentDatabase());
}
hiveDriver.setResourcesAddedForSession(sessionIdentifier, ctx.getDatabase());
@@ -2442,7 +2441,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
// Get pending session resources which needed to be added for this database
Collection<ResourceEntry> pendingResources =
session.getPendingSessionResourcesForDatabase(ctx.getDatabase());
- LOG.info("Adding pending " + pendingResources.size() + " session resources for session " + sessionIdentifier
+ log.info("Adding pending " + pendingResources.size() + " session resources for session " + sessionIdentifier
+ " for database " + ctx.getDatabase());
List<ResourceEntry> failedResources = addResources(pendingResources, sessionHandle, hiveDriver);
// Mark added resources so that we don't add them again. If any of the resources failed
@@ -2471,7 +2470,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
addSingleResourceToHive(hiveDriver, res, sessionHandle);
} catch (LensException exc) {
failedResources.add(res);
- LOG.error("Error adding resources for session "
+ log.error("Error adding resources for session "
+ sessionHandle.getPublicId().toString() + " resources: " + res.getLocation(), exc.getCause());
}
}
@@ -2489,7 +2488,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
String command = "add " + res.getType().toLowerCase() + " " + uri;
driver.execute(createResourceQuery(command, sessionHandle, driver));
- LOG.info("Added resource to hive driver for session "
+ log.info("Added resource to hive driver for session "
+ sessionIdentifier + " cmd: " + command);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index 9b6d6bc..34e14b6 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -42,22 +42,20 @@ import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.lens.server.error.UnSupportedQuerySubmitOpException;
import org.apache.commons.lang.StringUtils;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.glassfish.jersey.media.multipart.FormDataParam;
+import lombok.extern.slf4j.Slf4j;
+
/**
* queryapi resource
* <p/>
* This provides api for all things query.
*/
+@Slf4j
@Path("/queryapi")
public class QueryServiceResource {
- /** The Constant LOG. */
- public static final Logger LOG = LogManager.getLogger(QueryServiceResource.class);
-
/** The query server. */
private QueryExecutionService queryServer;
@@ -269,7 +267,7 @@ public class QueryServiceResource {
}
}
} catch (Exception e) {
- LOG.error("Error canceling queries", e);
+ log.error("Error canceling queries", e);
failed = true;
}
String msgString = (StringUtils.isBlank(state) ? "" : " in state" + state)
@@ -342,7 +340,7 @@ public class QueryServiceResource {
try {
sop = SubmitOp.valueOf(operation.toUpperCase());
} catch (IllegalArgumentException e) {
- LOG.error("Illegal argument for submitop: " + operation);
+ log.error("Illegal argument for submitop: " + operation, e);
}
if (sop == null) {
throw new BadRequestException("Invalid operation type: " + operation + prepareClue);
@@ -391,7 +389,7 @@ public class QueryServiceResource {
}
}
} catch (Exception e) {
- LOG.error("Error destroying prepared queries", e);
+ log.error("Error destroying prepared queries", e);
failed = true;
}
String msgString = (StringUtils.isBlank(user) ? "" : " for user " + user);
@@ -637,7 +635,7 @@ public class QueryServiceResource {
try {
sop = SubmitOp.valueOf(operation.toUpperCase());
} catch (IllegalArgumentException e) {
- LOG.warn("illegal argument for submit operation: " + operation, e);
+ log.warn("illegal argument for submit operation: " + operation, e);
}
if (sop == null) {
throw new BadRequestException("Invalid operation type: " + operation + submitPreparedClue);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index bb0d301..6dc4ea6 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -45,8 +45,6 @@ import org.apache.lens.server.query.QueryExecutionServiceImpl;
import org.apache.lens.server.session.LensSessionImpl.ResourceEntry;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -61,15 +59,14 @@ import org.apache.hive.service.cli.OperationHandle;
import lombok.AccessLevel;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class HiveSessionService.
*/
+@Slf4j
public class HiveSessionService extends LensService implements SessionService {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(HiveSessionService.class);
-
-
/** The restorable sessions. */
private List<LensSessionImpl.LensSessionPersistInfo> restorableSessions;
@@ -103,7 +100,7 @@ public class HiveSessionService extends LensService implements SessionService {
service.addResource(sessionid, type, path);
numAdded++;
} catch (LensException e) {
- LOG.error("Failed to add resource type:" + type + " path:" + path + " in service:" + service, e);
+ log.error("Failed to add resource type:" + type + " path:" + path + " in service:" + service, e);
error = true;
break;
}
@@ -205,7 +202,7 @@ public class HiveSessionService extends LensService implements SessionService {
Map<String, String> configuration)
throws LensException {
LensSessionHandle sessionid = super.openSession(username, password, configuration);
- LOG.info("Opened session " + sessionid + " for user " + username);
+ log.info("Opened session " + sessionid + " for user " + username);
notifyEvent(new SessionOpened(System.currentTimeMillis(), sessionid, username));
// Set current database
@@ -213,7 +210,7 @@ public class HiveSessionService extends LensService implements SessionService {
try {
if (!Hive.get(getSession(sessionid).getHiveConf()).databaseExists(database)) {
closeSession(sessionid);
- LOG.info("Closed session " + sessionid.getPublicId().toString() + " as db " + database + " does not exist");
+ log.info("Closed session " + sessionid.getPublicId().toString() + " as db " + database + " does not exist");
throw new NotFoundException("Database " + database + " does not exist");
}
} catch (Exception e) {
@@ -221,10 +218,10 @@ public class HiveSessionService extends LensService implements SessionService {
try {
closeSession(sessionid);
} catch (LensException e2) {
- LOG.error("Error closing session " + sessionid.getPublicId().toString(), e2);
+ log.error("Error closing session " + sessionid.getPublicId().toString(), e2);
}
- LOG.error("Error in checking if database exists " + database, e);
+ log.error("Error in checking if database exists " + database, e);
throw new LensException("Error in checking if database exists" + database, e);
} else {
throw (NotFoundException) e;
@@ -232,7 +229,7 @@ public class HiveSessionService extends LensService implements SessionService {
}
getSession(sessionid).setCurrentDatabase(database);
- LOG.info("Set database to " + database + " for session " + sessionid.getPublicId());
+ log.info("Set database to " + database + " for session " + sessionid.getPublicId());
}
// add auxuiliary jars
@@ -240,7 +237,7 @@ public class HiveSessionService extends LensService implements SessionService {
if (auxJars != null) {
for (String jar : auxJars) {
- LOG.info("Adding aux jar:" + jar);
+ log.info("Adding aux jar:" + jar);
addResourceToAllServices(sessionid, "jar", jar);
}
}
@@ -301,7 +298,7 @@ public class HiveSessionService extends LensService implements SessionService {
* @param addToSession the add to session
*/
protected void setSessionParameter(LensSessionHandle sessionid, String key, String value, boolean addToSession) {
- LOG.info("Request to Set param key:" + key + " value:" + value);
+ log.info("Request to Set param key:" + key + " value:" + value);
String command = "set" + " " + key + "= " + value;
try {
acquire(sessionid);
@@ -320,7 +317,7 @@ public class HiveSessionService extends LensService implements SessionService {
if (addToSession) {
getSession(sessionid).setConfig(key, value);
}
- LOG.info("Set param key:" + key + " value:" + value);
+ log.info("Set param key:" + key + " value:" + value);
} catch (HiveSQLException e) {
throw new WebApplicationException(e);
} finally {
@@ -354,7 +351,7 @@ public class HiveSessionService extends LensService implements SessionService {
// Restore sessions if any
if (restorableSessions == null || restorableSessions.size() <= 0) {
- LOG.info("No sessions to restore");
+ log.info("No sessions to restore");
return;
}
@@ -373,7 +370,7 @@ public class HiveSessionService extends LensService implements SessionService {
try {
addResource(sessionHandle, resourceEntry.getType(), resourceEntry.getLocation());
} catch (Exception e) {
- LOG.error("Failed to restore resource for session: " + session + " resource: " + resourceEntry);
+ log.error("Failed to restore resource for session: " + session + " resource: " + resourceEntry, e);
}
}
@@ -382,16 +379,17 @@ public class HiveSessionService extends LensService implements SessionService {
try {
setSessionParameter(sessionHandle, cfg.getKey(), cfg.getValue(), false);
} catch (Exception e) {
- LOG.error("Error setting parameter " + cfg.getKey() + "=" + cfg.getValue() + " for session: " + session);
+ log.error("Error setting parameter " + cfg.getKey() + "=" + cfg.getValue()
+ + " for session: " + session, e);
}
}
- LOG.info("Restored session " + persistInfo.getSessionHandle().getPublicId());
+ log.info("Restored session " + persistInfo.getSessionHandle().getPublicId());
notifyEvent(new SessionRestored(System.currentTimeMillis(), sessionHandle));
} catch (LensException e) {
throw new RuntimeException(e);
}
}
- LOG.info("Session service restoed " + restorableSessions.size() + " sessions");
+ log.info("Session service restoed " + restorableSessions.size() + " sessions");
}
/*
@@ -420,7 +418,7 @@ public class HiveSessionService extends LensService implements SessionService {
LensSessionImpl session = getSession(sessionHandle);
session.getLensSessionPersistInfo().writeExternal(out);
}
- LOG.info("Session service pesristed " + SESSION_MAP.size() + " sessions");
+ log.info("Session service pesristed " + SESSION_MAP.size() + " sessions");
}
/*
@@ -439,7 +437,7 @@ public class HiveSessionService extends LensService implements SessionService {
restorableSessions.add(persistInfo);
SESSION_MAP.put(persistInfo.getSessionHandle().getPublicId().toString(), persistInfo.getSessionHandle());
}
- LOG.info("Session service recovered " + SESSION_MAP.size() + " sessions");
+ log.info("Session service recovered " + SESSION_MAP.size() + " sessions");
}
/**
@@ -474,7 +472,7 @@ public class HiveSessionService extends LensService implements SessionService {
try {
getCliService().closeOperation(op);
} catch (HiveSQLException e) {
- LOG.error("Error closing operation " + op.getHandleIdentifier(), e);
+ log.error("Error closing operation " + op.getHandleIdentifier(), e);
}
}
}
@@ -511,13 +509,13 @@ public class HiveSessionService extends LensService implements SessionService {
try {
long lastAccessTime = getSession(sessionHandle).getLastAccessTime();
closeInternal(sessionHandle);
- LOG.info("Closed inactive session " + sessionHandle.getPublicId() + " last accessed at "
+ log.info("Closed inactive session " + sessionHandle.getPublicId() + " last accessed at "
+ new Date(lastAccessTime));
notifyEvent(new SessionExpired(System.currentTimeMillis(), sessionHandle));
} catch (ClientErrorException nfe) {
// Do nothing
} catch (LensException e) {
- LOG.error("Error closing session " + sessionHandle.getPublicId() + " reason " + e.getMessage());
+ log.error("Error closing session " + sessionHandle.getPublicId() + " reason " + e.getMessage(), e);
}
}
}
@@ -532,7 +530,7 @@ public class HiveSessionService extends LensService implements SessionService {
try {
runInternal();
} catch (Exception e) {
- LOG.warn("Unknown error while checking for inactive sessions - " + e.getMessage());
+ log.warn("Unknown error while checking for inactive sessions - " + e.getMessage());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/main/java/org/apache/lens/server/stats/event/query/QueryExecutionStatistics.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/event/query/QueryExecutionStatistics.java b/lens-server/src/main/java/org/apache/lens/server/stats/event/query/QueryExecutionStatistics.java
index 4acf748..020d540 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/event/query/QueryExecutionStatistics.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/event/query/QueryExecutionStatistics.java
@@ -34,9 +34,13 @@ import org.apache.hadoop.mapred.TextInputFormat;
import lombok.Getter;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
+
/**
* Statistics class used to capture query information.
*/
+@Slf4j
public class QueryExecutionStatistics extends LoggableLensStatistics {
/** The handle. */
@@ -147,7 +151,7 @@ public class QueryExecutionStatistics extends LoggableLensStatistics {
try {
table.setInputFormatClass(TextInputFormat.class.getName());
} catch (HiveException e) {
- e.printStackTrace();
+ log.error("Encountered hive exception.", e);
}
return table;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b0783db7/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
index aab9771..c05a2e7 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
@@ -39,6 +39,7 @@ import org.apache.lens.api.response.NoErrorPayload;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.commons.io.FileUtils;
+
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Database;
@@ -52,9 +53,12 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.testng.Assert;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class LensTestUtil.
*/
+@Slf4j
public final class LensTestUtil {
public static final String DB_WITH_JARS = "test_db_static_jars";
@@ -283,7 +287,7 @@ public final class LensTestUtil {
FileUtils.copyFile(testJarFile, new File(dbDir, jarOrder[2]));
FileUtils.copyFile(serdeJarFile, new File(dbDir, jarOrder[3]));
} catch (FileNotFoundException fnf) {
- fnf.printStackTrace();
+ log.error("File not found.", fnf);
}
}
}
[13/51] [abbrv] incubator-lens git commit: LENS-478 : Add counter for
DatabaseResourceService load errors (Yash Sharma via amareshwari)
Posted by jd...@apache.org.
LENS-478 : Add counter for DatabaseResourceService load errors (Yash Sharma via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/06e11557
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/06e11557
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/06e11557
Branch: refs/heads/current-release-line
Commit: 06e1155732f963c31c16aa4bf6b3278b6f144952
Parents: 70e4b92
Author: Yash Sharma <ya...@gmail.com>
Authored: Wed Jun 3 16:04:39 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jun 3 16:04:39 2015 +0530
----------------------------------------------------------------------
.../lens/server/api/metrics/MetricsService.java | 2 +
.../lens/server/metrics/MetricsServiceImpl.java | 12 +++++
.../server/session/DatabaseResourceService.java | 51 ++++++++++++++++++--
3 files changed, 62 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e11557/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
index ffde912..f7a23ab 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
@@ -133,6 +133,8 @@ public interface MetricsService {
String ACTIVE_SESSIONS = "active-sessions";
+ long getTotalDatabaseResourceLoadErrors();
+
long getQueuedQueries();
long getRunningQueries();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e11557/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index 66beeed..9ae0d66 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -42,6 +42,7 @@ import org.apache.lens.server.api.session.SessionEvent;
import org.apache.lens.server.api.session.SessionExpired;
import org.apache.lens.server.api.session.SessionOpened;
import org.apache.lens.server.api.session.SessionService;
+import org.apache.lens.server.session.DatabaseResourceService;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.AbstractService;
@@ -114,6 +115,9 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
/** The total cancelled queries. */
private Counter totalCancelledQueries;
+ /** The total errors while loading database resources */
+ private Counter totalDatabaseResourceLoadErrors;
+
/** The opened sessions */
private Gauge<Integer> activeSessions;
@@ -334,6 +338,9 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
}
});
+ totalDatabaseResourceLoadErrors = metricRegistry.counter(MetricRegistry.name(DatabaseResourceService.class,
+ DatabaseResourceService.LOAD_RESOURCES_ERRORS));
+
totalAcceptedQueries = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-"
+ ACCEPTED_QUERIES));
@@ -486,6 +493,11 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
}
@Override
+ public long getTotalDatabaseResourceLoadErrors() {
+ return totalDatabaseResourceLoadErrors.getCount();
+ }
+
+ @Override
public long getQueuedQueries() {
return queuedQueries.getValue();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e11557/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
index 608c679..2a5cce5 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
@@ -26,8 +26,10 @@ import java.net.URLClassLoader;
import java.nio.charset.Charset;
import java.util.*;
+import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
@@ -48,6 +50,36 @@ public class DatabaseResourceService extends AbstractService {
private Map<String, ClassLoader> classLoaderCache;
private Map<String, List<LensSessionImpl.ResourceEntry>> dbResEntryMap;
+ /**
+ * The metrics service.
+ */
+ private MetricsService metricsService;
+
+ /**
+ * The Constant LOAD_RESOURCES_ERRORS.
+ */
+ public static final String LOAD_RESOURCES_ERRORS = "total-load-resources-errors";
+
+ /**
+ * Incr counter.
+ *
+ * @param counter the counter
+ */
+ private void incrCounter(String counter) {
+ getMetrics().incrCounter(DatabaseResourceService.class, counter);
+ }
+
+ /**
+ * Gets counter value.
+ *
+ * @param counter the counter
+ */
+ private long getCounter(String counter) {
+ return getMetrics().getCounter(DatabaseResourceService.class, counter);
+ }
+
+
+
public DatabaseResourceService(String name) {
super(name);
}
@@ -68,7 +100,8 @@ public class DatabaseResourceService extends AbstractService {
loadDbResourceEntries();
loadResources();
} catch (LensException e) {
- LOG.warn("Failed to load DB resource mapping, resources must be added explicitly to session");
+ incrCounter(LOAD_RESOURCES_ERRORS);
+ LOG.warn("Failed to load DB resource mapping, resources must be added explicitly to session.");
}
}
@@ -92,6 +125,7 @@ public class DatabaseResourceService extends AbstractService {
Path resTopDirPath = new Path(resTopDir);
serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf());
if (!serverFs.exists(resTopDirPath)) {
+ incrCounter(LOAD_RESOURCES_ERRORS);
LOG.warn("Database resource location does not exist - " + resTopDir + ". Database jars will not be available");
return;
}
@@ -190,15 +224,15 @@ public class DatabaseResourceService extends AbstractService {
/**
* Load DB specific resources
- * @throws LensException
*/
- public void loadResources() throws LensException {
+ public void loadResources() {
for (String db : dbResEntryMap.keySet()) {
try {
createClassLoader(db);
loadDBJars(db, dbResEntryMap.get(db), true);
LOG.info("Loaded resources for db " + db + " resources: " + dbResEntryMap.get(db));
} catch (LensException exc) {
+ incrCounter(LOAD_RESOURCES_ERRORS);
LOG.warn("Failed to load resources for db " + db, exc);
classLoaderCache.remove(db);
}
@@ -245,6 +279,7 @@ public class DatabaseResourceService extends AbstractService {
try {
newUrls.add(new URL(res.getLocation()));
} catch (MalformedURLException e) {
+ incrCounter(LOAD_RESOURCES_ERRORS);
LOG.error("Invalid URL " + res.getLocation() + " adding to db " + database, e);
}
}
@@ -292,4 +327,14 @@ public class DatabaseResourceService extends AbstractService {
public Collection<LensSessionImpl.ResourceEntry> getResourcesForDatabase(String database) {
return dbResEntryMap.get(database);
}
+
+ private MetricsService getMetrics() {
+ if (metricsService == null) {
+ metricsService = (MetricsService) LensServices.get().getService(MetricsService.NAME);
+ if (metricsService == null) {
+ throw new NullPointerException("Could not get metrics service");
+ }
+ }
+ return metricsService;
+ }
}
[32/51] [abbrv] incubator-lens git commit: LENS-613: Build failure
due to checkstyle issues after LENS-611
Posted by jd...@apache.org.
LENS-613: Build failure due to checkstyle issues after LENS-611
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/37d2fc10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/37d2fc10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/37d2fc10
Branch: refs/heads/current-release-line
Commit: 37d2fc1050833cfe237e75b4306e6eb9dc07b745
Parents: ede537e
Author: Himanshu Gahlaut <ga...@gmail.com>
Authored: Wed Jun 17 16:38:46 2015 +0530
Committer: Himanshu Gahlaut <hi...@apache.org>
Committed: Wed Jun 17 16:38:46 2015 +0530
----------------------------------------------------------------------
.../lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/37d2fc10/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
index 398c45e..501a4b0 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/FieldsCannotBeQueriedTogetherTest.java
@@ -258,7 +258,7 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
/* newmeasure is not present in any derived cube, hence the query should be disallowed. */
testFieldsCannotBeQueriedTogetherError("select newmeasure from basecube where " + TWO_DAYS_RANGE,
- Arrays.asList( "d_time", "newmeasure"));
+ Arrays.asList("d_time", "newmeasure"));
}
@Test
@@ -324,8 +324,8 @@ public class FieldsCannotBeQueriedTogetherTest extends TestQueryRewrite {
Configuration queryConf = new Configuration(conf);
queryConf.setBoolean(CubeQueryConfUtil.REPLACE_TIMEDIM_WITH_PART_COL, false);
- testFieldsCannotBeQueriedTogetherError("select msr4 from basecube where " + "time_range_in(d_time, '" +
- getDateUptoHours(TWODAYS_BACK) + "','" + getDateUptoHours(CubeTestSetup.NOW) + "')",
+ testFieldsCannotBeQueriedTogetherError("select msr4 from basecube where " + "time_range_in(d_time, '"
+ + getDateUptoHours(TWODAYS_BACK) + "','" + getDateUptoHours(CubeTestSetup.NOW) + "')",
Arrays.asList("d_time", "msr4"), queryConf);
}
[22/51] [abbrv] incubator-lens git commit: LENS-599 : Include only
fact subqueries to filter fact records in columnar rewriter
Posted by jd...@apache.org.
LENS-599 : Include only fact subqueries to filter fact records in columnar rewriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/7a7bdecb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/7a7bdecb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/7a7bdecb
Branch: refs/heads/current-release-line
Commit: 7a7bdecb3a7b97b4860534fdd23b8ec02433a860
Parents: 0776290
Author: Sushil Mohanty <su...@gmail.com>
Authored: Fri Jun 12 11:28:33 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jun 12 11:28:33 2015 +0530
----------------------------------------------------------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 5 ++-
.../driver/jdbc/TestColumnarSQLRewriter.java | 41 +++++++++++++++++++-
2 files changed, 44 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7a7bdecb/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 51a06c4..9ba136f 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -537,7 +537,10 @@ public class ColumnarSQLRewriter implements QueryRewriter {
// dim_table.key2 = 'abc' and dim_table.key3 = 'xyz'"
subquery = queryphase1.concat(factFilters.toString().substring(0, factFilters.toString().lastIndexOf("and")))
.concat(")");
- allSubQueries.append(subquery).append(" and ");
+ // include subqueries which are applicable only to filter records from fact
+ if (subquery.matches("(.*)" + getFactAlias() + "(.*)")) {
+ allSubQueries.append(subquery).append(" and ");
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7a7bdecb/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
index 556ca2c..aab7f31 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestColumnarSQLRewriter.java
@@ -126,7 +126,7 @@ public class TestColumnarSQLRewriter {
*
* create table item_dim ( item_key integer, item_name varchar(500) );
*
- * create table branch_dim ( branch_key integer, branch_key varchar(100));
+ * create table branch_dim ( branch_key integer, branch_name varchar(100));
*
* create table location_dim (location_key integer,location_name varchar(100));
*/
@@ -595,6 +595,45 @@ public class TestColumnarSQLRewriter {
}
@Test
+ public void testFilter() throws ParseException, SemanticException, LensException {
+
+ String query = "select max(fact.dollars_sold) from sales_fact fact "
+ + "inner join time_dim time_dim on fact.time_key = time_dim.time_key "
+ + "inner join location_dim location_dim on fact.location_key = location_dim.location_key "
+ + "inner join item_dim item_dim on fact.item_key = item_dim.item_key and "
+ + "inner join branch_dim branch_dim on branch_dim.branch_key = location_dim.location_key "
+ + "and location_dim.location_name = 'test123' "
+ + "where time_dim.time_key between date_add('2013-01-01', 1) and date_sub('2013-01-31',3) "
+ + "and item_dim.item_name = 'item_1' group by fact.time_key,time_dim.day_of_week,time_dim.day,"
+ + "item_dim.item_key " + "order by dollars_sold";
+
+ SessionState.start(hconf);
+
+ String actual = qtest.rewrite(query, conf, hconf);
+ String expected = "select max(max_sales_fact___fact_dollars_sold) from (select sales_fact___fact.time_key, "
+ + "sales_fact___fact.location_key, sales_fact___fact.item_key,max(( sales_fact___fact . dollars_sold )) "
+ + "as max_sales_fact___fact_dollars_sold from sales_fact sales_fact___fact where sales_fact___fact.time_key "
+ + "in ( select time_dim .time_key from time_dim where ( time_dim. time_key ) "
+ + "between date_add( '2013-01-01' , interval 1 day) and date_sub( '2013-01-31' , interval 3 day) ) "
+ + "and sales_fact___fact.location_key in ( select location_dim .location_key from location_dim where "
+ + "(( location_dim. location_name ) = 'test123' ) ) and sales_fact___fact.item_key in "
+ + "( select item_dim .item_key from item_dim where (( item_dim. item_name ) = 'item_1' ) ) "
+ + "group by sales_fact___fact.time_key, sales_fact___fact.location_key, sales_fact___fact.item_key) "
+ + "sales_fact___fact inner join time_dim time_dim___time_dim on (( sales_fact___fact . time_key ) = "
+ + "( time_dim___time_dim . time_key )) inner join location_dim location_dim___location_dim on "
+ + "(( sales_fact___fact . location_key ) = ( location_dim___location_dim . location_key )) "
+ + "inner join item_dim item_dim___item_dim on ((( sales_fact___fact . item_key ) = "
+ + "( item_dim___item_dim . item_key )) and inner ) inner join branch_dim branch_dim___branch_dim "
+ + "on ((( branch_dim___branch_dim . branch_key ) = ( location_dim___location_dim . location_key )) "
+ + "and (( location_dim___location_dim . location_name ) = 'test123' )) "
+ + "where (( time_dim___time_dim . time_key ) between date_add( '2013-01-01' , interval 1 day) "
+ + "and date_sub( '2013-01-31' , interval 3 day) and (( item_dim___item_dim . item_name ) = 'item_1' )) "
+ + "group by ( sales_fact___fact . time_key ), ( time_dim___time_dim . day_of_week ), "
+ + "( time_dim___time_dim . day ), ( item_dim___item_dim . item_key ) order by dollars_sold asc";
+ compareQueries(expected, actual);
+ }
+
+ @Test
public void testCountReplace() throws ParseException, SemanticException, LensException {
String query = "SELECT count(location_dim.name) FROM location_dim "
[19/51] [abbrv] incubator-lens git commit: LENS-174: (incremental)
Cube rewriter changes to support multiple expressions
Posted by jd...@apache.org.
LENS-174: (incremental) Cube rewriter changes to support multiple expressions
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/099d19e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/099d19e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/099d19e8
Branch: refs/heads/current-release-line
Commit: 099d19e881332b90eadc1fb36b443d2954c11bd2
Parents: ad471b2
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Jun 9 12:16:01 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:16:01 2015 +0530
----------------------------------------------------------------------
.../lens/cube/parse/AggregateResolver.java | 6 +-
.../apache/lens/cube/parse/AliasReplacer.java | 6 +-
.../lens/cube/parse/CandidateTableResolver.java | 75 ++++++++++++--------
.../lens/cube/parse/CubeQueryContext.java | 15 ++--
.../lens/cube/parse/ExpressionResolver.java | 33 ++++-----
5 files changed, 75 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index 2ad2b7c..edf0f74 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -218,7 +218,7 @@ class AggregateResolver implements ContextRewriter {
if (node.getChild(0).getType() == HiveParser.Identifier) {
function = BaseSemanticAnalyzer.unescapeIdentifier(node.getChild(0).getText());
}
- } else if (cubeql.isMeasure(node)) {
+ } else if (cubeql.isCubeMeasure(node)) {
// Exit for the recursion
String colname;
@@ -291,7 +291,7 @@ class AggregateResolver implements ContextRewriter {
boolean isDistinct = hasDistinct;
if (exprTokenType == HiveParser.TOK_FUNCTIONDI || exprTokenType == HiveParser.TOK_SELECTDI) {
isDistinct = true;
- } else if (cubeql.isMeasure(node) && isDistinct) {
+ } else if (cubeql.isCubeMeasure(node) && isDistinct) {
// Exit for the recursion
return true;
}
@@ -310,7 +310,7 @@ class AggregateResolver implements ContextRewriter {
return false;
}
- if (cubeql.isMeasure(node)) {
+ if (cubeql.isCubeMeasure(node)) {
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
index 4d3443c..9309307 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AliasReplacer.java
@@ -120,11 +120,11 @@ class AliasReplacer implements ContextRewriter {
}
private void extractTabAliasForCol(CubeQueryContext cubeql) throws SemanticException {
- extractTabAliasForCol(cubeql.getColToTableAlias(), cubeql, cubeql);
+ extractTabAliasForCol(cubeql, cubeql);
}
- static void extractTabAliasForCol(Map<String, String> colToTableAlias, CubeQueryContext cubeql,
- TrackQueriedColumns tqc) throws SemanticException {
+ static void extractTabAliasForCol(CubeQueryContext cubeql, TrackQueriedColumns tqc) throws SemanticException {
+ Map<String, String> colToTableAlias = cubeql.getColToTableAlias();
Set<String> columns = tqc.getTblAliasToColumns().get(CubeQueryContext.DEFAULT_TABLE);
if (columns == null) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index d56fb80..53fb11f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -284,23 +284,35 @@ class CandidateTableResolver implements ContextRewriter {
(!queriedDimAttrs.isEmpty() ? queriedDimAttrs.toString() : "")
+ (!dimExprs.isEmpty() ? dimExprs.toString() : ""));
}
- // Find out candidate fact table sets which contain all the measures
- // queried
- List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts());
- Set<Set<CandidateFact>> cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs,
- cubeql.getQueriedExprsWithMeasures());
- LOG.info("Measure covering fact sets :" + cfactset);
- String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "")
- + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : "");
- if (cfactset.isEmpty()) {
- throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
- }
- cubeql.getCandidateFactSets().addAll(cfactset);
- cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs,
- cubeql.getQueriedExprsWithMeasures()));
+ Set<Set<CandidateFact>> cfactset;
+ if (queriedMsrs.isEmpty() && cubeql.getQueriedExprsWithMeasures().isEmpty()) {
+ // if no measures are queried, add all facts individually as single covering sets
+ cfactset = new HashSet<Set<CandidateFact>>();
+ for (CandidateFact cfact : cubeql.getCandidateFacts()) {
+ Set<CandidateFact> one = new LinkedHashSet<CandidateFact>();
+ one.add(cfact);
+ cfactset.add(one);
+ }
+ cubeql.getCandidateFactSets().addAll(cfactset);
+ } else {
+ // Find out candidate fact table sets which contain all the measures
+ // queried
+ List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts());
+ cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs,
+ cubeql.getQueriedExprsWithMeasures());
+ LOG.info("Measure covering fact sets :" + cfactset);
+ String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "")
+ + (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : "");
+ if (cfactset.isEmpty()) {
+ throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
+ }
+ cubeql.getCandidateFactSets().addAll(cfactset);
+ cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCause.columnNotFound(queriedMsrs,
+ cubeql.getQueriedExprsWithMeasures()));
- if (cubeql.getCandidateFacts().size() == 0) {
- throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
+ if (cubeql.getCandidateFacts().size() == 0) {
+ throw new SemanticException(ErrorMsg.NO_FACT_HAS_COLUMN, msrString);
+ }
}
}
}
@@ -312,10 +324,10 @@ class CandidateTableResolver implements ContextRewriter {
for (Iterator<CandidateFact> i = cfacts.iterator(); i.hasNext();) {
CandidateFact cfact = i.next();
i.remove();
- if (!checkForColumnExists(cfact, msrs)
+ // cfact does not contain any of msrs and none of exprsWithMeasures are evaluable.
+ if ((msrs.isEmpty() || !checkForColumnExists(cfact, msrs))
&& (exprsWithMeasures.isEmpty() || cubeql.getExprCtx().allNotEvaluable(exprsWithMeasures, cfact))) {
- // check if fact contains any of the maeasures
- // if not ignore the fact
+ // ignore the fact
continue;
} else if (cfact.getColumns().containsAll(msrs) && cubeql.getExprCtx().allEvaluable(cfact, exprsWithMeasures)) {
// return single set
@@ -324,18 +336,21 @@ class CandidateTableResolver implements ContextRewriter {
cfactset.add(one);
} else {
// find the remaining measures in other facts
- Set<String> remainingMsrs = new HashSet<String>(msrs);
- Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures);
- remainingMsrs.removeAll(cfact.getColumns());
- remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact));
- Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs);
- if (!coveringSets.isEmpty()) {
- for (Set<CandidateFact> set : coveringSets) {
- set.add(cfact);
- cfactset.add(set);
+ if (i.hasNext()) {
+ Set<String> remainingMsrs = new HashSet<String>(msrs);
+ Set<String> remainingExprs = new HashSet<String>(exprsWithMeasures);
+ remainingMsrs.removeAll(cfact.getColumns());
+ remainingExprs.removeAll(cubeql.getExprCtx().coveringExpressions(exprsWithMeasures, cfact));
+ Set<Set<CandidateFact>> coveringSets = findCoveringSets(cubeql, cfacts, remainingMsrs, remainingExprs);
+ if (!coveringSets.isEmpty()) {
+ for (Set<CandidateFact> set : coveringSets) {
+ set.add(cfact);
+ cfactset.add(set);
+ }
+ } else {
+ LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs
+ + " in " + cfactsPassed);
}
- } else {
- LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 0409af3..c5d8d91 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -21,7 +21,6 @@ package org.apache.lens.cube.parse;
import static org.apache.hadoop.hive.ql.parse.HiveParser.Identifier;
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TABLE_OR_COL;
-
import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE;
import java.io.ByteArrayOutputStream;
@@ -803,20 +802,20 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
// pick dimension tables required during expression expansion for the picked fact and dimensions
- Set<Dimension> exprDimTables = new HashSet<Dimension>();
+ Set<Dimension> exprDimensions = new HashSet<Dimension>();
if (cfacts != null) {
for (CandidateFact cfact : cfacts) {
Set<Dimension> factExprDimTables = exprCtx.rewriteExprCtx(cfact, dimsToQuery, cfacts.size() > 1);
- exprDimTables.addAll(factExprDimTables);
+ exprDimensions.addAll(factExprDimTables);
if (cfacts.size() > 1) {
factDimMap.get(cfact).addAll(factExprDimTables);
}
}
} else {
// dim only query
- exprDimTables.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false));
+ exprDimensions.addAll(exprCtx.rewriteExprCtx(null, dimsToQuery, false));
}
- dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimTables));
+ dimsToQuery.putAll(pickCandidateDimsToQuery(exprDimensions));
// pick denorm tables for the picked fact and dimensions
Set<Dimension> denormTables = new HashSet<Dimension>();
@@ -927,7 +926,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
if (split.length <= 1) {
col = col.trim().toLowerCase();
if (queriedExprs.contains(col)) {
- return exprCtx.getExpressionContext(col, getAliasForTableName(cube.getName())).isHasMeasures();
+ return exprCtx.getExpressionContext(col, getAliasForTableName(cube.getName())).hasMeasures();
} else {
return cube.getMeasureNames().contains(col);
}
@@ -936,7 +935,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
String colName = split[1].trim().toLowerCase();
if (cubeName.equalsIgnoreCase(cube.getName()) || cubeName.equals(getAliasForTableName(cube.getName()))) {
if (queriedExprs.contains(colName)) {
- return exprCtx.getExpressionContext(colName, cubeName).isHasMeasures();
+ return exprCtx.getExpressionContext(colName, cubeName).hasMeasures();
} else {
return cube.getMeasureNames().contains(colName.toLowerCase());
}
@@ -946,7 +945,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
}
- boolean isMeasure(ASTNode node) {
+ boolean isCubeMeasure(ASTNode node) {
String tabname = null;
String colname;
int nodeType = node.getToken().getType();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/099d19e8/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index f2a7039..539badb 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -70,9 +70,12 @@ class ExpressionResolver implements ContextRewriter {
private Set<CandidateTable> directlyAvailableIn = new HashSet<CandidateTable>();
private Map<CandidateTable, Set<ExprSpecContext>> evaluableExpressions =
new HashMap<CandidateTable, Set<ExprSpecContext>>();
- @Getter
private boolean hasMeasures = false;
+ public boolean hasMeasures() {
+ return hasMeasures;
+ }
+
ExpressionContext(CubeQueryContext cubeql, ExprColumn exprCol, AbstractBaseTable srcTable, String srcAlias)
throws SemanticException {
this.srcTable = srcTable;
@@ -82,13 +85,12 @@ class ExpressionResolver implements ContextRewriter {
allExprs.add(new ExprSpecContext(es, cubeql));
}
resolveColumnsAndAlias(cubeql);
+ log.debug("All exprs for {} are {}", exprCol.getName(), allExprs);
}
private void resolveColumnsAndAlias(CubeQueryContext cubeql) throws SemanticException {
for (ExprSpecContext esc : allExprs) {
esc.resolveColumns(cubeql);
- esc.replaceAliasInAST(cubeql, cubeql.getColToTableAlias());
- }
- for (ExprSpecContext esc : allExprs) {
+ esc.replaceAliasInAST(cubeql);
for (String table : esc.getTblAliasToColumns().keySet()) {
try {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
@@ -101,21 +103,18 @@ class ExpressionResolver implements ContextRewriter {
}
}
}
- resolveColumnsAndReplaceAlias(cubeql, allExprs, cubeql.getColToTableAlias());
+ resolveColumnsAndReplaceAlias(cubeql, allExprs);
}
- private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set<ExprSpecContext> exprs,
- Map<String, String> colToTableAlias) throws SemanticException {
+ private void resolveColumnsAndReplaceAlias(CubeQueryContext cubeql, Set<ExprSpecContext> exprs)
+ throws SemanticException {
Set<ExprSpecContext> nestedExpressions = new LinkedHashSet<ExprSpecContext>();
for (ExprSpecContext esc : exprs) {
- for (Map.Entry<String, Set<String>> entry : esc.tblAliasToColumns.entrySet()) {
+ for (Map.Entry<String, Set<String>> entry : esc.getTblAliasToColumns().entrySet()) {
if (entry.getKey().equals(CubeQueryContext.DEFAULT_TABLE)) {
continue;
}
AbstractBaseTable baseTable = (AbstractBaseTable)cubeql.getCubeTableForAlias(entry.getKey());
- // if (baseTable == null) {
- // continue;
- // }
Set<String> exprCols = new HashSet<String>();
for (String col : entry.getValue()) {
// col is an expression
@@ -129,11 +128,11 @@ class ExpressionResolver implements ContextRewriter {
}
for (ExprSpecContext esc : nestedExpressions) {
esc.resolveColumns(cubeql);
- esc.replaceAliasInAST(cubeql, colToTableAlias);
+ esc.replaceAliasInAST(cubeql);
for (String table : esc.getTblAliasToColumns().keySet()) {
try {
if (!CubeQueryContext.DEFAULT_TABLE.equalsIgnoreCase(table) && !srcAlias.equals(table)) {
- cubeql.addOptionalDimTable(table, null, true,
+ cubeql.addOptionalDimTable(table, null, false,
esc.getTblAliasToColumns().get(table).toArray(new String[0]));
esc.exprDims.add((Dimension) cubeql.getCubeTableForAlias(table));
}
@@ -217,10 +216,10 @@ class ExpressionResolver implements ContextRewriter {
exprSpecs.add(current);
finalAST = replaceAlias(node, cubeql);
}
- public void replaceAliasInAST(CubeQueryContext cubeql, Map<String, String> colToTableAlias)
+ public void replaceAliasInAST(CubeQueryContext cubeql)
throws SemanticException {
- AliasReplacer.extractTabAliasForCol(colToTableAlias, cubeql, this);
- AliasReplacer.replaceAliases(finalAST, 0, colToTableAlias);
+ AliasReplacer.extractTabAliasForCol(cubeql, this);
+ AliasReplacer.replaceAliases(finalAST, 0, cubeql.getColToTableAlias());
}
public void addColumnsQueried(String alias, String column) {
Set<String> cols = tblAliasToColumns.get(alias.toLowerCase());
@@ -368,12 +367,14 @@ class ExpressionResolver implements ContextRewriter {
if (!cTable.getColumns().contains(col.toLowerCase())) {
if (!cubeql.getDeNormCtx().addRefUsage(cTable, col, cTable.getBaseTable().getName())) {
// check if it is available as reference, if not expression is not evaluable
+ log.debug("{} = {} is not evaluable in {}", expr, esc, cTable);
isEvaluable = false;
break;
}
}
}
if (isEvaluable) {
+ log.debug("{} = {} is evaluable in {}", expr, esc, cTable);
ec.addEvaluable(cubeql, cTable, esc);
}
}
[15/51] [abbrv] incubator-lens git commit: LENS-8: Store test output
in test specific file
Posted by jd...@apache.org.
LENS-8: Store test output in test specific file
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/713104d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/713104d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/713104d9
Branch: refs/heads/current-release-line
Commit: 713104d928a79afe7bf2090fa896ceff3e544e26
Parents: 41b293c
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Tue Jun 9 12:03:14 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:03:14 2015 +0530
----------------------------------------------------------------------
lens-server/src/test/resources/log4j.properties | 2 +-
pom.xml | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/713104d9/lens-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/log4j.properties b/lens-server/src/test/resources/log4j.properties
index f3ccea2..e62afe1 100644
--- a/lens-server/src/test/resources/log4j.properties
+++ b/lens-server/src/test/resources/log4j.properties
@@ -35,7 +35,7 @@ log4j.appender.STDOUT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{
log4j.appender.STDOUT.Threshold=ERROR
log4j.appender.TEST_LOG_FILE=org.apache.log4j.RollingFileAppender
-log4j.appender.TEST_LOG_FILE.File=lens-server/target/test.log
+log4j.appender.TEST_LOG_FILE.File=target/test.log
log4j.appender.TEST_LOG_FILE.layout=org.apache.log4j.PatternLayout
log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %X{logSegregationId} [%t] %-5p %c %x - %m%n
log4j.appender.TEST_LOG_FILE.Threshold=ERROR
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/713104d9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index fb9df7c..1b2156f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -422,6 +422,7 @@
</excludes>
<failIfNoTests>false</failIfNoTests>
<forkedProcessTimeoutInSeconds>2400</forkedProcessTimeoutInSeconds>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
</plugin>
<plugin>
[27/51] [abbrv] incubator-lens git commit: LENS-555 : Add presubmit
hook in UserConfigLoader and call from drivers
Posted by jd...@apache.org.
LENS-555 : Add presubmit hook in UserConfigLoader and call from drivers
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/558ac1c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/558ac1c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/558ac1c7
Branch: refs/heads/current-release-line
Commit: 558ac1c75c154a1b483900174c32e62d3ea2350c
Parents: 022b589
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jun 12 16:01:34 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jun 12 16:01:34 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 27 ++++----
.../apache/lens/driver/hive/TestHiveDriver.java | 5 ++
.../lens/driver/hive/TestRemoteHiveDriver.java | 2 +
lens-driver-jdbc/pom.xml | 7 ++
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 26 ++++---
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 19 ++++--
.../lens/server/api/driver/LensDriver.java | 7 ++
.../lens/server/api/user/UserConfigLoader.java | 64 +++++++++++++++++
.../api/user/UserConfigLoaderException.java | 60 ++++++++++++++++
.../lens/server/api/driver/MockDriver.java | 4 ++
.../server/api/user/MockUserConfigLoader.java | 47 +++++++++++++
.../org/apache/lens/server/LensServices.java | 3 +-
.../server/query/QueryExecutionServiceImpl.java | 6 +-
.../server/user/CustomUserConfigLoader.java | 72 --------------------
.../server/user/DatabaseUserConfigLoader.java | 2 +
.../lens/server/user/FixedUserConfigLoader.java | 1 +
.../LDAPBackedDatabaseUserConfigLoader.java | 1 +
.../user/PropertyBasedUserConfigLoader.java | 2 +
.../lens/server/user/UserConfigLoader.java | 50 --------------
.../server/user/UserConfigLoaderException.java | 60 ----------------
.../server/user/UserConfigLoaderFactory.java | 30 ++++++--
.../lens/server/user/FooBarConfigLoader.java | 10 +++
.../lens/server/user/TestUserConfigLoader.java | 1 +
23 files changed, 288 insertions(+), 218 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 052d0e2..b7a3be7 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -18,18 +18,9 @@
*/
package org.apache.lens.driver.hive;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.TimeUnit;
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@@ -49,6 +40,7 @@ import org.apache.lens.server.api.priority.QueryPriorityDecider;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -135,6 +127,7 @@ public class HiveDriver implements LensDriver {
// package-local. Test case can change.
boolean whetherCalculatePriority;
+ private UserConfigLoader userConfigLoader;
private String sessionDbKey(String sessionHandle, String database) {
@@ -511,6 +504,9 @@ public class HiveDriver implements LensDriver {
+ "User query: " + ctx.getUserQuery(), e);
}
}
+ if (userConfigLoader != null) {
+ userConfigLoader.preSubmit(ctx);
+ }
OperationHandle op = getClient().executeStatementAsync(getSession(ctx), ctx.getSelectedDriverQuery(),
qdconf.getValByRegex(".*"));
ctx.setDriverOpHandle(op.toString());
@@ -727,7 +723,7 @@ public class HiveDriver implements LensDriver {
} catch (Exception e) {
checkInvalidSession(e);
log.warn("Error closing session for lens session: " + lensSessionDbKey + ", hive session: "
- + lensToHiveSession.get(lensSessionDbKey), e);
+ + lensToHiveSession.get(lensSessionDbKey), e);
}
}
lensToHiveSession.clear();
@@ -746,6 +742,11 @@ public class HiveDriver implements LensDriver {
driverListeners.add(driverEventListener);
}
+ @Override
+ public void registerUserConfigLoader(UserConfigLoader userConfigLoader) {
+ this.userConfigLoader = userConfigLoader;
+ }
+
protected CLIServiceClient getClient() throws LensException {
if (isEmbedded) {
if (embeddedConnection == null) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index b21258a..8355f29 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.driver.hive;
+import static org.apache.lens.server.api.user.MockUserConfigLoader.*;
+
import static org.testng.Assert.*;
import java.io.*;
@@ -42,6 +44,7 @@ import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.ExplainQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.user.MockUserConfigLoader;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -126,6 +129,7 @@ public class TestHiveDriver {
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
driver.configure(conf);
+ driver.registerUserConfigLoader(new MockUserConfigLoader(conf));
drivers = new ArrayList<LensDriver>() {
{
add(driver);
@@ -483,6 +487,7 @@ public class TestHiveDriver {
*/
protected void validateExecuteAsync(QueryContext ctx, DriverQueryState finalState, boolean isPersistent,
boolean formatNulls) throws Exception {
+ assertEquals(ctx.getSelectedDriverConf().get(KEY), VALUE);
validateExecuteAsync(ctx, finalState, isPersistent, formatNulls, driver);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index b4086b8..c144fd8 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -33,6 +33,7 @@ import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.user.MockUserConfigLoader;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -134,6 +135,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
driver = new HiveDriver();
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver.configure(conf);
+ driver.registerUserConfigLoader(new MockUserConfigLoader(conf));
drivers = new ArrayList<LensDriver>() {
{
add(driver);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml
index a7994c5..615410b 100644
--- a/lens-driver-jdbc/pom.xml
+++ b/lens-driver-jdbc/pom.xml
@@ -81,5 +81,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-server-api</artifactId>
+ <type>test-jar</type>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 70deaa4..bda6572 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -43,10 +43,8 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryRewriter;
+import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
@@ -93,6 +91,7 @@ public class JDBCDriver implements LensDriver {
private ConnectionProvider estimateConnectionProvider;
private LogSegregationContext logSegregationContext;
+ private UserConfigLoader userConfigLoader;
/**
* Data related to a query submitted to JDBCDriver.
@@ -545,6 +544,7 @@ public class JDBCDriver implements LensDriver {
}
private static final QueryCost JDBC_DRIVER_COST = new QueryCost(0, 0);
+
/**
* Dummy JDBC query Plan class to get min cost selector working.
*/
@@ -741,9 +741,9 @@ public class JDBCDriver implements LensDriver {
* @throws LensException
*/
private PreparedStatement prepareInternal(AbstractQueryContext pContext,
- boolean calledForEstimate,
- boolean checkConfigured,
- String metricCallStack) throws LensException {
+ boolean calledForEstimate,
+ boolean checkConfigured,
+ String metricCallStack) throws LensException {
// Caller might have already verified configured status and driver query, so we don't have
// to do this check twice. Caller must set checkConfigured to false in that case.
if (checkConfigured) {
@@ -888,6 +888,9 @@ public class JDBCDriver implements LensDriver {
String rewrittenQuery = rewriteQuery(context);
JdbcQueryContext jdbcCtx = new JdbcQueryContext(context, logSegregationContext);
jdbcCtx.setRewrittenQuery(rewrittenQuery);
+ if (userConfigLoader != null) {
+ userConfigLoader.preSubmit(context);
+ }
try {
Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext));
jdbcCtx.setResultFuture(future);
@@ -1057,7 +1060,7 @@ public class JDBCDriver implements LensDriver {
throw new LensException();
}
} catch (LensException e) {
- LOG.warn("Error closing prapared query : " + query , e);
+ LOG.warn("Error closing prapared query : " + query, e);
}
}
} finally {
@@ -1075,6 +1078,13 @@ public class JDBCDriver implements LensDriver {
}
+
+ @Override
+ public void registerUserConfigLoader(UserConfigLoader userConfigLoader) {
+ this.userConfigLoader = userConfigLoader;
+ }
+
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 2e3dcc7..be34164 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.driver.jdbc;
+import static org.apache.lens.server.api.user.MockUserConfigLoader.*;
+
import static org.testng.Assert.*;
import java.sql.Connection;
@@ -41,6 +43,7 @@ import org.apache.lens.server.api.metrics.LensMetricsRegistry;
import org.apache.lens.server.api.query.ExplainQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.user.MockUserConfigLoader;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -91,6 +94,7 @@ public class TestJdbcDriver {
driver = new JDBCDriver();
driver.configure(baseConf);
+ driver.registerUserConfigLoader(new MockUserConfigLoader(hConf));
assertNotNull(driver);
assertTrue(driver.configured);
@@ -556,7 +560,7 @@ public class TestJdbcDriver {
}
};
- driver.executeAsync(context);
+ executeAsync(context);
QueryHandle handle = context.getQueryHandle();
driver.registerForCompletionNotification(handle, 0, listener);
@@ -633,7 +637,7 @@ public class TestJdbcDriver {
QueryContext ctx = new QueryContext(query, "SA", new LensConf(), baseConf, drivers);
for (int i = 0; i < JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE_DEFAULT; i++) {
- driver.executeAsync(ctx);
+ executeAsync(ctx);
driver.updateStatus(ctx);
System.out.println("@@@@ QUERY " + (i + 1));
}
@@ -641,7 +645,7 @@ public class TestJdbcDriver {
String validQuery = "SELECT * FROM invalid_conn_close";
QueryContext validCtx = createQueryContext(validQuery);
System.out.println("@@@ Submitting valid query");
- driver.executeAsync(validCtx);
+ executeAsync(validCtx);
// Wait for query to finish
while (true) {
@@ -655,6 +659,11 @@ public class TestJdbcDriver {
driver.closeQuery(validCtx.getQueryHandle());
}
+ private void executeAsync(QueryContext ctx) throws LensException {
+ driver.executeAsync(ctx);
+ assertEquals(ctx.getSelectedDriverConf().get(KEY), VALUE);
+ }
+
/**
* Test connection close for successful queries.
*
@@ -706,7 +715,7 @@ public class TestJdbcDriver {
final String query = "SELECT * FROM cancel_query_test";
QueryContext context = createQueryContext(query);
System.out.println("@@@ test_cancel:" + context.getQueryHandle());
- driver.executeAsync(context);
+ executeAsync(context);
QueryHandle handle = context.getQueryHandle();
boolean isCancelled = driver.cancelQuery(handle);
driver.updateStatus(context);
@@ -753,7 +762,7 @@ public class TestJdbcDriver {
}
};
- driver.executeAsync(ctx);
+ executeAsync(ctx);
QueryHandle handle = ctx.getQueryHandle();
driver.registerForCompletionNotification(handle, 0, listener);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 8511ce6..0c5ee0a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -28,6 +28,7 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.conf.Configuration;
@@ -182,4 +183,10 @@ public interface LensDriver extends Externalizable {
* @param driverEventListener the driver event listener
*/
void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener);
+
+ /**
+ * Add the user config loader to driver for use
+ * @param userConfigLoader
+ */
+ void registerUserConfigLoader(UserConfigLoader userConfigLoader);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java
new file mode 100644
index 0000000..93e11a2
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.user;
+
+import java.util.Map;
+
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * The Class UserConfigLoader. It's initialized once in the server lifetime. After that, t:
+ * 1. Gets session configs for the user on each session open. This config applies to the particular session
+ * and is forwarded for all actions. One Use case is to decide driver specific details e.g. priority/queue of
+ * all queries of the user.
+ * 2. Provides a pre-submit hook. Just before submission
+ */
+@Slf4j
+public abstract class UserConfigLoader {
+
+ /** The hive conf. */
+ protected final HiveConf hiveConf;
+
+ /**
+ * Instantiates a new user config loader.
+ *
+ * @param conf the conf
+ */
+ protected UserConfigLoader(HiveConf conf) {
+ this.hiveConf = conf;
+ }
+
+ /**
+ * Gets the user config.
+ *
+ * @param loggedInUser the logged in user
+ * @return the user config
+ * @throws UserConfigLoaderException the user config loader exception
+ */
+ public abstract Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException;
+
+ public void preSubmit(QueryContext ctx) throws LensException {
+ log.debug("Pre submit " + ctx + " on " + ctx.getSelectedDriver());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java
new file mode 100644
index 0000000..c1cbb24
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoaderException.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.user;
+
+/**
+ * The Class UserConfigLoaderException.
+ */
+public class UserConfigLoaderException extends RuntimeException {
+
+ /**
+ * Instantiates a new user config loader exception.
+ */
+ public UserConfigLoaderException() {
+ super();
+ }
+
+ /**
+ * Instantiates a new user config loader exception.
+ *
+ * @param s the s
+ */
+ public UserConfigLoaderException(String s) {
+ super(s);
+ }
+
+ /**
+ * Instantiates a new user config loader exception.
+ *
+ * @param e the e
+ */
+ public UserConfigLoaderException(Throwable e) {
+ super(e);
+ }
+
+ /**
+ * Instantiates a new user config loader exception.
+ *
+ * @param message the message
+ * @param cause the cause
+ */
+ public UserConfigLoaderException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 30c1a71..4b80918 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -35,6 +35,7 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hive.service.cli.ColumnDescriptor;
@@ -182,7 +183,10 @@ public class MockDriver implements LensDriver {
*/
@Override
public void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener) {
+ }
+ @Override
+ public void registerUserConfigLoader(UserConfigLoader userConfigLoader) {
}
/*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java
new file mode 100644
index 0000000..f6ef0d4
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.user;
+
+import java.util.Map;
+
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import com.beust.jcommander.internal.Maps;
+
+public class MockUserConfigLoader extends UserConfigLoader {
+ public static final String KEY = "TEST_KEY";
+ public static final String VALUE = "TEST_VALUE";
+
+ public MockUserConfigLoader(HiveConf conf) {
+ super(conf);
+ }
+
+ @Override
+ public Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException {
+ return Maps.newHashMap();
+ }
+
+ @Override
+ public void preSubmit(QueryContext ctx) throws LensException {
+ ctx.getSelectedDriverConf().set(KEY, VALUE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index c27717f..5fe3935 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -191,7 +191,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
serviceMode = conf.getEnum(SERVER_MODE,
SERVICE_MODE.valueOf(DEFAULT_SERVER_MODE));
cliService = new CLIService();
-
+ UserConfigLoaderFactory.init(conf);
// Add default services
addService(cliService);
addService(new EventServiceImpl(LensEventService.NAME));
@@ -262,7 +262,6 @@ public class LensServices extends CompositeService implements ServiceProvider {
snapShotInterval = conf.getLong(SERVER_SNAPSHOT_INTERVAL,
DEFAULT_SERVER_SNAPSHOT_INTERVAL);
LOG.info("Initialized services: " + services.keySet().toString());
- UserConfigLoaderFactory.init(conf);
timer = new Timer("lens-server-snapshotter", true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 7623743..e41a0d2 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -55,6 +55,7 @@ import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
+import org.apache.lens.server.user.UserConfigLoaderFactory;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.collections.CollectionUtils;
@@ -307,7 +308,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
Class<?> clazz = Class.forName(driverClass);
LensDriver driver = (LensDriver) clazz.newInstance();
driver.configure(LensServerConf.getConf());
-
+ driver.registerUserConfigLoader(UserConfigLoaderFactory.getUserConfigLoader());
if (driver instanceof HiveDriver) {
driver.registerDriverEventListener(driverEventListener);
}
@@ -895,7 +896,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
loadDriversAndSelector();
} catch (LensException e) {
- throw new IllegalStateException("Could not load drivers");
+ LOG.error(e);
+ throw new IllegalStateException("Could not load drivers", e);
}
maxFinishedQueries = conf.getInt(LensConfConstants.MAX_NUMBER_OF_FINISHED_QUERY,
LensConfConstants.DEFAULT_FINISHED_QUERIES);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java
deleted file mode 100644
index 9a6a350..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/user/CustomUserConfigLoader.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.user;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.Map;
-
-import org.apache.lens.server.api.LensConfConstants;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * The Class CustomUserConfigLoader.
- */
-public class CustomUserConfigLoader extends UserConfigLoader {
-
- /** The custom handler class. */
- Class<? extends UserConfigLoader> customHandlerClass;
-
- /** The custom provider. */
- UserConfigLoader customProvider;
-
- /**
- * Instantiates a new custom user config loader.
- *
- * @param conf the conf
- */
- public CustomUserConfigLoader(HiveConf conf) {
- super(conf);
- this.customHandlerClass = (Class<? extends UserConfigLoader>) hiveConf.getClass(
- LensConfConstants.USER_RESOLVER_CUSTOM_CLASS, UserConfigLoader.class);
- try {
- this.customProvider = customHandlerClass.getConstructor(HiveConf.class).newInstance(conf);
- // in java6, these four extend directly from Exception. So have to handle separately. In java7,
- // the common subclass is ReflectiveOperationException
- } catch (InvocationTargetException e) {
- throw new UserConfigLoaderException(e);
- } catch (NoSuchMethodException e) {
- throw new UserConfigLoaderException(e);
- } catch (InstantiationException e) {
- throw new UserConfigLoaderException(e);
- } catch (IllegalAccessException e) {
- throw new UserConfigLoaderException(e);
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.user.UserConfigLoader#getUserConfig(java.lang.String)
- */
- @Override
- public Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException {
- return customProvider.getUserConfig(loggedInUser);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
index d267948..ce0d112 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
@@ -26,6 +26,8 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.user.UserConfigLoader;
+import org.apache.lens.server.api.user.UserConfigLoaderException;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.dbcp.BasicDataSource;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
index b81b994..5813ac0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
@@ -22,6 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.hive.conf.HiveConf;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java
index 108a012..82b76c9 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/LDAPBackedDatabaseUserConfigLoader.java
@@ -34,6 +34,7 @@ import javax.naming.directory.SearchResult;
import javax.naming.ldap.InitialLdapContext;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.user.UserConfigLoaderException;
import org.apache.commons.dbutils.QueryRunner;
import org.apache.hadoop.hive.conf.HiveConf;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
index 16a9a70..009e34a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
@@ -27,6 +27,8 @@ import java.util.Map;
import java.util.Properties;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.user.UserConfigLoader;
+import org.apache.lens.server.api.user.UserConfigLoaderException;
import org.apache.hadoop.hive.conf.HiveConf;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java
deleted file mode 100644
index 21eea3b..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoader.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.user;
-
-import java.util.Map;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * The Class UserConfigLoader.
- */
-public abstract class UserConfigLoader {
-
- /** The hive conf. */
- protected final HiveConf hiveConf;
-
- /**
- * Instantiates a new user config loader.
- *
- * @param conf the conf
- */
- protected UserConfigLoader(HiveConf conf) {
- this.hiveConf = conf;
- }
-
- /**
- * Gets the user config.
- *
- * @param loggedInUser the logged in user
- * @return the user config
- * @throws UserConfigLoaderException the user config loader exception
- */
- public abstract Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java
deleted file mode 100644
index 0899746..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderException.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.user;
-
-/**
- * The Class UserConfigLoaderException.
- */
-public class UserConfigLoaderException extends RuntimeException {
-
- /**
- * Instantiates a new user config loader exception.
- */
- public UserConfigLoaderException() {
- super();
- }
-
- /**
- * Instantiates a new user config loader exception.
- *
- * @param s the s
- */
- public UserConfigLoaderException(String s) {
- super(s);
- }
-
- /**
- * Instantiates a new user config loader exception.
- *
- * @param e the e
- */
- public UserConfigLoaderException(Throwable e) {
- super(e);
- }
-
- /**
- * Instantiates a new user config loader exception.
- *
- * @param message the message
- * @param cause the cause
- */
- public UserConfigLoaderException(String message, Throwable cause) {
- super(message, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
index 383aaf5..f92584b 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
@@ -18,9 +18,14 @@
*/
package org.apache.lens.server.user;
+import static org.apache.lens.server.api.LensConfConstants.USER_RESOLVER_CUSTOM_CLASS;
+
+import java.lang.reflect.InvocationTargetException;
import java.util.Map;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.user.UserConfigLoader;
+import org.apache.lens.server.api.user.UserConfigLoaderException;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -31,6 +36,7 @@ public final class UserConfigLoaderFactory {
private UserConfigLoaderFactory() {
}
+
/** The conf. */
private static HiveConf conf;
@@ -44,13 +50,13 @@ public final class UserConfigLoaderFactory {
*/
public static void init(HiveConf c) {
conf = c;
- userConfigLoader = initializeUserConfigLoader();
+ userConfigLoader = null;
}
/**
* The Enum RESOLVER_TYPE.
*/
- public static enum RESOLVER_TYPE {
+ public enum RESOLVER_TYPE {
/** The fixed. */
FIXED,
@@ -68,6 +74,13 @@ public final class UserConfigLoaderFactory {
CUSTOM
}
+ public static UserConfigLoader getUserConfigLoader() {
+ if (userConfigLoader == null) {
+ userConfigLoader = initializeUserConfigLoader();
+ }
+ return userConfigLoader;
+ }
+
/**
* Initialize user config loader.
*
@@ -80,7 +93,7 @@ public final class UserConfigLoaderFactory {
}
for (RESOLVER_TYPE type : RESOLVER_TYPE.values()) {
if (type.name().equals(resolverType)) {
- return getQueryUserResolver(type);
+ return createUserConfigLoader(type);
}
}
throw new UserConfigLoaderException("user resolver type not determined. provided value: " + resolverType);
@@ -92,7 +105,7 @@ public final class UserConfigLoaderFactory {
* @param resolverType the resolver type
* @return the query user resolver
*/
- public static UserConfigLoader getQueryUserResolver(RESOLVER_TYPE resolverType) {
+ public static UserConfigLoader createUserConfigLoader(RESOLVER_TYPE resolverType) {
switch (resolverType) {
case PROPERTYBASED:
return new PropertyBasedUserConfigLoader(conf);
@@ -101,7 +114,12 @@ public final class UserConfigLoaderFactory {
case LDAP_BACKED_DATABASE:
return new LDAPBackedDatabaseUserConfigLoader(conf);
case CUSTOM:
- return new CustomUserConfigLoader(conf);
+ try {
+ return (conf.getClass(USER_RESOLVER_CUSTOM_CLASS, UserConfigLoader.class, UserConfigLoader.class))
+ .getConstructor(HiveConf.class).newInstance(conf);
+ } catch (InvocationTargetException | NoSuchMethodException | IllegalAccessException | InstantiationException e) {
+ throw new UserConfigLoaderException(e);
+ }
case FIXED:
default:
return new FixedUserConfigLoader(conf);
@@ -115,6 +133,6 @@ public final class UserConfigLoaderFactory {
* @return the user config
*/
public static Map<String, String> getUserConfig(String loggedInUser) {
- return userConfigLoader.getUserConfig(loggedInUser);
+ return getUserConfigLoader().getUserConfig(loggedInUser);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java b/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
index 5b33440..0a319ef 100644
--- a/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
+++ b/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
@@ -21,6 +21,10 @@ package org.apache.lens.server.user;
import java.util.HashMap;
import java.util.Map;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.user.UserConfigLoader;
+
import org.apache.hadoop.hive.conf.HiveConf;
/**
@@ -53,4 +57,10 @@ public class FooBarConfigLoader extends UserConfigLoader {
public Map<String, String> getUserConfig(String loggedInUser) {
return CONST_HASH_MAP;
}
+
+ @Override
+ public void preSubmit(QueryContext ctx) throws LensException {
+ super.preSubmit(ctx);
+ ctx.getSelectedDriverConf().set("random.conf.key", "random.conf.value");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/558ac1c7/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
index 4f399c2..a122d1e 100644
--- a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
+++ b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.dbcp.BasicDataSource;
[16/51] [abbrv] incubator-lens git commit: LENS-294: Make
SessionExpiryService run period configurable
Posted by jd...@apache.org.
LENS-294: Make SessionExpiryService run period configurable
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/10d647ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/10d647ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/10d647ff
Branch: refs/heads/current-release-line
Commit: 10d647fff0dfcefe8ac6c46d0ce6a3876f9c4ef1
Parents: 713104d
Author: Raju Bairishetti <ra...@inmobi.com>
Authored: Tue Jun 9 12:05:48 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jun 9 12:05:48 2015 +0530
----------------------------------------------------------------------
.../lens/server/api/LensConfConstants.java | 8 +++
.../lens/server/session/HiveSessionService.java | 15 ++++-
.../src/main/resources/lensserver-default.xml | 5 ++
.../lens/server/session/TestSessionExpiry.java | 45 ++++++++++++-
src/site/apt/admin/config.apt | 66 ++++++++++----------
5 files changed, 105 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index e53e66c..ad27d78 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -712,6 +712,14 @@ public final class LensConfConstants {
*/
public static final String AUX_JARS = SESSION_PFX + "aux.jars";
+ /**
+ * Interval at which lens session expiry service runs
+ */
+ public static final String SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS = SERVER_PFX
+ + "session.expiry.serivce.interval.secs";
+
+ public static final int DEFAULT_SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS = 3600;
+
// Statistics Store configuration keys
/**
* The Constant STATS_STORE_CLASS.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index 6dc4ea6..fa061ec 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -81,6 +81,11 @@ public class HiveSessionService extends LensService implements SessionService {
private DatabaseResourceService databaseResourceService;
/**
+ * The conf.
+ */
+ private Configuration conf;
+
+ /**
* Instantiates a new hive session service.
*
* @param cliService the cli service
@@ -334,6 +339,7 @@ public class HiveSessionService extends LensService implements SessionService {
public synchronized void init(HiveConf hiveConf) {
this.databaseResourceService = new DatabaseResourceService(DatabaseResourceService.NAME);
addService(this.databaseResourceService);
+ this.conf = hiveConf;
super.init(hiveConf);
}
@@ -347,7 +353,9 @@ public class HiveSessionService extends LensService implements SessionService {
super.start();
sessionExpiryThread = Executors.newSingleThreadScheduledExecutor();
- sessionExpiryThread.scheduleWithFixedDelay(sessionExpiryRunnable, 60, 60, TimeUnit.MINUTES);
+ int sessionExpiryInterval = getSessionExpiryInterval();
+ sessionExpiryThread.scheduleWithFixedDelay(sessionExpiryRunnable, sessionExpiryInterval,
+ sessionExpiryInterval, TimeUnit.SECONDS);
// Restore sessions if any
if (restorableSessions == null || restorableSessions.size() <= 0) {
@@ -392,6 +400,11 @@ public class HiveSessionService extends LensService implements SessionService {
log.info("Session service restoed " + restorableSessions.size() + " sessions");
}
+ private int getSessionExpiryInterval() {
+ return conf.getInt(LensConfConstants.SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS,
+ LensConfConstants.DEFAULT_SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS);
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index 1a3f82f..da00c0d 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -695,4 +695,9 @@
</description>
</property>
+ <property>
+ <name>lens.server.session.expiry.service.interval.secs</name>
+ <value>3600</value>
+ <description>Interval at which lens session expiry service runs</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
index f494fd5..52402a0 100644
--- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
+++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionExpiry.java
@@ -29,14 +29,18 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.hadoop.hive.conf.HiveConf;
+
import org.apache.hive.service.cli.CLIService;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestSessionExpiry.
*/
@Test(groups = "unit-test")
+@Slf4j
public class TestSessionExpiry {
/**
@@ -45,7 +49,7 @@ public class TestSessionExpiry {
* @throws Exception the exception
*/
public void testSessionExpiry() throws Exception {
- HiveConf conf = LensServerConf.getHiveConf();
+ HiveConf conf = LensServerConf.createHiveConf();
conf.setVar(HiveConf.ConfVars.HIVE_SESSION_IMPL_CLASSNAME, LensSessionImpl.class.getName());
conf.setLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, 1L);
CLIService cliService = new CLIService();
@@ -64,6 +68,45 @@ public class TestSessionExpiry {
assertFalse(session.isActive());
// run the expiry thread
lensService.getSessionExpiryRunnable().run();
+ log.info("Keeping a sleep of 3 seconds to make sure SessionExpiryService gets enough time to close"
+ + " inactive sessions");
+ Thread.sleep(3000);
+ assertTrue(metricSvc.getTotalExpiredSessions() >= 1);
+ assertTrue(metricSvc.getTotalClosedSessions() >= 1);
+
+ try {
+ lensService.getSession(sessionHandle);
+ // should throw exception since session should be expired by now
+ fail("Expected get session to fail for session " + sessionHandle.getPublicId());
+ } catch (Exception e) {
+ // pass
+ }
+ } finally {
+ lensService.stop();
+ }
+ }
+
+ public void testSessionExpiryInterval() throws Exception {
+ HiveConf conf = LensServerConf.createHiveConf();
+ conf.setVar(HiveConf.ConfVars.HIVE_SESSION_IMPL_CLASSNAME, LensSessionImpl.class.getName());
+ conf.setLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, 1L);
+ conf.setInt(LensConfConstants.SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS, 1);
+ CLIService cliService = new CLIService();
+ cliService.init(conf);
+ HiveSessionService lensService = new HiveSessionService(cliService);
+ lensService.init(conf);
+ lensService.start();
+ MetricsService metricSvc = (MetricsService) LensServices.get().getService(MetricsService.NAME);
+ try {
+ LensSessionHandle sessionHandle = lensService.openSession("foo", "bar", new HashMap<String, String>());
+ LensSessionImpl session = lensService.getSession(sessionHandle);
+ assertTrue(session.isActive());
+ session.setLastAccessTime(session.getLastAccessTime() - 2000
+ * conf.getLong(LensConfConstants.SESSION_TIMEOUT_SECONDS, LensConfConstants.SESSION_TIMEOUT_SECONDS_DEFAULT));
+ assertFalse(session.isActive());
+ log.info("Keeping a sleep of 3 seconds to make sure SessionExpiryService is running and gets enough time to"
+ + "close inactive sessions");
+ Thread.sleep(3000);
assertTrue(metricSvc.getTotalExpiredSessions() >= 1);
assertTrue(metricSvc.getTotalClosedSessions() >= 1);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/10d647ff/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index 0ff204b..b4e63bc 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -149,68 +149,70 @@ Lens server configuration
*--+--+---+--+
|61|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|62|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|62|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
*--+--+---+--+
-|63|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|63|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|64|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|64|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|65|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
+|65|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|66|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|66|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
*--+--+---+--+
-|67|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|67|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|68|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|68|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|69|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|69|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|70|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|70|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|71|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
+|71|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|72|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
+|72|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
*--+--+---+--+
-|73|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|73|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
*--+--+---+--+
-|74|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
+|74|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|75|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|75|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|76|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|76|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|77|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|77|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|78|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|78|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|79|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|79|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|80|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|80|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|81|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|81|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|82|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|82|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|83|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|83|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|84|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|84|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|85|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|85|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|86|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|86|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|87|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|87|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|88|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|88|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|89|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|89|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|90|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|90|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
-|91|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|91|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|92|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|92|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|93|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|93|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+*--+--+---+--+
+|94|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values
[28/51] [abbrv] incubator-lens git commit: LENS-575 : Fix NPE when
unreachable dims are queried (amareshwari)
Posted by jd...@apache.org.
LENS-575 : Fix NPE when unreachable dims are queried (amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/b33e6e18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/b33e6e18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/b33e6e18
Branch: refs/heads/current-release-line
Commit: b33e6e18da3578afc442b016a7a168c8f0ba88db
Parents: 558ac1c
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Jun 12 16:15:46 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jun 12 16:15:46 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 2 +-
.../lens/cube/parse/CandidateTableResolver.java | 23 +++++++++---
.../apache/lens/cube/parse/CubeTestSetup.java | 39 ++++++++++++++++++++
.../lens/cube/parse/TestJoinResolver.java | 14 +++++++
4 files changed, 71 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index 2ae606f..cd898cc 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -448,7 +448,7 @@
</xs:documentation>
</xs:annotation>
</xs:attribute>
- <xs:attribute type="xs:boolean" name="join_key" default="false">
+ <xs:attribute type="xs:boolean" name="join_key" default="true">
<xs:annotation>
<xs:documentation>
This flag will tell whether the attribute can be used as a join key or not
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 53fb11f..79d6d43 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -493,19 +493,30 @@ class CandidateTableResolver implements ContextRewriter {
return;
}
// check for source columns for denorm columns
- Map<CandidateTable, List<String>> removedCandidates = new HashMap<CandidateTable, List<String>>();
+ Map<CandidateTable, Collection<String>> removedCandidates = new HashMap<CandidateTable, Collection<String>>();
for (Map.Entry<Dimension, OptionalDimCtx> optdimEntry : cubeql.getOptionalDimensionMap().entrySet()) {
Dimension dim = optdimEntry.getKey();
OptionalDimCtx optdim = optdimEntry.getValue();
Iterator<CandidateTable> iter = optdim.requiredForCandidates.iterator();
while (iter.hasNext()) {
CandidateTable candidate = iter.next();
- List<String> colSet = cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim).get(candidate.getBaseTable());
- if (!checkForColumnExists(candidate, colSet)) {
- LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as columns:" + colSet
- + " do not exist");
+ boolean remove = false;
+ if (cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim) == null) {
+ LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as no join paths"
+ + " exist");
+ remove = true;
+ removedCandidates.put(candidate, optdim.colQueried);
+ } else {
+ List<String> colSet = cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim).get(candidate.getBaseTable());
+ if (!checkForColumnExists(candidate, colSet)) {
+ LOG.info("Removing candidate" + candidate + " from requiredForCandidates of" + dim + ", as columns:"
+ + colSet + " do not exist");
+ remove = true;
+ removedCandidates.put(candidate, colSet);
+ }
+ }
+ if (remove) {
iter.remove();
- removedCandidates.put(candidate, colSet);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 554e709..49fabff 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -616,6 +616,8 @@ public class CubeTestSetup {
new TableReference("testdim2", "id")));
cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("cdim2", "int", "ref dim"), "Dim2 refer",
new TableReference("cycledim1", "id"), NOW, null, null));
+ cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer",
+ new TableReference("unreachableDim", "id"), null, null, null, false, 10L));
// denormalized reference
cubeDimensions.add(new ReferencedDimAtrribute(new FieldSchema("dim2big1", "bigint", "ref dim"), "Dim2 refer",
@@ -1573,6 +1575,8 @@ public class CubeTestSetup {
new TableReference("citydim", "id")));
dimAttrs.add(new ReferencedDimAtrribute(new FieldSchema("cityname", "string", "name"), "cityid",
new TableReference("citydim", "name"), null, null, 0.0, false));
+ dimAttrs.add(new ReferencedDimAtrribute(new FieldSchema("urdimid", "int", "ref dim"), "urdim refer",
+ new TableReference("unreachableDim", "id"), null, null, null, false, 10L));
// add ref dim through chain
dimAttrs.add(new ReferencedDimAtrribute(
@@ -1909,6 +1913,40 @@ public class CubeTestSetup {
client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables);
}
+ private void createUnReachabletable(CubeMetastoreClient client) throws Exception {
+ String dimName = "unreachableDim";
+
+ Set<CubeDimAttribute> dimAttrs = new HashSet<CubeDimAttribute>();
+ dimAttrs.add(new BaseDimAttribute(new FieldSchema("id", "int", "code")));
+ dimAttrs.add(new BaseDimAttribute(new FieldSchema("name", "int", "code")));
+ Map<String, String> dimProps = new HashMap<String, String>();
+ dimProps.put(MetastoreUtil.getDimTimedDimensionKey(dimName), TestCubeMetastoreClient.getDatePartitionKey());
+ Dimension urDim = new Dimension(dimName, dimAttrs, dimProps, 0L);
+ client.createDimension(urDim);
+
+ String dimTblName = "unreachableDimTable";
+ List<FieldSchema> dimColumns = new ArrayList<FieldSchema>();
+ dimColumns.add(new FieldSchema("id", "int", "code"));
+ dimColumns.add(new FieldSchema("name", "string", "field1"));
+
+ Map<String, UpdatePeriod> dumpPeriods = new HashMap<String, UpdatePeriod>();
+ ArrayList<FieldSchema> partCols = new ArrayList<FieldSchema>();
+ List<String> timePartCols = new ArrayList<String>();
+ partCols.add(TestCubeMetastoreClient.getDatePartition());
+ timePartCols.add(TestCubeMetastoreClient.getDatePartitionKey());
+ StorageTableDesc s1 = new StorageTableDesc();
+ s1.setInputFormat(TextInputFormat.class.getCanonicalName());
+ s1.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ s1.setPartCols(partCols);
+ s1.setTimePartCols(timePartCols);
+ dumpPeriods.put(c1, HOURLY);
+
+ Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
+ storageTables.put(c1, s1);
+
+ client.createCubeDimensionTable(dimName, dimTblName, dimColumns, 0L, dumpPeriods, dimProps, storageTables);
+ }
+
private void createCountryTable(CubeMetastoreClient client) throws Exception {
String dimName = "countrydim";
@@ -2050,6 +2088,7 @@ public class CubeTestSetup {
createCountryTable(client);
createStateTable(client);
createCubeFactsWithValidColumns(client);
+ createUnReachabletable(client);
} catch (Exception exc) {
log.error("Exception while creating sources.", exc);
throw exc;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/b33e6e18/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
index d121ba4..ed08605 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestJoinResolver.java
@@ -21,6 +21,8 @@ package org.apache.lens.cube.parse;
import static org.apache.lens.cube.parse.CubeTestSetup.*;
+import static org.testng.Assert.*;
+
import java.util.*;
import org.apache.lens.cube.metadata.*;
@@ -30,6 +32,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -739,4 +742,15 @@ public class TestJoinResolver extends TestQueryRewrite {
Assert.assertFalse(cdimTables.contains("citytable3"));
Assert.assertFalse(cdimTables.contains("citytable4"));
}
+
+ @Test
+ public void testUnreachableDim() throws ParseException, LensException {
+ SemanticException e1 = getSemanticExceptionInRewrite("select urdimid from testdim2", hconf);
+ assertNotNull(e1);
+ assertEquals(e1.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_DIM_HAS_COLUMN.getErrorCode());
+
+ SemanticException e2 = getSemanticExceptionInRewrite("select urdimid from testcube where " + TWO_DAYS_RANGE, hconf);
+ assertNotNull(e2);
+ assertEquals(e2.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode());
+ }
}
[10/51] [abbrv] incubator-lens git commit: LENS-588 : Fix building
source on windows environment (debargho chatterjee via amareshwari)
Posted by jd...@apache.org.
LENS-588 : Fix building source on windows environment (debargho chatterjee via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/85da8c83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/85da8c83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/85da8c83
Branch: refs/heads/current-release-line
Commit: 85da8c83101531f1ebfe2bb3d16ea0b352e6ed31
Parents: 1584f00
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Wed Jun 3 15:42:43 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jun 3 15:42:43 2015 +0530
----------------------------------------------------------------------
checkstyle/src/main/resources/checkstyle.xml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/85da8c83/checkstyle/src/main/resources/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml
index 031d664..0a37d01 100644
--- a/checkstyle/src/main/resources/checkstyle.xml
+++ b/checkstyle/src/main/resources/checkstyle.xml
@@ -57,7 +57,9 @@
<!-- Checks whether files end with a new line. -->
<!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
- <module name="NewlineAtEndOfFile"/>
+ <module name="NewlineAtEndOfFile">
+ <property name="lineSeparator" value="lf"/>
+ </module>
<module name="FileLength">
<property name="max" value="3500"/>
[33/51] [abbrv] incubator-lens git commit: LENS-515 : Support to run
spark in yarn-client mode in lens-ml module
Posted by jd...@apache.org.
LENS-515 : Support to run spark in yarn-client mode in lens-ml module
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/44a6f516
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/44a6f516
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/44a6f516
Branch: refs/heads/current-release-line
Commit: 44a6f5162858e243968280f2fda27ae614f36e4e
Parents: 37d2fc1
Author: Sharad Agarwal <sh...@apache.org>
Authored: Wed Jun 17 16:50:54 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jun 17 16:50:54 2015 +0530
----------------------------------------------------------------------
lens-docker/lens-test/lens-bootstrap.sh | 6 +++++-
lens-ml-dist/pom.xml | 20 ++++++++++++++++++++
lens-ml-lib/pom.xml | 4 ++++
.../lens/ml/algo/spark/SparkMLDriver.java | 7 +++++++
pom.xml | 5 +++++
tools/conf-pseudo-distr/server/lens-site.xml | 2 +-
tools/scripts/lens-ctl | 4 +++-
7 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-docker/lens-test/lens-bootstrap.sh
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index 11cf1b9..3407dca 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -28,7 +28,11 @@ echo "SPARK_HOME " $SPARK_HOME
LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh`
export LENS_EXT_CLASSPATH
-HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.2.0-beta-incubating-SNAPSHOT.jar,$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar
+SPARK_YARN_JAR=$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar
+export SPARK_YARN_JAR
+echo "SPARK_YARN_JAR " $SPARK_YARN_JAR
+
+HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.2.0-beta-incubating-SNAPSHOT.jar,$SPARK_YARN_JAR
export HIVE_AUX_JARS_PATH
echo "HIVE_AUX_JARS_PATH " $HIVE_AUX_JARS_PATH
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
index 54c9e51..26be98d 100644
--- a/lens-ml-dist/pom.xml
+++ b/lens-ml-dist/pom.xml
@@ -56,6 +56,26 @@
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index f952f8a..63d72da 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -88,6 +88,10 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.10</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-yarn_2.10</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
index 6543fed..21ed87d 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
@@ -172,7 +172,14 @@ public class SparkMLDriver implements MLDriver {
throw new IllegalArgumentException("Spark home is not set");
}
+ // set spark.yarn.jar
+ String yarnJars = System.getenv("SPARK_YARN_JAR");
+ if (StringUtils.isNotBlank(yarnJars)) {
+ sparkConf.set("spark.yarn.jar", yarnJars);
+ }
+
LOG.info("Spark home is set to " + sparkConf.get("spark.home"));
+ LOG.info("spark.yarn.jar is set to " + yarnJars);
}
sparkConf.setAppName("lens-ml");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 067f9b4..df0b766 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1165,6 +1165,11 @@
<artifactId>spark-mllib_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-yarn_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/tools/conf-pseudo-distr/server/lens-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/lens-site.xml b/tools/conf-pseudo-distr/server/lens-site.xml
index 6799987..f43d07e 100644
--- a/tools/conf-pseudo-distr/server/lens-site.xml
+++ b/tools/conf-pseudo-distr/server/lens-site.xml
@@ -76,7 +76,7 @@
<property>
<name>lens.ml.sparkdriver.spark.master</name>
- <value>local</value>
+ <value>yarn-client</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/tools/scripts/lens-ctl
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-ctl b/tools/scripts/lens-ctl
index ec6b5fe..1335469 100755
--- a/tools/scripts/lens-ctl
+++ b/tools/scripts/lens-ctl
@@ -116,7 +116,9 @@ start() {
MAPRED_JARS=$MAPRED_JARS:`ls ${HADOOP_HOME}/../hadoop-mapreduce/aws-java-sdk-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
MAPRED_JARS=$MAPRED_JARS:`ls ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
- HADOOP_JARPATH=$CORE_JARS:$LIB_JARS:$COMMON_JARS:$HDFS_JARS:$MAPRED_JARS
+ YARN_JARS=`ls ${HADOOP_HOME}/share/hadoop/yarn/hadoop-yarn*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+
+ HADOOP_JARPATH=$CORE_JARS:$LIB_JARS:$COMMON_JARS:$HDFS_JARS:$MAPRED_JARS:$YARN_JARS
LENSCPPATH=${LENSCPPATH}:$HADOOP_JARPATH
fi
[12/51] [abbrv] incubator-lens git commit: LENS-341 : Suppress
javadoc errors for java8 build (Raju Bairishetti via amareshwari)
Posted by jd...@apache.org.
LENS-341 : Suppress javadoc errors for java8 build (Raju Bairishetti via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/70e4b922
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/70e4b922
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/70e4b922
Branch: refs/heads/current-release-line
Commit: 70e4b92231c39e87eff358d42c8493dded987a7a
Parents: fa83c48
Author: Raju Bairishetti <ra...@gmail.com>
Authored: Wed Jun 3 15:51:42 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jun 3 15:51:42 2015 +0530
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/70e4b922/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a9a3969..fb9df7c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -324,6 +324,9 @@
<artifactId>maven-javadoc-plugin</artifactId>
<version>${javadoc.plugin.version}</version>
<inherited>false</inherited>
+ <configuration>
+ <additionalparam>-Xdoclint:none</additionalparam>
+ </configuration>
<reportSets>
<reportSet>
<id>aggregate</id>
[38/51] [abbrv] incubator-lens git commit: LENS-604 : Add "time
dimension not supported" as a fact prune cause
Posted by jd...@apache.org.
LENS-604 : Add "time dimension not supported" as a fact prune cause
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/51cdd988
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/51cdd988
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/51cdd988
Branch: refs/heads/current-release-line
Commit: 51cdd98831f2f392e24f16718b3cd4409d2250ed
Parents: 2542874
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jun 19 10:42:35 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jun 19 10:42:35 2015 +0530
----------------------------------------------------------------------
.../cube/parse/CandidateTablePruneCause.java | 41 +++++++++----
.../lens/cube/parse/StorageTableResolver.java | 61 +++++++++++++-------
.../lens/cube/parse/TestCubeRewriter.java | 8 +--
.../server/query/QueryAPIErrorResponseTest.java | 2 +-
4 files changed, 74 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 5a1f8f9..75d5581 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
+
import java.util.*;
import org.codehaus.jackson.annotate.JsonWriteNullProperties;
@@ -98,7 +100,19 @@ public class CandidateTablePruneCause {
// no candidate storges for cube table, storage cause will have why each
// storage is not a candidate
NO_CANDIDATE_STORAGES("No candidate storages for any table"),
- // cube table has more weight
+ // time dimension not supported. Either directly or indirectly.
+ TIMEDIM_NOT_SUPPORTED("Queried data not available for time dimensions: %s") {
+ @Override
+ Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) {
+ Set<String> dims = Sets.newHashSet();
+ for(CandidateTablePruneCause cause: causes){
+ dims.addAll(cause.getUnsupportedTimeDims());
+ }
+ return new Object[]{
+ dims.toString(),
+ };
+ }
+ },
NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE("No fact update periods for given range"),
NO_COLUMN_PART_OF_A_JOIN_PATH("No column part of a join path. Join columns: [%s]") {
Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) {
@@ -222,7 +236,9 @@ public class CandidateTablePruneCause {
private List<String> joinColumns;
// the columns that are missing default aggregate. only set in case of MISSING_DEFAULT_AGGREGATE
private List<String> columnsMissingDefaultAggregate;
-
+ // if a time dim is not supported by the fact. Would be set if and only if
+ // the fact is not partitioned by part col of the time dim and time dim is not a dim attribute
+ private Set<String> unsupportedTimeDims;
// time covered
private MaxCoveringFactResolver.TimeCovered maxTimeCovered;
// ranges in which fact is invalid
@@ -234,17 +250,22 @@ public class CandidateTablePruneCause {
// Different static constructors for different causes.
public static CandidateTablePruneCause factNotAvailableInRange(List<TimeRange> ranges) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.FACT_NOT_AVAILABLE_IN_RANGE);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(FACT_NOT_AVAILABLE_IN_RANGE);
cause.invalidRanges = ranges;
return cause;
}
+ public static CandidateTablePruneCause timeDimNotSupported(Set<String> unsupportedTimeDims) {
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(TIMEDIM_NOT_SUPPORTED);
+ cause.unsupportedTimeDims = unsupportedTimeDims;
+ return cause;
+ }
public static CandidateTablePruneCause columnNotFound(Collection<String>... missingColumns) {
List<String> colList = new ArrayList<String>();
for (Collection<String> missing : missingColumns) {
colList.addAll(missing);
}
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.COLUMN_NOT_FOUND);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(COLUMN_NOT_FOUND);
cause.setMissingColumns(colList);
return cause;
}
@@ -262,27 +283,27 @@ public class CandidateTablePruneCause {
for (String column : exprs) {
colList.add(column);
}
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(EXPRESSION_NOT_EVALUABLE);
cause.setMissingExpressions(colList);
return cause;
}
public static CandidateTablePruneCause missingPartitions(Set<String> nonExistingParts) {
CandidateTablePruneCause cause =
- new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_PARTITIONS);
+ new CandidateTablePruneCause(MISSING_PARTITIONS);
cause.setMissingPartitions(nonExistingParts);
return cause;
}
public static CandidateTablePruneCause lessData(MaxCoveringFactResolver.TimeCovered timeCovered) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.LESS_DATA);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(LESS_DATA);
cause.setMaxTimeCovered(timeCovered);
return cause;
}
public static CandidateTablePruneCause noColumnPartOfAJoinPath(final Collection<String> colSet) {
CandidateTablePruneCause cause =
- new CandidateTablePruneCause(CandidateTablePruneCode.NO_COLUMN_PART_OF_A_JOIN_PATH);
+ new CandidateTablePruneCause(NO_COLUMN_PART_OF_A_JOIN_PATH);
cause.setJoinColumns(new ArrayList<String>() {
{
addAll(colSet);
@@ -292,7 +313,7 @@ public class CandidateTablePruneCause {
}
public static CandidateTablePruneCause noCandidateStorages(Map<String, SkipStorageCause> storageCauses) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.NO_CANDIDATE_STORAGES);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(NO_CANDIDATE_STORAGES);
cause.setStorageCauses(new HashMap<String, SkipStorageCause>());
for (Map.Entry<String, SkipStorageCause> entry : storageCauses.entrySet()) {
String key = entry.getKey();
@@ -303,7 +324,7 @@ public class CandidateTablePruneCause {
}
public static CandidateTablePruneCause missingDefaultAggregate(String... names) {
- CandidateTablePruneCause cause = new CandidateTablePruneCause(CandidateTablePruneCode.MISSING_DEFAULT_AGGREGATE);
+ CandidateTablePruneCause cause = new CandidateTablePruneCause(MISSING_DEFAULT_AGGREGATE);
cause.setColumnsMissingDefaultAggregate(Lists.newArrayList(names));
return cause;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 049826a..ce6f434 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST;
import static org.apache.lens.cube.parse.DateUtil.WSPACE;
import static org.apache.lens.cube.parse.StorageUtil.joinWithAnd;
@@ -44,6 +47,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or
@@ -58,16 +62,15 @@ class StorageTableResolver implements ContextRewriter {
CubeMetastoreClient client;
private final boolean failOnPartialData;
private final List<String> validDimTables;
- private final Map<CubeFactTable, Map<UpdatePeriod, Set<String>>> validStorageMap =
- new HashMap<CubeFactTable, Map<UpdatePeriod, Set<String>>>();
+ private final Map<CubeFactTable, Map<UpdatePeriod, Set<String>>> validStorageMap = new HashMap<>();
private String processTimePartCol = null;
private final UpdatePeriod maxInterval;
- private final Map<String, Set<String>> nonExistingPartitions = new HashMap<String, Set<String>>();
+ private final Map<String, Set<String>> nonExistingPartitions = new HashMap<>();
private TimeRangeWriter rangeWriter;
private DateFormat partWhereClauseFormat = null;
private PHASE phase;
- static enum PHASE {
+ enum PHASE {
FACT_TABLES, FACT_PARTITIONS, DIM_TABLE_AND_PARTITIONS;
static PHASE first() {
@@ -218,7 +221,7 @@ class StorageTableResolver implements ContextRewriter {
}
if (storageTables.isEmpty()) {
LOG.info("Not considering dim table:" + dimtable + " as no candidate storage tables eixst");
- cubeql.addDimPruningMsgs(dim, dimtable, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses));
+ cubeql.addDimPruningMsgs(dim, dimtable, noCandidateStorages(skipStorageCauses));
i.remove();
continue;
}
@@ -291,7 +294,7 @@ class StorageTableResolver implements ContextRewriter {
}
if (storageTableMap.isEmpty()) {
LOG.info("Not considering fact table:" + fact + " as it does not" + " have any storage tables");
- cubeql.addFactPruningMsgs(fact, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses));
+ cubeql.addFactPruningMsgs(fact, noCandidateStorages(skipStorageCauses));
i.remove();
}
}
@@ -352,24 +355,33 @@ class StorageTableResolver implements ContextRewriter {
private void resolveFactStoragePartitions(CubeQueryContext cubeql) throws SemanticException {
// Find candidate tables wrt supported storages
Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator();
- Map<TimeRange, String> whereClasueForFallback = new LinkedHashMap<TimeRange, String>();
+ Map<TimeRange, String> whereClauseForFallback = new LinkedHashMap<TimeRange, String>();
while (i.hasNext()) {
CandidateFact cfact = i.next();
- List<FactPartition> answeringParts = new ArrayList<FactPartition>();
- HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<String, SkipStorageCause>();
+ List<FactPartition> answeringParts = new ArrayList<>();
+ HashMap<String, SkipStorageCause> skipStorageCauses = new HashMap<>();
PartitionRangesForPartitionColumns missingParts = new PartitionRangesForPartitionColumns();
boolean noPartsForRange = false;
+ Set<String> unsupportedTimeDims = Sets.newHashSet();
for (TimeRange range : cubeql.getTimeRanges()) {
StringBuilder extraWhereClause = new StringBuilder();
Set<FactPartition> rangeParts = getPartitions(cfact.fact, range, skipStorageCauses, missingParts);
// If no partitions were found, then we'll fallback.
- String partcol = range.getPartitionColumn();
+ String partCol = range.getPartitionColumn();
+ boolean partColNotSupported = rangeParts.isEmpty();
+ for(String storage: cfact.fact.getStorages()) {
+ String storageTableName = MetastoreUtil.getFactStorageTableName(cfact.fact.getName(), storage).toLowerCase();
+ partColNotSupported &= skipStorageCauses.containsKey(storageTableName)
+ && skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST)
+ && skipStorageCauses.get(storageTableName).getNonExistantPartCols().contains(partCol);
+ }
TimeRange prevRange = range;
String sep = "";
while (rangeParts.isEmpty()) {
// TODO: should we add a condition whether on range's partcol any missing partitions are not there
- String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partcol);
- if (!cfact.getColumns().contains(timeDim)) {
+ String timeDim = cubeql.getBaseCube().getTimeDimOfPartitionColumn(partCol);
+ if (partColNotSupported && !cfact.getColumns().contains(timeDim)) {
+ unsupportedTimeDims.add(cubeql.getBaseCube().getTimeDimOfPartitionColumn(range.getPartitionColumn()));
break;
}
TimeRange fallBackRange = getFallbackRange(prevRange, cfact, cubeql);
@@ -381,13 +393,13 @@ class StorageTableResolver implements ContextRewriter {
extraWhereClause.append(sep)
.append(prevRange.toTimeDimWhereClause(cubeql.getAliasForTableName(cubeql.getCube()), timeDim));
sep = " AND ";
- partcol = fallBackRange.getPartitionColumn();
prevRange = fallBackRange;
+ partCol = prevRange.getPartitionColumn();
if (!rangeParts.isEmpty()) {
break;
}
}
- whereClasueForFallback.put(range, extraWhereClause.toString());
+ whereClauseForFallback.put(range, extraWhereClause.toString());
if (rangeParts.isEmpty()) {
LOG.info("No partitions for fallback range:" + range);
noPartsForRange = true;
@@ -399,7 +411,7 @@ class StorageTableResolver implements ContextRewriter {
for (FactPartition factPart : rangeParts) {
for (String table : factPart.getStorageTables()) {
if (!tablePartMap.containsKey(table)) {
- tablePartMap.put(table, new LinkedHashSet<FactPartition>(Arrays.asList(factPart)));
+ tablePartMap.put(table, new LinkedHashSet<>(Collections.singletonList(factPart)));
} else {
LinkedHashSet<FactPartition> storagePart = tablePartMap.get(table);
storagePart.add(factPart);
@@ -414,6 +426,13 @@ class StorageTableResolver implements ContextRewriter {
cubeql.getAliasForTableName(cubeql.getCube().getName()), rangeParts);
cfact.getRangeToWhereClause().put(range, joinWithAnd(rangeWhereClause, extraWhereClause.toString()));
}
+ if (!unsupportedTimeDims.isEmpty()) {
+ LOG.info("Not considering fact table:" + cfact.fact + " as it doesn't support time dimensions: "
+ + unsupportedTimeDims);
+ cubeql.addFactPruningMsgs(cfact.fact, timeDimNotSupported(unsupportedTimeDims));
+ i.remove();
+ continue;
+ }
Set<String> nonExistingParts = missingParts.toSet();
if (!nonExistingParts.isEmpty()) {
addNonExistingParts(cfact.fact.getName(), nonExistingParts);
@@ -428,13 +447,13 @@ class StorageTableResolver implements ContextRewriter {
* 3. Storage tables do not have the update period for the timerange queried.
*/
if (failOnPartialData && !nonExistingParts.isEmpty()) {
- cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.missingPartitions(nonExistingParts));
+ cubeql.addFactPruningMsgs(cfact.fact, missingPartitions(nonExistingParts));
} else if (!skipStorageCauses.isEmpty()) {
- CandidateTablePruneCause cause = CandidateTablePruneCause.noCandidateStorages(skipStorageCauses);
+ CandidateTablePruneCause cause = noCandidateStorages(skipStorageCauses);
cubeql.addFactPruningMsgs(cfact.fact, cause);
} else {
CandidateTablePruneCause cause =
- new CandidateTablePruneCause(CandidateTablePruneCode.NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE);
+ new CandidateTablePruneCause(NO_FACT_UPDATE_PERIODS_FOR_GIVEN_RANGE);
cubeql.addFactPruningMsgs(cfact.fact, cause);
}
i.remove();
@@ -445,7 +464,7 @@ class StorageTableResolver implements ContextRewriter {
StorageUtil.getMinimalAnsweringTables(answeringParts, minimalStorageTables);
if (minimalStorageTables.isEmpty()) {
LOG.info("Not considering fact table:" + cfact + " as it does not" + " have any storage tables");
- cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.noCandidateStorages(skipStorageCauses));
+ cubeql.addFactPruningMsgs(cfact.fact, noCandidateStorages(skipStorageCauses));
i.remove();
continue;
}
@@ -463,10 +482,10 @@ class StorageTableResolver implements ContextRewriter {
LinkedHashSet<FactPartition> rangeParts = cfact.getRangeToStoragePartMap().get(trange).get(table);
LinkedHashSet<FactPartition> minimalPartsCopy = new LinkedHashSet<FactPartition>(minimalParts);
minimalPartsCopy.retainAll(rangeParts);
- if (!StringUtils.isEmpty(whereClasueForFallback.get(trange))) {
+ if (!StringUtils.isEmpty(whereClauseForFallback.get(trange))) {
rangeToWhere.put(
rangeWriter.getTimeRangeWhereClause(cubeql, cubeql.getAliasForTableName(cubeql.getCube().getName()),
- minimalPartsCopy) + " and " + whereClasueForFallback.get(trange), table);
+ minimalPartsCopy) + " and " + whereClauseForFallback.get(trange), table);
} else {
rangeToWhere.put(rangeWriter.getTimeRangeWhereClause(cubeql,
cubeql.getAliasForTableName(cubeql.getCube().getName()), minimalPartsCopy), table);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index 0f1ad6e..e04d04c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -947,12 +947,8 @@ public class TestCubeRewriter extends TestQueryRewrite {
MISSING_PARTITIONS);
assertEquals(pruneCauses.getDetails().get("cheapfact").iterator().next().getCause(),
NO_CANDIDATE_STORAGES);
- assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
- .getCause(), NO_CANDIDATE_STORAGES);
- assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
- .getStorageCauses().values().iterator().next().getCause(), SkipStorageCode.PART_COL_DOES_NOT_EXIST);
- assertEquals(pruneCauses.getDetails().get("summary4").iterator().next()
- .getStorageCauses().values().iterator().next().getNonExistantPartCols(), Arrays.asList("dt"));
+ assertEquals(pruneCauses.getDetails().get("summary4").iterator().next().getCause(), TIMEDIM_NOT_SUPPORTED);
+ assertTrue(pruneCauses.getDetails().get("summary4").iterator().next().getUnsupportedTimeDims().contains("d_time"));
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/51cdd988/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 4c2a7a4..720229d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -68,7 +68,7 @@ import org.testng.annotations.Test;
import com.google.common.base.Optional;
import lombok.NonNull;
-@Test
+@Test(groups = "unit-test")
public class QueryAPIErrorResponseTest extends LensJerseyTest {
private static final String MOCK_QUERY = "mock-query";