You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by sh...@apache.org on 2015/07/29 06:56:48 UTC
[01/50] [abbrv] incubator-lens git commit: LENS-626 : Update release
documentation for 2.2-beta release
Repository: incubator-lens
Updated Branches:
refs/heads/LENS-581 64934716e -> 6c8057050
LENS-626 : Update release documentation for 2.2-beta release
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/f85e2c20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/f85e2c20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/f85e2c20
Branch: refs/heads/LENS-581
Commit: f85e2c20be410224c967da13a7e5c7ee259223c0
Parents: 50c3617
Author: Jaideep Dhok <jd...@apache.org>
Authored: Mon Jul 6 16:21:54 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Jul 6 16:40:37 2015 +0530
----------------------------------------------------------------------
src/site/apt/releases/download.apt | 2 +-
src/site/apt/releases/release-history.apt | 23 +++++++++++++++++++++++
2 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f85e2c20/src/site/apt/releases/download.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/releases/download.apt b/src/site/apt/releases/download.apt
index 2c99186..e9d4821 100644
--- a/src/site/apt/releases/download.apt
+++ b/src/site/apt/releases/download.apt
@@ -20,7 +20,7 @@
Download
The latest release of Apache Lens can be
- {{{http://www.apache.org/dyn/closer.cgi/incubator/lens/2.1-beta-incubating}downloaded from the ASF}}.
+ {{{http://www.apache.org/dyn/closer.cgi/incubator/lens/2.2-beta-incubating}downloaded from the ASF}}.
Older releases may be found {{{http://archive.apache.org/dist/incubator/lens} in the archives}}.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f85e2c20/src/site/apt/releases/release-history.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/releases/release-history.apt b/src/site/apt/releases/release-history.apt
index 82625c7..58ac279 100644
--- a/src/site/apt/releases/release-history.apt
+++ b/src/site/apt/releases/release-history.apt
@@ -23,6 +23,8 @@ All Apache Lens releases
*--+--+---+--+--+--+--+
|Release version|Major features|Release documentation|Release Notes|Download|Incompatibilities|More Info |
*--+--+---+--+--+--+--+
+|2.2.0-beta-incubating| CLI Improvements, Support for multiple expressions and and union queries, More descriptive error codes, Zeppelin integration, Code cleanup| {{{../versions/2.2.0-beta-incubating/index.html} 2.2.x-incubating docs}} | {{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?version=12329586&projectId=12315923} 2.2 release notes}}| {{{http://archive.apache.org/dist/incubator/lens/2.2-beta-incubating/} Download}} | {{{#a2.2.x_from_2.1.x} 2.2.x-incompatibilities}} | - |
+*--+--+---+--+--+--+--+
|2.1.0-beta-incubating| Estimate api, Partition timeline, DB resources, Timers on REST api| {{{../versions/2.1.0-beta-incubating/index.html} 2.1.x-incubating docs}} | {{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315923&version=12328991} 2.1 release notes}}| {{{http://archive.apache.org/dist/incubator/lens/2.1-beta-incubating/} Download}} | {{{#a2.1.x_from_2.0.x} 2.1.x-incompatibilities}} | - |
*--+--+---+--+--+--+--+
|2.0.1-beta-incubating| First release in Apache| {{{../versions/2.0.1-beta-incubating/index.html} 2.0.x-incubating docs}} | {{{https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12315923&version=12328990} 2.0 release notes}}| {{{http://archive.apache.org/dist/incubator/lens/2.0-beta-incubating/} Download}} | - | - |
@@ -36,6 +38,27 @@ All Apache Lens releases
* Incompatibilities
+** 2.2.x from 2.1.x
+
+ * LENS-484 : Fact table weight is a mandatory field now, for creating facts from the web interface. Please refer to
+ cube xsd for further details.
+
+ * LENS-553 : Java compilation version moved to Java 7 from Java 6.
+
+ * LENS-546 : Parameter names in Lens CLI will have _ (underscore) instead of - (hyphen) as a separator
+
+ * LENS-256 : Use explicit keywords for different parameter names in Lens-CLI
+
+ * LENS-486 : More structured and informative error responses for the REST interface
+
+ * LENS-487 : Consistent error response display in Lens-CLI. The output shown is different than the previous version.
+
+ * LENS-550 : Removed support for multi table select in cube queries
+
+ * LENS-563 : There was a length limitation of the property value of string that specified 'valid' column list for
+ a fact table. This change has fixed the length limitation by allowing users to set a size parameter, and multiple
+ numerically indexed keys for specifying the bigger column list.
+
** 2.1.x from 2.0.x
* LENS-437 : Removed counts and weights from QueryPlan. The values are no longer available. If any usecase requires
[16/50] [abbrv] incubator-lens git commit: LENS-672 : Add incubator
disclaimer in downloads page
Posted by sh...@apache.org.
LENS-672 : Add incubator disclaimer in downloads page
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/bbb93516
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/bbb93516
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/bbb93516
Branch: refs/heads/LENS-581
Commit: bbb93516c7f280e33a7e0890cc67b76eed8900ae
Parents: 588baef
Author: Yash Sharma <ya...@gmail.com>
Authored: Mon Jul 20 15:40:01 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Jul 20 15:40:01 2015 +0530
----------------------------------------------------------------------
src/site/apt/releases/download.apt | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/bbb93516/src/site/apt/releases/download.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/releases/download.apt b/src/site/apt/releases/download.apt
index e9d4821..5322f99 100644
--- a/src/site/apt/releases/download.apt
+++ b/src/site/apt/releases/download.apt
@@ -24,3 +24,7 @@ Download
Older releases may be found {{{http://archive.apache.org/dist/incubator/lens} in the archives}}.
++---
+Disclaimer
+Apache Lens is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
++---
[06/50] [abbrv] incubator-lens git commit: LENS-630: Using Duration
and Fact Weight Based Query Cost Calculator for Hive Driver
Posted by sh...@apache.org.
LENS-630: Using Duration and Fact Weight Based Query Cost Calculator for Hive Driver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/2e5748a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/2e5748a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/2e5748a8
Branch: refs/heads/LENS-581
Commit: 2e5748a8c5dcff9627b8d125820371f1d4667d61
Parents: 8ebe7f4
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jul 9 13:05:53 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 9 13:05:53 2015 +0530
----------------------------------------------------------------------
.../main/java/org/apache/lens/api/Addable.java | 32 ++++
.../org/apache/lens/api/query/QueryCost.java | 80 ----------
.../apache/lens/api/query/QueryCostType.java | 26 ++++
.../org/apache/lens/api/query/QueryPlan.java | 4 +-
.../lens/api/query/QuerySubmitResult.java | 4 +-
.../org/apache/lens/api/result/QueryCostTO.java | 75 ++++++++++
.../lens/cube/metadata/FactPartition.java | 16 +-
.../apache/lens/cube/metadata/UpdatePeriod.java | 36 +++--
.../FactPartitionBasedQueryCostCalculator.java | 98 ++++++++++++
.../apache/lens/driver/cube/RewriterPlan.java | 2 +-
.../lens/cube/metadata/TestFactPartition.java | 57 +++++++
...stFactPartitionBasedQueryCostCalculator.java | 96 ++++++++++++
.../org/apache/lens/driver/hive/HiveDriver.java | 75 ++++++----
.../apache/lens/driver/hive/HiveQueryPlan.java | 25 +---
.../DurationBasedQueryPriorityDecider.java | 139 -----------------
.../apache/lens/driver/hive/TestHiveDriver.java | 149 +++++++++----------
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 8 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 31 ++--
.../regression/core/helpers/QueryHelper.java | 13 +-
.../lens/server/api/driver/DriverQueryPlan.java | 12 +-
.../lens/server/api/driver/LensDriver.java | 8 +-
.../server/api/driver/MinQueryCostSelector.java | 2 +-
.../api/priority/CostToPriorityRangeConf.java | 74 ---------
.../api/priority/QueryPriorityDecider.java | 39 -----
.../lens/server/api/priority/RangeConf.java | 111 --------------
.../server/api/query/AbstractQueryContext.java | 35 +++--
.../api/query/DriverSelectorQueryContext.java | 10 +-
.../server/api/query/QueryExecutionService.java | 1 +
.../query/cost/FactPartitionBasedQueryCost.java | 64 ++++++++
.../lens/server/api/query/cost/QueryCost.java | 38 +++++
.../api/query/cost/QueryCostCalculator.java | 32 ++++
.../api/query/cost/QueryCostTOBuilder.java | 50 +++++++
.../priority/CostRangePriorityDecider.java | 42 ++++++
.../query/priority/CostToPriorityRangeConf.java | 74 +++++++++
.../query/priority/QueryPriorityDecider.java | 33 ++++
.../server/api/query/priority/RangeConf.java | 111 ++++++++++++++
.../lens/server/api/driver/MockDriver.java | 9 +-
.../lens/server/api/driver/MockFailDriver.java | 2 +-
.../cost/TestFactPartitionBasedQueryCost.java | 77 ++++++++++
.../server/query/QueryExecutionServiceImpl.java | 63 ++++----
.../lens/server/query/QueryServiceResource.java | 5 +-
.../lens/server/common/RestAPITestUtil.java | 6 +-
.../lens/server/query/TestQueryService.java | 18 +--
43 files changed, 1198 insertions(+), 684 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/Addable.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/Addable.java b/lens-api/src/main/java/org/apache/lens/api/Addable.java
new file mode 100644
index 0000000..5a55691
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/Addable.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.api;
+
+public interface Addable<T extends Addable<T>> {
+ /**
+ * Add another object of same type object of the same type.
+ * Implement this if add operation makes sense for the class
+ * @param other
+ * @return new T object after logical addition
+ */
+ T add(T other);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java
deleted file mode 100644
index 1a37c20..0000000
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryCost.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- *
- */
-package org.apache.lens.api.query;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import lombok.*;
-
-/**
- * The Class QueryCost.
- */
-@XmlRootElement
-/**
- * Instantiates a new query cost.
- *
- * @param estimatedExecTimeMillis
- * the estimated exec time millis
- * @param estimatedResourceUsage
- * the estimated resource usage
- */
-@AllArgsConstructor
-/**
- * Instantiates a new query cost.
- */
-@NoArgsConstructor(access = AccessLevel.PROTECTED)
-@ToString
-public class QueryCost extends QuerySubmitResult implements Comparable<QueryCost> {
-
- /**
- * The estimated exec time millis.
- */
- @XmlElement
- @Getter
- private long estimatedExecTimeMillis;
-
- /**
- * The estimated resource usage.
- */
- @XmlElement
- @Getter
- private double estimatedResourceUsage;
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Comparable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(QueryCost other) {
- if (estimatedExecTimeMillis == other.estimatedExecTimeMillis) {
- if (estimatedResourceUsage == other.estimatedResourceUsage) {
- return 0;
- } else {
- return (int) (estimatedResourceUsage - other.estimatedResourceUsage);
- }
- } else {
- return (int) (estimatedExecTimeMillis - other.estimatedExecTimeMillis);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
new file mode 100644
index 0000000..1485e1d
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryCostType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.api.query;
+
+public enum QueryCostType {
+ LOW, MEDIUM, HIGH
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java
index a836b1e..164195b 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryPlan.java
@@ -29,6 +29,8 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.lens.api.result.QueryCostTO;
+
import lombok.*;
/**
@@ -110,7 +112,7 @@ public class QueryPlan extends QuerySubmitResult {
*/
@XmlElement
@Getter
- private QueryCost queryCost;
+ private QueryCostTO queryCost;
/**
* The error.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java b/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java
index 2177c78..f3fb6db 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QuerySubmitResult.java
@@ -24,12 +24,14 @@ package org.apache.lens.api.query;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlSeeAlso;
+import org.apache.lens.api.result.QueryCostTO;
+
/**
* The Class QuerySubmitResult.
*/
@XmlRootElement
@XmlSeeAlso({QueryHandle.class, QueryPrepareHandle.class, QueryHandleWithResultSet.class,
- org.apache.lens.api.query.QueryPlan.class, QueryCost.class})
+ QueryPlan.class, QueryCostTO.class})
public abstract class QuerySubmitResult {
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java b/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java
new file mode 100644
index 0000000..a247e3c
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/result/QueryCostTO.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.api.result;
+
+import java.io.Serializable;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.lens.api.query.QueryCostType;
+import org.apache.lens.api.query.QuerySubmitResult;
+
+import lombok.*;
+
+/**
+ * The Class QueryCostTO.
+ */
+@XmlRootElement
+/**
+ * Instantiates a new query cost.
+ *
+ * @param estimatedExecTimeMillis
+ * the estimated exec time millis
+ * @param estimatedResourceUsage
+ * the estimated resource usage
+ */
+@AllArgsConstructor
+/**
+ * Instantiates a new query cost.
+ */
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@ToString
+public class QueryCostTO extends QuerySubmitResult implements Serializable {
+
+ /**
+ * The estimated exec time millis.
+ */
+ @XmlElement
+ @Getter
+ private Long estimatedExecTimeMillis;
+
+ /**
+ * The estimated resource usage.
+ */
+ @XmlElement
+ @Getter
+ private Double estimatedResourceUsage;
+
+ /**
+ * Normalized query cost, calculated by driver
+ */
+ @XmlElement
+ @Getter
+ private QueryCostType type;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
index 3465513..fc2d85b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactPartition.java
@@ -20,12 +20,11 @@
package org.apache.lens.cube.metadata;
import java.text.DateFormat;
-import java.util.Date;
-import java.util.LinkedHashSet;
-import java.util.Set;
+import java.util.*;
import org.apache.lens.server.api.error.LensException;
+import com.google.common.collect.ImmutableMap;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
@@ -166,4 +165,15 @@ public class FactPartition implements Comparable<FactPartition> {
public TimePartition getTimePartition() throws LensException {
return TimePartition.of(getPeriod(), getPartSpec());
}
+
+ public double getAllTableWeights(ImmutableMap<String, Double> tableWeights) {
+ float weight = 0;
+ for (String tblName : getStorageTables()) {
+ Double tblWeight = tableWeights.get(tblName);
+ if (tblWeight != null) {
+ weight += tblWeight;
+ }
+ }
+ return weight;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
index 3c9268b..966dbdf 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
@@ -19,24 +19,35 @@
package org.apache.lens.cube.metadata;
+import static java.util.Calendar.*;
+
import java.text.DateFormat;
import java.text.SimpleDateFormat;
-import java.util.Calendar;
import java.util.Comparator;
import java.util.Date;
import org.apache.lens.cube.parse.DateUtil;
public enum UpdatePeriod implements Named {
- SECONDLY(Calendar.SECOND, 1000, "yyyy-MM-dd-HH-mm-ss"), MINUTELY(Calendar.MINUTE, 60 * SECONDLY.weight(),
- "yyyy-MM-dd-HH-mm"), HOURLY(Calendar.HOUR_OF_DAY, 60 * MINUTELY.weight(), "yyyy-MM-dd-HH"), DAILY(
- Calendar.DAY_OF_MONTH, 24 * HOURLY.weight(), "yyyy-MM-dd"), WEEKLY(Calendar.WEEK_OF_YEAR, 7 * DAILY.weight(),
- "yyyy-'W'ww"), MONTHLY(Calendar.MONTH, 30 * DAILY.weight(), "yyyy-MM"), QUARTERLY(Calendar.MONTH, 3 * MONTHLY
- .weight(), "yyyy-MM"), YEARLY(Calendar.YEAR, 12 * MONTHLY.weight(), "yyyy");
-
- public static final long MIN_INTERVAL = SECONDLY.weight();
+ SECONDLY(SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"),
+ MINUTELY(MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"),
+ HOURLY(HOUR_OF_DAY, 60 * MINUTELY.weight(), 1.3f, "yyyy-MM-dd-HH"),
+ DAILY(DAY_OF_MONTH, 24 * HOURLY.weight(), 1f, "yyyy-MM-dd"),
+ WEEKLY(WEEK_OF_YEAR, 7 * DAILY.weight(), 0.7f, "yyyy-'W'ww"),
+ MONTHLY(MONTH, 30 * DAILY.weight(), 0.6f, "yyyy-MM"),
+ QUARTERLY(MONTH, 3 * MONTHLY.weight(), 0.55f, "yyyy-MM"),
+ YEARLY(YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy");
+
+ public static final long MIN_INTERVAL = values()[0].weight();
private final int calendarField;
private final long weight;
+ /**
+ * Normalization factor is calculated in comparison with daily update period. What it means is that
+ * for a fixed time range, reading partitions of this update period is expensive/cheap as compared to
+ * reading partitions of daily update period by this factor. Values are tentatively picked based on
+ * similar logic in an existing system at InMobi.
+ */
+ private final float normalizationFactor;
private final String format;
private static DateFormat getSecondlyFormat() {
@@ -144,9 +155,10 @@ public enum UpdatePeriod implements Named {
private static ThreadLocal<DateFormat> quarterlyFormat;
private static ThreadLocal<DateFormat> yearlyFormat;
- UpdatePeriod(int calendarField, long diff, String format) {
+ UpdatePeriod(int calendarField, long diff, float normalizationFactor, String format) {
this.calendarField = calendarField;
this.weight = diff;
+ this.normalizationFactor = normalizationFactor;
this.format = format;
}
@@ -181,7 +193,7 @@ public enum UpdatePeriod implements Named {
case YEARLY:
return getYearlyFormat();
default:
- return null;
+ throw new IllegalArgumentException("Update period illegal, or doesn't have defined format");
}
}
@@ -198,6 +210,10 @@ public enum UpdatePeriod implements Named {
return formatStr().replaceAll("'", "").length() == dateString.length();
}
+ public float getNormalizationFactor() {
+ return normalizationFactor;
+ }
+
public static class UpdatePeriodComparator implements Comparator<UpdatePeriod> {
@Override
public int compare(UpdatePeriod o1, UpdatePeriod o2) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
new file mode 100644
index 0000000..d56e1c7
--- /dev/null
+++ b/lens-cube/src/main/java/org/apache/lens/cube/query/cost/FactPartitionBasedQueryCostCalculator.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.query.cost;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lens.cube.metadata.FactPartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.QueryCostCalculator;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+public class FactPartitionBasedQueryCostCalculator implements QueryCostCalculator {
+
+ public static final String UPDATE_PERIOD_WEIGHT_PREFIX = "update.period.weight.";
+
+ /**
+ * Calculates total cost based on weights of selected tables and their selected partitions
+ *
+ * @param queryContext
+ * @param driver
+ * @return Query Cost
+ * @throws LensException
+ */
+
+ @SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting
+ private double getTotalPartitionCost(final AbstractQueryContext queryContext, LensDriver driver)
+ throws LensException {
+ double cost = 0;
+ for (Map.Entry<String, Set<?>> entry : getAllPartitions(queryContext, driver).entrySet()) {
+ // Have to do instanceof check, since it can't be handled by polymorphism.
+ // The '?' is either a FactPartition or a String. When we decide to write a
+ // DimtablePartition, we can probably think about using polymorphism.
+ if (!entry.getValue().isEmpty() && entry.getValue().iterator().next() instanceof FactPartition) {
+ Set<FactPartition> factParts = (Set<FactPartition>) entry.getValue();
+ for (FactPartition partition : factParts) {
+ double allTableWeights =
+ partition.getAllTableWeights(ImmutableMap.copyOf(queryContext.getTableWeights(driver)));
+ if (allTableWeights == 0) {
+ allTableWeights = 1;
+ }
+ cost += allTableWeights * getNormalizedUpdatePeriodCost(partition.getPeriod(), driver);
+ }
+ }
+ }
+ return cost;
+ }
+
+ /**
+ * Normalized cost of an update period. update period weight multiplied by number of partitions of that update period.
+ *
+ * @param updatePeriod
+ * @param driver
+ * @return normalized cost.
+ * @throws LensException
+ */
+ private double getNormalizedUpdatePeriodCost(final UpdatePeriod updatePeriod, LensDriver driver)
+ throws LensException {
+ double weight = driver.getConf().getDouble(UPDATE_PERIOD_WEIGHT_PREFIX + updatePeriod.name().toLowerCase(),
+ updatePeriod.getNormalizationFactor());
+ return weight * updatePeriod.weight() / UpdatePeriod.DAILY.weight();
+ }
+
+ @Override
+ public QueryCost calculateCost(final AbstractQueryContext queryContext, LensDriver driver) throws LensException {
+ return new FactPartitionBasedQueryCost(getTotalPartitionCost(queryContext, driver));
+ }
+
+ public Map<String, Set<?>> getAllPartitions(AbstractQueryContext queryContext, LensDriver driver) {
+ if (queryContext.getDriverRewriterPlan(driver) != null) {
+ return queryContext.getDriverRewriterPlan(driver).getPartitions();
+ }
+ return Maps.newHashMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
index 841946d..7725043 100644
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
+++ b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriterPlan.java
@@ -22,12 +22,12 @@ import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.cube.metadata.FactPartition;
import org.apache.lens.cube.parse.CandidateTable;
import org.apache.lens.cube.parse.CubeQueryContext;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.DriverQueryPlan;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java
new file mode 100644
index 0000000..3f2ae6f
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestFactPartition.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.cube.metadata;
+
+import static org.apache.lens.cube.metadata.UpdatePeriod.DAILY;
+import static org.apache.lens.cube.metadata.UpdatePeriod.HOURLY;
+
+import static org.testng.Assert.assertEquals;
+
+import java.util.Date;
+import java.util.Map;
+
+import org.testng.annotations.Test;
+
+import com.beust.jcommander.internal.Maps;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+
+public class TestFactPartition {
+ public static final Date DATE = new Date();
+ FactPartition fp1 = new FactPartition("p", DATE, DAILY, null, null, Sets.newHashSet("st1", "st2"));
+ FactPartition fp2 = new FactPartition("q", DATE, HOURLY, fp1, null, Sets.newHashSet("st3", "st4"));
+
+ @Test
+ public void testGetFormattedFilter() throws Exception {
+ String dailyFormat = DAILY.format().format(DATE);
+ String hourlyFormat = HOURLY.format().format(DATE);
+ assertEquals(fp1.getFormattedFilter("table"), "table.p = '" + dailyFormat + "'");
+ assertEquals(fp2.getFormattedFilter("table2"),
+ "table2.p = '" + dailyFormat + "' AND table2.q = '" + hourlyFormat + "'");
+ }
+
+ @Test
+ public void testGetAllTableWeights() throws Exception {
+ Map<String, Double> weights = Maps.newHashMap();
+ assertEquals(fp1.getAllTableWeights(ImmutableMap.copyOf(weights)), 0.0);
+ weights.put("st1", 0.2);
+ weights.put("st2", 0.3);
+ assertEquals(fp1.getAllTableWeights(ImmutableMap.copyOf(weights)), 0.5);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
new file mode 100644
index 0000000..8a60224
--- /dev/null
+++ b/lens-cube/src/test/java/org/apache/lens/cube/query/cost/TestFactPartitionBasedQueryCostCalculator.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.cube.query.cost;
+
+import static org.apache.lens.cube.metadata.UpdatePeriod.*;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Set;
+
+import org.apache.lens.cube.metadata.FactPartition;
+import org.apache.lens.cube.metadata.UpdatePeriod;
+import org.apache.lens.server.api.driver.DriverQueryPlan;
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
+import junit.framework.Assert;
+
+public class TestFactPartitionBasedQueryCostCalculator {
+ AbstractQueryContext queryContext;
+ FactPartitionBasedQueryCostCalculator calculator = new FactPartitionBasedQueryCostCalculator();
+ LensDriver driver;
+
+ @BeforeTest
+ public void setUp() {
+ driver = mock(LensDriver.class);
+ when(driver.getConf()).thenReturn(new Configuration());
+ queryContext = mock(AbstractQueryContext.class);
+
+ ImmutableMap<String, Double> tableWeights = mock(ImmutableMap.class);
+
+ FactPartition fp1 = mockFactPartition(DAILY, tableWeights, 0.7);
+ FactPartition fp2 = mockFactPartition(HOURLY, tableWeights, 0.8);
+ FactPartition fp3 = mockFactPartition(SECONDLY, tableWeights, 0.4);
+ FactPartition fp4 = mockFactPartition(MONTHLY, tableWeights, 0);
+
+ when(queryContext.getTableWeights(driver)).thenReturn(tableWeights);
+
+ HashMap<String, Set<?>> partitions = new HashMap<>();
+ partitions.put("st1", Sets.newHashSet(fp1, fp2));
+ partitions.put("st2", Sets.newHashSet(fp3, fp4));
+ DriverQueryPlan plan = mock(DriverQueryPlan.class);
+ when(queryContext.getDriverRewriterPlan(driver)).thenReturn(plan);
+ when(plan.getPartitions()).thenReturn(partitions);
+ when(calculator.getAllPartitions(queryContext, driver)).thenReturn(partitions);
+ }
+
+ private FactPartition mockFactPartition(UpdatePeriod mockPeriod, ImmutableMap<String, Double> tableWeights,
+ double mockAllTableWeight) {
+ FactPartition fp = mock(FactPartition.class);
+ when(fp.getPeriod()).thenReturn(mockPeriod);
+ when(fp.getAllTableWeights(tableWeights)).thenReturn(mockAllTableWeight);
+ return fp;
+ }
+
+ @Test
+ public void testCalculateCost() throws Exception {
+ QueryCost cost = calculator.calculateCost(queryContext, driver);
+ Assert.assertTrue(cost.getEstimatedResourceUsage() > 18.0);
+ Assert.assertTrue(cost.getEstimatedResourceUsage() < 19.0);
+ }
+
+ @Test
+ public void testGetAllPartitions() throws Exception {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index b7a3be7..e87ed23 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -27,19 +27,23 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
-import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider;
+import org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
-import org.apache.lens.server.api.priority.QueryPriorityDecider;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.QueryCostCalculator;
+import org.apache.lens.server.api.query.priority.CostRangePriorityDecider;
+import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
+import org.apache.lens.server.api.query.priority.QueryPriorityDecider;
import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.commons.lang.StringUtils;
@@ -72,21 +76,16 @@ public class HiveDriver implements LensDriver {
public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.hs2.connection.expiry.delay";
public static final String HS2_CALCULATE_PRIORITY = "lens.driver.hive.calculate.priority";
+ public static final String HS2_COST_CALCULATOR = "lens.driver.hive.cost.calculator.class";
/**
* Config param for defining priority ranges.
*/
public static final String HS2_PRIORITY_RANGES = "lens.driver.hive.priority.ranges";
- public static final String HS2_PARTITION_WEIGHT_MONTHLY = "lens.driver.hive.priority.partition.weight.monthly";
- public static final String HS2_PARTITION_WEIGHT_DAILY = "lens.driver.hive.priority.partition.weight.daily";
- public static final String HS2_PARTITION_WEIGHT_HOURLY = "lens.driver.hive.priority.partition.weight.hourly";
// Default values of conf params
public static final long DEFAULT_EXPIRY_DELAY = 600 * 1000;
public static final String HS2_PRIORITY_DEFAULT_RANGES = "VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW";
- public static final float MONTHLY_PARTITION_WEIGHT_DEFAULT = 0.5f;
- public static final float DAILY_PARTITION_WEIGHT_DEFAULT = 0.75f;
- public static final float HOURLY_PARTITION_WEIGHT_DEFAULT = 1.0f;
public static final String SESSION_KEY_DELIMITER = ".";
/** The driver conf- which will merged with query conf */
@@ -123,8 +122,9 @@ public class HiveDriver implements LensDriver {
/** The driver listeners. */
private List<LensEventListener<DriverEvent>> driverListeners;
- QueryPriorityDecider queryPriorityDecider;
+ QueryCostCalculator queryCostCalculator;
+ QueryPriorityDecider queryPriorityDecider;
// package-local. Test case can change.
boolean whetherCalculatePriority;
private UserConfigLoader userConfigLoader;
@@ -331,15 +331,25 @@ public class HiveDriver implements LensDriver {
isEmbedded = (connectionClass.getName().equals(EmbeddedThriftConnection.class.getName()));
connectionExpiryTimeout = this.driverConf.getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
whetherCalculatePriority = this.driverConf.getBoolean(HS2_CALCULATE_PRIORITY, true);
- queryPriorityDecider = new DurationBasedQueryPriorityDecider(
- this,
- this.driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES),
- this.driverConf.getFloat(HS2_PARTITION_WEIGHT_MONTHLY, MONTHLY_PARTITION_WEIGHT_DEFAULT),
- this.driverConf.getFloat(HS2_PARTITION_WEIGHT_DAILY, DAILY_PARTITION_WEIGHT_DEFAULT),
- this.driverConf.getFloat(HS2_PARTITION_WEIGHT_HOURLY, HOURLY_PARTITION_WEIGHT_DEFAULT)
+ Class<? extends QueryCostCalculator> queryCostCalculatorClass = this.driverConf.getClass(HS2_COST_CALCULATOR,
+ FactPartitionBasedQueryCostCalculator.class, QueryCostCalculator.class);
+ try {
+ queryCostCalculator = queryCostCalculatorClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Can't instantiate query cost calculator of class: " + queryCostCalculatorClass, e);
+ }
+ queryPriorityDecider = new CostRangePriorityDecider(
+ new CostToPriorityRangeConf(driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES))
);
}
+ private QueryCost calculateQueryCost(AbstractQueryContext qctx) throws LensException {
+ if (qctx.isOlapQuery()) {
+ return queryCostCalculator.calculateCost(qctx, this);
+ } else {
+ return new FactPartitionBasedQueryCost(Double.MAX_VALUE);
+ }
+ }
@Override
public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
log.info("Estimate: " + qctx.getDriverQuery(this));
@@ -352,14 +362,17 @@ public class HiveDriver implements LensDriver {
}
if (qctx.isOlapQuery()) {
// if query is olap query and rewriting takes care of semantic validation
- // estimate is not doing anything as of now
- return HiveQueryPlan.HIVE_DRIVER_COST;
+ // estimate is calculating cost of the query
+ // the calculation is done only for cube queries
+ // for all other native table queries, the cost will be maximum
+ return calculateQueryCost(qctx);
} else {
- // its native table query. Do explain and return cost
+ // its native table query. validate and return cost
return explain(qctx).getCost();
}
}
+
/*
* (non-Javadoc)
*
@@ -385,14 +398,14 @@ public class HiveDriver implements LensDriver {
// Get result set of explain
HiveInMemoryResultSet inMemoryResultSet = (HiveInMemoryResultSet) execute(explainQueryCtx);
- List<String> explainOutput = new ArrayList<String>();
+ List<String> explainOutput = new ArrayList<>();
while (inMemoryResultSet.hasNext()) {
explainOutput.add((String) inMemoryResultSet.next().getValues().get(0));
}
closeQuery(explainQueryCtx.getQueryHandle());
try {
hiveConf.setClassLoader(explainCtx.getConf().getClassLoader());
- HiveQueryPlan hqp = new HiveQueryPlan(explainOutput, null, hiveConf);
+ HiveQueryPlan hqp = new HiveQueryPlan(explainOutput, null, hiveConf, calculateQueryCost(explainCtx));
explainCtx.getDriverContext().setDriverQueryPlan(this, hqp);
return hqp;
} catch (HiveException e) {
@@ -494,7 +507,10 @@ public class HiveDriver implements LensDriver {
if (whetherCalculatePriority) {
try {
// Inside try since non-data fetching queries can also be executed by async method.
- String priority = queryPriorityDecider.decidePriority(ctx).toString();
+ if (ctx.getDriverQueryCost(this) == null) {
+ ctx.setDriverCost(this, queryCostCalculator.calculateCost(ctx, this));
+ }
+ String priority = queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(this)).toString();
qdconf.set("mapred.job.priority", priority);
log.info("set priority to " + priority);
} catch (Exception e) {
@@ -942,7 +958,7 @@ public class HiveDriver implements LensDriver {
*/
QueryCompletionNotifier(QueryHandle handle, long timeoutMillis, QueryCompletionListener listener)
throws LensException {
- hiveHandle = getHiveHandle(handle);
+ this.handle = handle;
this.timeoutMillis = timeoutMillis;
this.listener = listener;
this.pollInterval = timeoutMillis / 10;
@@ -957,12 +973,17 @@ public class HiveDriver implements LensDriver {
public void run() {
// till query is complete or timeout has reached
long timeSpent = 0;
- String error = null;
+ String error;
try {
while (timeSpent <= timeoutMillis) {
- if (isFinished(hiveHandle)) {
- listener.onCompletion(handle);
- return;
+ try {
+ hiveHandle = getHiveHandle(handle);
+ if (isFinished(hiveHandle)) {
+ listener.onCompletion(handle);
+ return;
+ }
+ } catch(LensException e) {
+ log.debug("query handle: {} Not yet launched on driver", handle);
}
Thread.sleep(pollInterval);
timeSpent += pollInterval;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
index b0a04df..f38b1da 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
@@ -18,15 +18,12 @@
*/
package org.apache.lens.driver.hive;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.DriverQueryPlan;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -34,6 +31,7 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
+import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/**
@@ -42,11 +40,11 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class HiveQueryPlan extends DriverQueryPlan {
+ @Getter
+ private final QueryCost cost;
/** The explain output. */
private String explainOutput;
- static final QueryCost HIVE_DRIVER_COST = new QueryCost(1, 1.0);
-
/**
* The Enum ParserState.
*/
@@ -100,8 +98,9 @@ public class HiveQueryPlan extends DriverQueryPlan {
* @param metastoreConf the metastore conf
* @throws HiveException the hive exception
*/
- public HiveQueryPlan(List<String> explainOutput, QueryPrepareHandle prepared, HiveConf metastoreConf)
+ public HiveQueryPlan(List<String> explainOutput, QueryPrepareHandle prepared, HiveConf metastoreConf, QueryCost cost)
throws HiveException {
+ this.cost = cost;
setPrepareHandle(prepared);
setExecMode(ExecMode.BATCH);
setScanMode(ScanMode.PARTIAL_SCAN);
@@ -196,7 +195,7 @@ public class HiveQueryPlan extends DriverQueryPlan {
}
}
break;
- default :
+ default:
break;
}
}
@@ -242,12 +241,4 @@ public class HiveQueryPlan extends DriverQueryPlan {
public String getPlan() {
return explainOutput;
}
-
- @Override
- public QueryCost getCost() {
- /*
- * Return query cost as 1 so that if JDBC storage and other storage is present, JDBC is given preference.
- */
- return HIVE_DRIVER_COST;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
deleted file mode 100644
index bc1ec1c..0000000
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/priority/DurationBasedQueryPriorityDecider.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.hive.priority;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lens.api.Priority;
-import org.apache.lens.cube.metadata.FactPartition;
-import org.apache.lens.cube.metadata.UpdatePeriod;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.priority.CostToPriorityRangeConf;
-import org.apache.lens.server.api.priority.QueryPriorityDecider;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-
-public class DurationBasedQueryPriorityDecider implements QueryPriorityDecider {
-
-
- private final CostToPriorityRangeConf costToPriorityRangeMap;
-
- /** Partition Weights for priority calculation based on selected partitions **/
-
- /** weight of monthly partition * */
- private final float monthlyPartitionWeight;
- /** weight of daily partition * */
- private final float dailyPartitionWeight;
- /** weight of hourly partition * */
- private final float hourlyPartitionWeight;
- private final LensDriver driver;
-
-
- /**
- * Constructor. Takes three weights for partitions.
- * @param driver
- * @param ranges
- * @param monthlyPartitoinWeight
- * @param dailyPartitionWeight
- * @param hourlyPartitionWeight
- */
- public DurationBasedQueryPriorityDecider(LensDriver driver,
- String ranges, float monthlyPartitoinWeight, float dailyPartitionWeight, float hourlyPartitionWeight) {
- this.driver = driver;
- this.costToPriorityRangeMap = new CostToPriorityRangeConf(ranges);
- this.monthlyPartitionWeight = monthlyPartitoinWeight;
- this.dailyPartitionWeight = dailyPartitionWeight;
- this.hourlyPartitionWeight = hourlyPartitionWeight;
- }
-
- /**
- * The Implementation
- *
- * @param abstractQueryContext
- * @return decided Priority
- * @throws LensException Exception occurs mostly when one of drivers/explained queries/plans is null
- */
- public Priority decidePriority(AbstractQueryContext abstractQueryContext) throws LensException {
- float cost = getDurationCost(abstractQueryContext);
- Priority priority = costToPriorityRangeMap.get(cost);
- LOG.info("Deciding Priority " + priority + " since cost = " + cost);
- return priority;
- }
-
- /**
- * Calculates total cost based on weights of selected tables and their selected partitions
- *
- * @param queryContext
- * @return Query Cost
- * @throws LensException
- */
-
- @SuppressWarnings("unchecked") // required for (Set<FactPartition>) casting
- private float getDurationCost(AbstractQueryContext queryContext) throws LensException {
- float cost = 0;
- if (queryContext.getDriverContext().getDriverRewriterPlan(driver) != null) {
- // the calculation is done only for cube queries involving fact tables
- // for all other native table queries and dimension only queries, the cost will be zero and priority will
- // be the highest one associated with zero cost
- for (Map.Entry<String, Set<?>> entry : queryContext.getDriverContext().getDriverRewriterPlan(driver)
- .getPartitions().entrySet()) {
- if (!entry.getValue().isEmpty() && entry.getValue().iterator().next() instanceof FactPartition) {
- Set<FactPartition> factParts = (Set<FactPartition>)entry.getValue();
- for (FactPartition partition : factParts) {
- cost += getTableWeights(partition.getStorageTables(), queryContext) * getNormalizedPartitionCost(
- partition.getPeriod());
- }
- }
- }
- }
- return cost;
- }
-
- private float getTableWeights(Set<String> tables, AbstractQueryContext queryContext) {
- float weight = 0;
- for (String tblName : tables) {
- Double tblWeight = queryContext.getDriverContext().getDriverRewriterPlan(driver).getTableWeight(tblName);
- if (tblWeight != null) {
- weight += tblWeight;
- }
- }
- return weight == 0 ? 1 : weight;
- }
-
- /**
- * Normalized cost of a partition. PartitionWeight multiplied by number of days in that partition.
- *
- * @param partition
- * @return normalized cost.
- * @throws LensException
- */
- private float getNormalizedPartitionCost(UpdatePeriod updatePeriod) throws LensException {
- switch (updatePeriod) {
- case MONTHLY: //monthly
- return 30 * monthlyPartitionWeight;
- case DAILY: // daily
- return 1 * dailyPartitionWeight;
- case HOURLY: // hourly
- return (1.0f / 24) * hourlyPartitionWeight;
- default:
- throw new LensException("Weight not defined for " + updatePeriod);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 8355f29..67a4667 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -18,7 +18,8 @@
*/
package org.apache.lens.driver.hive;
-import static org.apache.lens.server.api.user.MockUserConfigLoader.*;
+import static org.apache.lens.server.api.user.MockUserConfigLoader.KEY;
+import static org.apache.lens.server.api.user.MockUserConfigLoader.VALUE;
import static org.testng.Assert.*;
@@ -28,30 +29,23 @@ import java.util.*;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.Priority;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.cube.metadata.FactPartition;
import org.apache.lens.cube.metadata.UpdatePeriod;
-import org.apache.lens.driver.hive.priority.DurationBasedQueryPriorityDecider;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.DriverQueryPlan;
+import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.driver.LensResultSet;
-import org.apache.lens.server.api.driver.LensResultSetMetadata;
import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.ExplainQueryContext;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.priority.CostRangePriorityDecider;
+import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
import org.apache.lens.server.api.user.MockUserConfigLoader;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.HiveDriverRunHook;
@@ -60,12 +54,8 @@ import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.ColumnDescriptor;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
+import org.testng.annotations.*;
-import junit.framework.Assert;
/**
* The Class TestHiveDriver.
@@ -165,7 +155,7 @@ public class TestHiveDriver {
protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf,
- drivers);
+ drivers);
ectx.setLensSessionIdentifier(sessionid);
return ectx;
}
@@ -201,7 +191,7 @@ public class TestHiveDriver {
context = createContext(dataLoad, conf);
resultSet = driver.execute(context);
assertNull(resultSet);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
}
/**
@@ -227,7 +217,7 @@ public class TestHiveDriver {
context = createContext(dataLoad, conf);
resultSet = driver.execute(context);
assertNull(resultSet);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
}
// Tests
@@ -263,15 +253,15 @@ public class TestHiveDriver {
QueryContext context = createContext(query, conf);
LensResultSet resultSet = driver.execute(context);
assertNull(resultSet);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
// fetch results from temp table
String select = "SELECT * FROM test_temp_output";
context = createContext(select, conf);
resultSet = driver.execute(context);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
validateInMemoryResult(resultSet, "test_temp_output");
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
}
/**
@@ -288,7 +278,7 @@ public class TestHiveDriver {
String select = "SELECT ID FROM test_execute";
QueryContext context = createContext(select, conf);
resultSet = driver.execute(context);
- Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.name"));
+ assertNotNull(context.getDriverConf(driver).get("mapred.job.name"));
validateInMemoryResult(resultSet);
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
context = createContext(select, conf);
@@ -302,7 +292,7 @@ public class TestHiveDriver {
context = createContext(select, conf);
resultSet = driver.execute(context);
validatePersistentResult(resultSet, TEST_DATA_FILE, context.getHDFSResultDir(), true);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
}
/**
@@ -407,30 +397,30 @@ public class TestHiveDriver {
failConf.set("hive.exec.driver.run.hooks", FailHook.class.getCanonicalName());
QueryContext context = createContext(expectFail, failConf);
driver.executeAsync(context);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.FAILED, true, false);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
driver.closeQuery(context.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
// Async select query
String select = "SELECT ID FROM test_execute_sync";
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
context = createContext(select, conf);
driver.executeAsync(context);
- Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.name"));
- Assert.assertNotNull(context.getDriverConf(driver).get("mapred.job.priority"));
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertNotNull(context.getDriverConf(driver).get("mapred.job.name"));
+ assertNotNull(context.getDriverConf(driver).get("mapred.job.priority"));
+ assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, false, false);
driver.closeQuery(context.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
conf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, true);
context = createContext(select, conf);
driver.executeAsync(context);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, true, false);
driver.closeQuery(context.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT,
"ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
@@ -439,10 +429,10 @@ public class TestHiveDriver {
select = "SELECT ID, null, ID FROM test_execute_sync";
context = createContext(select, conf);
driver.executeAsync(context);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(context, DriverQueryState.SUCCESSFUL, true, true);
driver.closeQuery(context.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
}
/**
@@ -506,7 +496,7 @@ public class TestHiveDriver {
driver.updateStatus(context);
assertEquals(context.getDriverStatus().getState(), DriverQueryState.CANCELED, "Expecting query to be cancelled");
driver.closeQuery(context.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
try {
driver.cancelQuery(context.getQueryHandle());
@@ -589,14 +579,14 @@ public class TestHiveDriver {
QueryContext ctx = createContext("SELECT ID FROM test_persistent_result_set", conf);
LensResultSet resultSet = driver.execute(ctx);
validatePersistentResult(resultSet, TEST_DATA_FILE, ctx.getHDFSResultDir(), false);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
ctx = createContext("SELECT ID FROM test_persistent_result_set", conf);
driver.executeAsync(ctx);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(ctx, DriverQueryState.SUCCESSFUL, true, false);
driver.closeQuery(ctx.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
conf.set(LensConfConstants.QUERY_OUTPUT_DIRECTORY_FORMAT,
"ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'"
@@ -604,17 +594,17 @@ public class TestHiveDriver {
+ " 'field.delim'=',' ) STORED AS TEXTFILE ");
ctx = createContext("SELECT ID, null, ID FROM test_persistent_result_set", conf);
resultSet = driver.execute(ctx);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
validatePersistentResult(resultSet, TEST_DATA_FILE, ctx.getHDFSResultDir(), true);
driver.closeQuery(ctx.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
ctx = createContext("SELECT ID, null, ID FROM test_persistent_result_set", conf);
driver.executeAsync(ctx);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(ctx, DriverQueryState.SUCCESSFUL, true, true);
driver.closeQuery(ctx.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
}
/**
@@ -640,38 +630,40 @@ public class TestHiveDriver {
}
}
- @Test
+ @Test(expectedExceptions = {UnsupportedOperationException.class})
public void testEstimateNativeQuery() throws Exception {
createTestTable("test_estimate");
SessionState.setCurrentSessionState(ss);
QueryCost cost = driver.estimate(createExplainContext("SELECT ID FROM test_estimate", conf));
- Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 1);
- Assert.assertEquals(cost.getEstimatedResourceUsage(), 1.0);
+ assertEquals(cost.getEstimatedResourceUsage(), Double.MAX_VALUE);
+ cost.getEstimatedExecTimeMillis();
+
}
- @Test
+ @Test(expectedExceptions = {UnsupportedOperationException.class})
public void testEstimateOlapQuery() throws Exception {
SessionState.setCurrentSessionState(ss);
ExplainQueryContext ctx = createExplainContext("cube SELECT ID FROM test_cube", conf);
ctx.setOlapQuery(true);
QueryCost cost = driver.estimate(ctx);
- Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 1);
- Assert.assertEquals(cost.getEstimatedResourceUsage(), 1.0);
+ assertEquals(cost.getEstimatedResourceUsage(), 0.0);
+ cost.getEstimatedExecTimeMillis();
}
@Test
- public void testEstimateNativeFailingQuery() throws Exception {
+ public void testExplainNativeFailingQuery() throws Exception {
SessionState.setCurrentSessionState(ss);
try {
driver.estimate(createExplainContext("SELECT ID FROM nonexist", conf));
- Assert.fail("Should not reach here");
+ fail("Should not reach here");
} catch (LensException e) {
- Assert.assertEquals(LensUtil.getCauseMessage(e), "Error while"
+ assertEquals(LensUtil.getCauseMessage(e), "Error while"
+ " compiling statement: FAILED: SemanticException [Error 10001]: Line 1:32 Table not found 'nonexist'");
}
}
// explain
+
/**
* Test explain.
*
@@ -686,7 +678,7 @@ public class TestHiveDriver {
DriverQueryPlan plan = driver.explain(createExplainContext("SELECT ID FROM test_explain", conf));
assertTrue(plan instanceof HiveQueryPlan);
assertEquals(plan.getTableWeight(dataBase + ".test_explain"), 500.0);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
// test execute prepare
PreparedQueryContext pctx = new PreparedQueryContext("SELECT ID FROM test_explain", null, conf, drivers);
@@ -699,7 +691,7 @@ public class TestHiveDriver {
plan = driver.explainAndPrepare(pctx);
QueryContext qctx = createContext(pctx, inConf);
LensResultSet result = driver.execute(qctx);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
validateInMemoryResult(result);
// test execute prepare async
@@ -707,28 +699,28 @@ public class TestHiveDriver {
driver.executeAsync(qctx);
assertNotNull(qctx.getDriverOpHandle());
validateExecuteAsync(qctx, DriverQueryState.SUCCESSFUL, true, false);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
driver.closeQuery(qctx.getQueryHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
// for backward compatibility
qctx = createContext(pctx, inConf);
qctx.setQueryHandle(new QueryHandle(pctx.getPrepareHandle().getPrepareHandleId()));
result = driver.execute(qctx);
assertNotNull(qctx.getDriverOpHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
validateInMemoryResult(result);
// test execute prepare async
qctx = createContext(pctx, conf);
qctx.setQueryHandle(new QueryHandle(pctx.getPrepareHandle().getPrepareHandleId()));
driver.executeAsync(qctx);
- Assert.assertEquals(1, driver.getHiveHandleSize());
+ assertEquals(1, driver.getHiveHandleSize());
validateExecuteAsync(qctx, DriverQueryState.SUCCESSFUL, true, false);
driver.closeQuery(qctx.getQueryHandle());
driver.closePreparedQuery(pctx.getPrepareHandle());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
}
/**
@@ -742,7 +734,7 @@ public class TestHiveDriver {
// acquire
SessionState.setCurrentSessionState(ss);
DriverQueryPlan plan = driver.explain(createExplainContext("SELECT ID FROM test_part_table", conf));
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
assertTrue(plan instanceof HiveQueryPlan);
assertNotNull(plan.getTablesQueried());
assertEquals(plan.getTablesQueried().size(), 1);
@@ -751,8 +743,8 @@ public class TestHiveDriver {
System.out.println("Parts:" + plan.getPartitions());
assertFalse(plan.getPartitions().isEmpty());
assertEquals(plan.getPartitions().size(), 1);
- assertTrue(((String)plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("today"));
- assertTrue(((String)plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("dt"));
+ assertTrue(((String) plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("today"));
+ assertTrue(((String) plan.getPartitions().get(dataBase + ".test_part_table").iterator().next()).contains("dt"));
}
/**
@@ -770,7 +762,7 @@ public class TestHiveDriver {
+ " explain_test_1 join explain_test_2 on explain_test_1.ID = explain_test_2.ID"
+ " WHERE explain_test_1.ID = 'foo' or explain_test_2.ID = 'bar'" + " GROUP BY explain_test_1.ID", conf));
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
assertTrue(plan instanceof HiveQueryPlan);
assertNotNull(plan.getTablesQueried());
assertEquals(plan.getTablesQueried().size(), 2);
@@ -797,13 +789,13 @@ public class TestHiveDriver {
pctx.setLensSessionIdentifier(sessionid);
DriverQueryPlan plan2 = driver.explainAndPrepare(pctx);
// assertNotNull(plan2.getResultDestination());
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
assertNotNull(plan2.getTablesQueried());
assertEquals(plan2.getTablesQueried().size(), 1);
assertTrue(plan2.getTableWeights().containsKey(dataBase + ".explain_test_1"));
QueryContext ctx = createContext(pctx, conf);
LensResultSet resultSet = driver.execute(ctx);
- Assert.assertEquals(0, driver.getHiveHandleSize());
+ assertEquals(0, driver.getHiveHandleSize());
HivePersistentResultSet persistentResultSet = (HivePersistentResultSet) resultSet;
String path = persistentResultSet.getOutputPath();
assertEquals(ctx.getHdfsoutPath(), path);
@@ -820,10 +812,8 @@ public class TestHiveDriver {
@Test
public void testPriority() throws IOException, LensException, ParseException {
Configuration conf = new Configuration();
- DurationBasedQueryPriorityDecider alwaysNormalPriorityDecider = new DurationBasedQueryPriorityDecider(driver,
- "",
- HiveDriver.MONTHLY_PARTITION_WEIGHT_DEFAULT,
- HiveDriver.DAILY_PARTITION_WEIGHT_DEFAULT, HiveDriver.HOURLY_PARTITION_WEIGHT_DEFAULT);
+ CostRangePriorityDecider alwaysNormalPriorityDecider =
+ new CostRangePriorityDecider(new CostToPriorityRangeConf(""));
BufferedReader br = new BufferedReader(new InputStreamReader(
TestHiveDriver.class.getResourceAsStream("/priority_tests.data")));
String line;
@@ -862,8 +852,9 @@ public class TestHiveDriver {
}
});
}
- Assert.assertEquals(expected, driver.queryPriorityDecider.decidePriority(ctx));
- Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx));
+ ctx.setDriverCost(driver, driver.queryCostCalculator.calculateCost(ctx, driver));
+ assertEquals(expected, driver.queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver)));
+ assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver)));
i++;
}
// test priority without fact partitions
@@ -893,13 +884,15 @@ public class TestHiveDriver {
put("table1", 1.0);
}
});
- Assert.assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx));
- Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx));
+ ctx.setDriverCost(driver, driver.queryCostCalculator.calculateCost(ctx, driver));
+ assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver)));
+ assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver)));
// test priority without rewriter plan
ctx = createContext("test priority query", conf);
- Assert.assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx));
- Assert.assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx));
+ ctx.setDriverCost(driver, driver.queryCostCalculator.calculateCost(ctx, driver));
+ assertEquals(Priority.VERY_HIGH, driver.queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver)));
+ assertEquals(Priority.NORMAL, alwaysNormalPriorityDecider.decidePriority(ctx.getDriverQueryCost(driver)));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index bda6572..0f2b740 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -33,9 +33,7 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lens.api.LensConf;
-import org.apache.lens.api.query.QueryCost;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryPrepareHandle;
+import org.apache.lens.api.query.*;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
@@ -44,6 +42,8 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
@@ -543,7 +543,7 @@ public class JDBCDriver implements LensDriver {
return rewrittenQuery;
}
- private static final QueryCost JDBC_DRIVER_COST = new QueryCost(0, 0);
+ static final QueryCost JDBC_DRIVER_COST = new FactPartitionBasedQueryCost(0);
/**
* Dummy JDBC query Plan class to get min cost selector working.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index be34164..a6c97ae 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -18,31 +18,28 @@
*/
package org.apache.lens.driver.jdbc;
-import static org.apache.lens.server.api.user.MockUserConfigLoader.*;
+import static org.apache.lens.server.api.user.MockUserConfigLoader.KEY;
+import static org.apache.lens.server.api.user.MockUserConfigLoader.VALUE;
import static org.testng.Assert.*;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
+import java.sql.*;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.lens.api.LensConf;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.ResultRow;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
-import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.metrics.LensMetricsRegistry;
import org.apache.lens.server.api.query.ExplainQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.user.MockUserConfigLoader;
import org.apache.lens.server.api.util.LensUtil;
@@ -52,14 +49,10 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.service.cli.ColumnDescriptor;
import org.testng.Assert;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
+import org.testng.annotations.*;
import com.codahale.metrics.MetricRegistry;
import com.mchange.v2.c3p0.ComboPooledDataSource;
-
import lombok.extern.slf4j.Slf4j;
/**
@@ -133,7 +126,7 @@ public class TestJdbcDriver {
protected ExplainQueryContext createExplainContext(final String query, Configuration conf) {
ExplainQueryContext ectx = new ExplainQueryContext(UUID.randomUUID().toString(), query, "testuser", null, conf,
- drivers);
+ drivers);
return ectx;
}
@@ -265,8 +258,7 @@ public class TestJdbcDriver {
ExplainQueryContext ctx = createExplainContext(query1, baseConf);
Assert.assertNull(ctx.getFinalDriverQuery(driver));
QueryCost cost = driver.estimate(ctx);
- Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0);
- Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0);
+ Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
Assert.assertNotNull(ctx.getFinalDriverQuery(driver));
// Test connection leak for estimate
@@ -339,8 +331,7 @@ public class TestJdbcDriver {
// run estimate and execute - because server would first run estimate and then execute with same context
QueryContext ctx = createQueryContext(query1, metricConf);
QueryCost cost = driver.estimate(ctx);
- Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0);
- Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0);
+ Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
LensResultSet result = driver.execute(ctx);
Assert.assertNotNull(result);
@@ -348,15 +339,13 @@ public class TestJdbcDriver {
// run estimate and prepare - because server would first run estimate and then prepare with same context
PreparedQueryContext pContext = new PreparedQueryContext(query1, "SA", metricConf, drivers);
cost = driver.estimate(pContext);
- Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0);
- Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0);
+ Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
driver.prepare(pContext);
// test explain and prepare
PreparedQueryContext pContext2 = new PreparedQueryContext(query1, "SA", metricConf, drivers);
cost = driver.estimate(pContext2);
- Assert.assertEquals(cost.getEstimatedExecTimeMillis(), 0);
- Assert.assertEquals(cost.getEstimatedResourceUsage(), 0.0);
+ Assert.assertEquals(cost, JDBCDriver.JDBC_DRIVER_COST);
driver.prepare(pContext2);
driver.explainAndPrepare(pContext2);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
index 8957aef..5d58ef2 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
@@ -30,6 +30,7 @@ import javax.xml.bind.JAXBException;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.query.*;
import org.apache.lens.api.result.LensAPIResult;
+import org.apache.lens.api.result.QueryCostTO;
import org.apache.lens.regression.core.constants.QueryURL;
import org.apache.lens.regression.core.type.FormBuilder;
import org.apache.lens.regression.core.type.MapBuilder;
@@ -253,7 +254,7 @@ public class QueryHelper extends ServiceManagerHelper {
* @return the Estimate result
*/
- public QueryCost estimateQuery(String queryString, String sessionHandleString, String conf) throws
+ public QueryCostTO estimateQuery(String queryString, String sessionHandleString, String conf) throws
InstantiationException, IllegalAccessException, JAXBException, LensException {
FormBuilder formData = new FormBuilder();
formData.add("sessionid", sessionHandleString);
@@ -266,20 +267,20 @@ public class QueryHelper extends ServiceManagerHelper {
String queryCostString = response.readEntity(String.class);
logger.info(queryCostString);
LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryCostString, LensAPIResult.class);
- QueryCost queryCost = (QueryCost) successResponse.getData();
- if (queryCost == null) {
+ QueryCostTO queryCostTO = (QueryCostTO) successResponse.getData();
+ if (queryCostTO == null) {
throw new LensException("Estimate Failed");
}
- return queryCost;
+ return queryCostTO;
}
- public QueryCost estimateQuery(String queryString, String sessionHandleString) throws
+ public QueryCostTO estimateQuery(String queryString, String sessionHandleString) throws
InstantiationException, IllegalAccessException, JAXBException, LensException {
return estimateQuery(queryString, sessionHandleString,
"<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?><conf />");
}
- public QueryCost estimateQuery(String queryString) throws
+ public QueryCostTO estimateQuery(String queryString) throws
InstantiationException, IllegalAccessException, JAXBException, LensException {
return estimateQuery(queryString, sessionHandleString);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
index 51b1217..c3ec099 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryPlan.java
@@ -22,14 +22,18 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPlan;
import org.apache.lens.api.query.QueryPrepareHandle;
+import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.cost.QueryCostTOBuilder;
+
+import lombok.extern.slf4j.Slf4j;
/**
* The Class DriverQueryPlan.
*/
+@Slf4j
public abstract class DriverQueryPlan {
/**
@@ -126,7 +130,7 @@ public abstract class DriverQueryPlan {
/**
* Get the cost associated with the plan
*
- * @return QueryCost object
+ * @return QueryCostTO object
*/
public abstract QueryCost getCost();
@@ -299,8 +303,8 @@ public abstract class DriverQueryPlan {
* @throws UnsupportedEncodingException the unsupported encoding exception
*/
public QueryPlan toQueryPlan() throws UnsupportedEncodingException {
- return new QueryPlan(new ArrayList<String>(tablesQueried), hasSubQuery, execMode != null ? execMode.name() : null,
+ return new QueryPlan(new ArrayList<>(tablesQueried), hasSubQuery, execMode != null ? execMode.name() : null,
scanMode != null ? scanMode.name() : null, handle,
- URLEncoder.encode(getPlan(), "UTF-8"), getCost(), false, null);
+ URLEncoder.encode(getPlan(), "UTF-8"), new QueryCostTOBuilder(getCost()).build(), false, null);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 3dc4e73..b56c67c 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -20,14 +20,12 @@ package org.apache.lens.server.api.driver;
import java.io.Externalizable;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.conf.Configuration;
@@ -57,7 +55,7 @@ public interface LensDriver extends Externalizable {
*
* @param qctx The query context
*
- * @return The QueryCost object
+ * @return The QueryCostTO object
*
* @throws LensException the lens exception if driver cannot estimate
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
index 43237f7..c3fe8eb 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
@@ -21,8 +21,8 @@ package org.apache.lens.server.api.driver;
import java.util.Collections;
import java.util.Comparator;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.hadoop.conf.Configuration;
import org.apache.log4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
deleted file mode 100644
index 97574b8..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/CostToPriorityRangeConf.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.priority;
-
-
-import org.apache.lens.api.Priority;
-
-/**
- * Cost to priority range conf class.
- *
- * @see RangeConf
- */
-
-public class CostToPriorityRangeConf extends RangeConf<Float, Priority> {
- /**
- * Super constructor
- *
- * @param confValue
- * @see RangeConf#RangeConf(String)
- */
- public CostToPriorityRangeConf(String confValue) {
- super(confValue);
- }
-
- /**
- * Parse key method
- *
- * @param s
- * @return parsed float from string s
- * @see RangeConf#parseKey(String)
- */
- @Override
- protected Float parseKey(String s) {
- return Float.parseFloat(s);
- }
-
- /**
- * Parse value method
- *
- * @param s
- * @return parsed Priority from String s
- * @see RangeConf#parseValue(String)
- */
- @Override
- protected Priority parseValue(String s) {
- return Priority.valueOf(s);
- }
-
- /**
- * Default value is "NORMAL". Which implies {@link #get(Comparable)} will always return Priotity.NORMAL
- *
- * @return "NORMAL"
- */
- @Override
- protected String getDefaultConf() {
- return Priority.NORMAL.toString();
- }
-}
[36/50] [abbrv] incubator-lens git commit: LENS-690 : Add video demo
to the website
Posted by sh...@apache.org.
LENS-690 : Add video demo to the website
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c611bd7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c611bd7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c611bd7b
Branch: refs/heads/LENS-581
Commit: c611bd7b1ccdffe4c0a010ae37475843b2540688
Parents: cb28e06
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jul 24 13:17:44 2015 +0530
Committer: arshad-matin <ar...@apache.org>
Committed: Fri Jul 24 13:17:44 2015 +0530
----------------------------------------------------------------------
src/site/site.xml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c611bd7b/src/site/site.xml
----------------------------------------------------------------------
diff --git a/src/site/site.xml b/src/site/site.xml
index f26551c..9cf7f71 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -31,7 +31,7 @@
<href>http://lens.incubator.apache.org</href>
<height>54px</height>
</bannerLeft>
-
+
<bannerRight>
<name>Apache Incubator</name>
<src>/images/apache-incubator-logo.png</src>
@@ -44,6 +44,8 @@
<item name="Getting Started" href="./lenshome/quick-start.html" />
<item name="Installing and Running Lens" href="./lenshome/install-and-run.html" />
<item name="Pseudo Distributed Setup" href="./lenshome/pseudo-distributed-setup.html" />
+ <item name="20 Minute Video demo" target="_new"
+ href="http://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples" />
</menu>
<menu name="Releases" >
<item name="Download" href="./releases/download.html" />
[29/50] [abbrv] incubator-lens git commit: LENS-619: Applying Query
Launching Constraints before allowing a query to be launched (Also contains
LENS-687 and LENS-688)
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 63569d8..9e5fcd5 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -20,6 +20,7 @@ package org.apache.lens.server.query;
import static org.apache.lens.api.query.QueryStatus.Status.*;
import static org.apache.lens.server.api.LensConfConstants.*;
+import static org.apache.lens.server.api.util.LensUtil.getImplementations;
import static org.apache.lens.server.session.LensSessionImpl.ResourceEntry;
import java.io.*;
@@ -28,6 +29,7 @@ import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.NotFoundException;
@@ -53,13 +55,19 @@ import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
+import org.apache.lens.server.query.collect.*;
+import org.apache.lens.server.query.constraint.DefaultQueryLaunchingConstraintsChecker;
+import org.apache.lens.server.query.constraint.QueryLaunchingConstraintsChecker;
import org.apache.lens.server.rewrite.RewriteUtil;
import org.apache.lens.server.rewrite.UserQueryToCubeQueryRewriter;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
+import org.apache.lens.server.util.FairPriorityBlockingQueue;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.collections.CollectionUtils;
@@ -80,6 +88,7 @@ import org.codehaus.jackson.map.*;
import org.codehaus.jackson.map.module.SimpleModule;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
@@ -133,12 +142,17 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* The accepted queries.
*/
- private PriorityBlockingQueue<QueryContext> queuedQueries = new PriorityBlockingQueue<QueryContext>();
+ private FairPriorityBlockingQueue<QueryContext> queuedQueries = new FairPriorityBlockingQueue<QueryContext>();
/**
* The launched queries.
*/
- private List<QueryContext> launchedQueries = new ArrayList<QueryContext>();
+ private EstimatedQueryCollection launchedQueries;
+
+ /**
+ * The waiting queries.
+ */
+ private EstimatedQueryCollection waitingQueries;
/**
* The finished queries.
@@ -169,12 +183,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* The query submitter runnable.
*/
- private final QuerySubmitter querySubmitterRunnable = new QuerySubmitter(LensServices.get().getErrorCollection());
+ private QuerySubmitter querySubmitterRunnable;
/**
* The query submitter.
*/
- protected final Thread querySubmitter = new Thread(querySubmitterRunnable, "QuerySubmitter");
+ protected Thread querySubmitter;
/**
* The status poller.
@@ -240,6 +254,23 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private final ErrorCollection errorCollection = LensServices.get().getErrorCollection();
+ private QueryLaunchingConstraintsChecker queryConstraintsChecker;
+
+ private WaitingQueriesSelector waitingQueriesSelector;
+
+ /**
+ * If the query taken out of queued queries is added to waiting, it will be re processed from waiting
+ * queries, when the next launched query is finished and removed from launched queries. If a query is
+ * removed from launched queries, while query not allowed to launch was still to be added to waiting
+ * queries, then waiting query will have to wait for next launched query to finish.
+ * This sort of delay in waiting query execution can be avoided if removal of a query from launched
+ * queries is locked using removalFromLaunchedQueriesLock, until the decision to add to waiting queries
+ * and actual addition to waiting query is complete.
+ * */
+ private final ReentrantLock removalFromLaunchedQueriesLock = new ReentrantLock();
+
+ private final ExecutorService waitingQueriesSelectionSvc = Executors.newSingleThreadExecutor();
+
/**
* The driver event listener.
*/
@@ -486,8 +517,17 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private final ErrorCollection errorCollection;
- public QuerySubmitter(@NonNull final ErrorCollection errorCollection) {
+ private final EstimatedQueryCollection waitingQueries;
+
+ private final QueryLaunchingConstraintsChecker constraintsChecker;
+
+ public QuerySubmitter(@NonNull final ErrorCollection errorCollection,
+ @NonNull final EstimatedQueryCollection waitingQueries,
+ @NonNull final QueryLaunchingConstraintsChecker constraintsChecker) {
+
this.errorCollection = errorCollection;
+ this.waitingQueries = waitingQueries;
+ this.constraintsChecker = constraintsChecker;
}
/*
@@ -500,43 +540,69 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("Starting QuerySubmitter thread");
while (!pausedForTest && !stopped && !querySubmitter.isInterrupted()) {
try {
- QueryContext ctx = queuedQueries.take();
+ QueryContext query = queuedQueries.take();
+ synchronized (query) {
+
+ /* Setting log segregation id */
+ logSegregationContext.setLogSegragationAndQueryId(query.getQueryHandleString());
+
+ if (!query.queued()) {
+ log.info("Probably the query got cancelled. Skipping it. Query Status:{}", query.getStatus());
+ continue;
+ }
- /* Setting log segregation id */
- logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString());
+ log.info("Processing query:{}", query.getUserQuery());
+ try {
+ // acquire session before any query operation.
+ acquire(query.getLensSessionIdentifier());
+ // the check to see if the query was already rewritten and selected driver's rewritten query is set
+ if (!query.isDriverQueryExplicitlySet()) {
+ rewriteAndSelect(query);
+ } else {
+ log.debug("Query is already rewritten");
+ }
- synchronized (ctx) {
- if (ctx.getStatus().getStatus().equals(QUEUED)) {
- log.info("Launching query:" + ctx.getUserQuery());
+ /* Check javadoc of QueryExecutionServiceImpl#removalFromLaunchedQueriesLock for reason for existence
+ of this lock. */
+ log.debug("Acquiring lock in QuerySubmitter");
+ removalFromLaunchedQueriesLock.lock();
try {
- // acquire session before any query operation.
- acquire(ctx.getLensSessionIdentifier());
- // the check to see if the query was already rewritten and selected driver's rewritten query is set
- if (!ctx.isDriverQueryExplicitlySet()) {
- rewriteAndSelect(ctx);
+
+ boolean isQueryAllowedToLaunch = this.constraintsChecker.canLaunch(query, launchedQueries);
+
+ log.debug("isQueryAllowedToLaunch:{}", isQueryAllowedToLaunch);
+ if (isQueryAllowedToLaunch) {
+
+ /* Query is not going to be added to waiting queries. No need to keep the lock.
+ First release lock, then launch query */
+ removalFromLaunchedQueriesLock.unlock();
+ launchQuery(query);
} else {
- log.info("Submitting to already selected driver");
+
+ /* Query is going to be added to waiting queries. Keep holding the lock to avoid any removal from
+ launched queries. First add to waiting queries, then release lock */
+ addToWaitingQueries(query);
+ removalFromLaunchedQueriesLock.unlock();
}
- // Check if we need to pass session's effective resources to selected driver
- addSessionResourcesToDriver(ctx);
- ctx.getSelectedDriver().executeAsync(ctx);
- } catch (LensException e) {
-
- LOG.error("Error launching query " + ctx.getQueryHandle(), e);
- String reason = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
- setFailedStatus(ctx, "Launching query failed", reason, e.buildLensErrorTO(this.errorCollection));
- continue;
-
- } catch (Exception e) {
- log.error("Error launching query " + ctx.getQueryHandle(), e);
- String reason = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
- setFailedStatus(ctx, "Launching query failed", reason, null);
- continue;
} finally {
- release(ctx.getLensSessionIdentifier());
+ if (removalFromLaunchedQueriesLock.isHeldByCurrentThread()) {
+ removalFromLaunchedQueriesLock.unlock();
+ }
}
- setLaunchedStatus(ctx);
- log.info("Launched query " + ctx.getQueryHandle());
+ } catch (LensException e) {
+
+ log.error("Error launching query: {}", query.getQueryHandle(), e);
+ String reason = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
+ setFailedStatus(query, "Launching query failed", reason, e.buildLensErrorTO(this.errorCollection));
+ continue;
+
+ } catch (Exception e) {
+ log.error("Error launching query: {}", query.getQueryHandle(), e);
+ String reason = e.getCause() != null ? e.getCause().getMessage() : e.getMessage();
+ setFailedStatus(query, "Launching query failed", reason, null);
+ continue;
+ } finally {
+ release(query.getLensSessionIdentifier());
}
}
} catch (InterruptedException e) {
@@ -549,6 +615,40 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
log.info("QuerySubmitter exited");
}
+
+ private void launchQuery(final QueryContext query) throws LensException {
+
+ checkEstimatedQueriesState(query);
+ QueryStatus oldStatus = query.getStatus();
+ QueryStatus newStatus = new QueryStatus(query.getStatus().getProgress(), QueryStatus.Status.LAUNCHED,
+ "Query is launched on driver", false, null, null, null);
+ query.validateTransition(newStatus);
+
+ // Check if we need to pass session's effective resources to selected driver
+ addSessionResourcesToDriver(query);
+ query.getSelectedDriver().executeAsync(query);
+ query.setStatusSkippingTransitionTest(newStatus);
+ query.setLaunchTime(System.currentTimeMillis());
+ query.clearTransientStateAfterLaunch();
+
+ launchedQueries.add(query);
+ log.info("Added to launched queries. QueryId:{}", query.getQueryHandleString());
+ fireStatusChangeEvent(query, newStatus, oldStatus);
+ }
+
+ private void addToWaitingQueries(final QueryContext query) throws LensException {
+
+ checkEstimatedQueriesState(query);
+ this.waitingQueries.add(query);
+ log.info("Added to waiting queries. QueryId:{}", query.getQueryHandleString());
+ }
+
+ private void checkEstimatedQueriesState(final QueryContext query) throws LensException {
+ if (query.getSelectedDriver() == null || query.getSelectedDriverQueryCost() == null) {
+ throw new LensException("selected driver: " + query.getSelectedDriver() +" OR selected driver query cost: "
+ + query.getSelectedDriverQueryCost() + " is null. Query doesn't appear to be an estimated query.");
+ }
+ }
}
// used in tests
@@ -580,8 +680,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("Starting Status poller thread");
while (!stopped && !statusPoller.isInterrupted()) {
try {
- List<QueryContext> launched = new ArrayList<QueryContext>();
- launched.addAll(launchedQueries);
+ Set<QueryContext> launched = launchedQueries.getQueries();
+
for (QueryContext ctx : launched) {
if (stopped || statusPoller.isInterrupted()) {
return;
@@ -627,16 +727,6 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
}
- private void setLaunchedStatus(QueryContext ctx) throws LensException {
- QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(ctx.getStatus().getProgress(), LAUNCHED, "launched on the driver",
- false, null, null, null));
- launchedQueries.add(ctx);
- ctx.setLaunchTime(System.currentTimeMillis());
- fireStatusChangeEvent(ctx, ctx.getStatus(), before);
- ctx.clearTransientStateAfterLaunch();
- }
-
/**
* Sets the cancelled status.
*
@@ -660,10 +750,18 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private void updateFinishedQuery(QueryContext ctx, QueryStatus before) {
// before would be null in case of server restart
if (before != null) {
- if (before.getStatus().equals(QUEUED)) {
+ if (before.queued()) {
+ /* Seems like query is cancelled, remove it from both queuedQueries and waitingQueries because we don't know
+ * where it is right now. It might happen that when we remove it from queued, it was in waiting OR
+ * when we removed it from waiting, it was in queued. We might just miss removing it from everywhere due to this
+ * hide and seek. Then QuerySubmitter thread will come to rescue, as it always checks that a query should be in
+ * queued state before processing it after deque. If it is in cancelled state, then it will skip it. */
queuedQueries.remove(ctx);
+ waitingQueries.remove(ctx);
} else {
- launchedQueries.remove(ctx);
+ if (removeFromLaunchedQueries(ctx)) {
+ processWaitingQueriesAsync(ctx);
+ }
}
}
finishedQueries.add(new FinishedQuery(ctx));
@@ -689,8 +787,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (ctx != null) {
synchronized (ctx) {
QueryStatus before = ctx.getStatus();
- if (!ctx.getStatus().getStatus().equals(QUEUED) && !ctx.getDriverStatus().isFinished()
- && !ctx.getStatus().finished()) {
+ if (!ctx.queued() && !ctx.finished() && !ctx.getDriverStatus().isFinished()) {
log.info("Updating status for " + ctx.getQueryHandle());
try {
ctx.getSelectedDriver().updateStatus(ctx);
@@ -909,6 +1006,26 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public synchronized void init(HiveConf hiveConf) {
super.init(hiveConf);
this.conf = hiveConf;
+
+ this.launchedQueries
+ = new ThreadSafeEstimatedQueryCollection(new DefaultEstimatedQueryCollection(new DefaultQueryCollection()));
+ this.waitingQueries
+ = new ThreadSafeEstimatedQueryCollection(new DefaultEstimatedQueryCollection(new DefaultQueryCollection()));
+
+ ImmutableSet<QueryLaunchingConstraint> queryConstraints = getImplementations(
+ QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, hiveConf);
+
+ this.queryConstraintsChecker = new DefaultQueryLaunchingConstraintsChecker(queryConstraints);
+
+ this.querySubmitterRunnable = new QuerySubmitter(LensServices.get().getErrorCollection(), this.waitingQueries,
+ this.queryConstraintsChecker);
+ this.querySubmitter = new Thread(querySubmitterRunnable, "QuerySubmitter");
+
+ ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies = getImplementations(
+ WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, hiveConf);
+
+ this.waitingQueriesSelector = new IntersectingWaitingQueriesSelector(selectionPolicies);
+
try {
this.userQueryToCubeQueryRewriter = new UserQueryToCubeQueryRewriter(conf);
} catch (LensException e) {
@@ -992,6 +1109,9 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
public synchronized void stop() {
super.stop();
+
+ waitingQueriesSelectionSvc.shutdown();
+
for (Thread th : new Thread[]{querySubmitter, statusPoller, queryPurger, prepareQueryPurger}) {
try {
log.debug("Waiting for" + th.getName());
@@ -1082,7 +1202,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
private void rewriteAndSelect(final AbstractQueryContext ctx) throws LensException {
MethodMetricsContext parallelCallGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
- PARALLEL_CALL_GAUGE);
+ PARALLEL_CALL_GAUGE);
try {
userQueryToCubeQueryRewriter.rewrite(ctx);
// Initially we obtain individual runnables for rewrite and estimate calls
@@ -1170,12 +1290,13 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
MethodMetricsContext selectGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
- DRIVER_SELECTOR_GAUGE);
+ DRIVER_SELECTOR_GAUGE);
// 2. select driver to run the query
LensDriver driver = driverSelector.select(ctx, conf);
- selectGauge.markSuccess();
-
ctx.setSelectedDriver(driver);
+ QueryCost selectedDriverQueryCost = ctx.getDriverContext().getDriverQueryCost(driver);
+ ctx.setSelectedDriverQueryCost(selectedDriverQueryCost);
+ selectGauge.markSuccess();
} finally {
parallelCallGauge.markSuccess();
}
@@ -1493,7 +1614,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
long timeoutMillis, LensConf conf, String queryName) throws LensException {
try {
log.info("ExecutePrepare: " + sessionHandle.toString() + " query:" + prepareHandle.getPrepareHandleId()
- + " timeout:" + timeoutMillis);
+ + " timeout:" + timeoutMillis);
acquire(sessionHandle);
PreparedQueryContext pctx = getPreparedQueryContext(sessionHandle, prepareHandle);
Configuration qconf = getLensConf(sessionHandle, conf);
@@ -1577,6 +1698,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
QueryStatus before = ctx.getStatus();
ctx.setStatus(new QueryStatus(0.0, QUEUED, "Query is queued", false, null, null, null));
queuedQueries.add(ctx);
+ log.debug("Added to Queued Queries:{}", ctx.getQueryHandleString());
allQueries.put(ctx.getQueryHandle(), ctx);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
log.info("Returning handle " + ctx.getQueryHandle().getHandleId());
@@ -1596,7 +1718,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("UpdateQueryConf:" + sessionHandle.toString() + " query: " + queryHandle);
acquire(sessionHandle);
QueryContext ctx = getQueryContext(sessionHandle, queryHandle);
- if (ctx != null && ctx.getStatus().getStatus() == QUEUED) {
+ if (ctx != null && (ctx.queued())) {
ctx.updateConf(newconf.getProperties());
// TODO COnf changed event tobe raised
return true;
@@ -1642,6 +1764,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
QueryContext ctx = allQueries.get(queryHandle);
if (ctx == null) {
FinishedLensQuery query = lensServerDao.getQuery(queryHandle.toString());
+ log.info("FinishedLensQuery:{}", query);
if (query == null) {
throw new NotFoundException("Query not found " + queryHandle);
}
@@ -1748,7 +1871,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
QueryHandle handle = executeAsyncInternal(sessionHandle, ctx);
QueryHandleWithResultSet result = new QueryHandleWithResultSet(handle);
// getQueryContext calls updateStatus, which fires query events if there's a change in status
- while (getQueryContext(sessionHandle, handle).getStatus().getStatus().equals(QUEUED)) {
+
+ while (isQueued(sessionHandle, handle)) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
@@ -1785,6 +1909,15 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
return result;
}
+ private boolean isQueued(final LensSessionHandle sessionHandle, final QueryHandle handle)
+ throws LensException {
+ // getQueryContext calls updateStatus, which fires query events if there's a change in status
+ QueryContext query = getQueryContext(sessionHandle, handle);
+ synchronized (query) {
+ return query.queued();
+ }
+ }
+
/**
* The Class QueryCompletionListenerImpl.
*/
@@ -1914,24 +2047,26 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("CancelQuery: " + sessionHandle.toString() + " query:" + queryHandle);
acquire(sessionHandle);
QueryContext ctx = getQueryContext(sessionHandle, queryHandle);
- if (ctx.getStatus().finished()) {
- return false;
- }
+
synchronized (ctx) {
- if (ctx.getStatus().getStatus().equals(LAUNCHED)
- || ctx.getStatus().getStatus().equals(RUNNING)) {
+
+ if (ctx.finished()) {
+ return false;
+ }
+
+ if (ctx.launched() || ctx.running()) {
boolean ret = ctx.getSelectedDriver().cancelQuery(queryHandle);
if (!ret) {
return false;
}
- setCancelledStatus(ctx, "Query is cancelled");
- return true;
}
+
+ setCancelledStatus(ctx, "Query is cancelled");
+ return true;
}
} finally {
release(sessionHandle);
}
- return false;
}
/*
@@ -2238,15 +2373,20 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
// populate the query queues
+ final List<QueryContext> allRestoredQueuedQueries = new LinkedList<QueryContext>();
for (QueryContext ctx : allQueries.values()) {
switch (ctx.getStatus().getStatus()) {
case NEW:
case QUEUED:
- queuedQueries.add(ctx);
+ allRestoredQueuedQueries.add(ctx);
break;
case LAUNCHED:
case RUNNING:
- launchedQueries.add(ctx);
+ try {
+ launchedQueries.add(ctx);
+ } catch (final Exception e) {
+ log.error("Query not restored:QueryContext:{}", ctx, e);
+ }
break;
case SUCCESSFUL:
case FAILED:
@@ -2257,6 +2397,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
allQueries.remove(ctx.getQueryHandle());
}
}
+ queuedQueries.addAll(allRestoredQueuedQueries);
log.info("Recovered " + allQueries.size() + " queries");
}
}
@@ -2273,23 +2414,27 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
synchronized (drivers) {
out.writeInt(drivers.size());
for (LensDriver driver : drivers.values()) {
- out.writeUTF(driver.getClass().getName());
- driver.writeExternal(out);
+ synchronized (driver) {
+ out.writeUTF(driver.getClass().getName());
+ driver.writeExternal(out);
+ }
}
}
// persist allQueries
synchronized (allQueries) {
out.writeInt(allQueries.size());
for (QueryContext ctx : allQueries.values()) {
- out.writeObject(ctx);
- boolean isDriverAvailable = (ctx.getSelectedDriver() != null);
- out.writeBoolean(isDriverAvailable);
- if (isDriverAvailable) {
- out.writeUTF(ctx.getSelectedDriver().getClass().getName());
+ synchronized (ctx) {
+ out.writeObject(ctx);
+ boolean isDriverAvailable = (ctx.getSelectedDriver() != null);
+ out.writeBoolean(isDriverAvailable);
+ if (isDriverAvailable) {
+ out.writeUTF(ctx.getSelectedDriver().getClass().getName());
+ }
}
}
+ log.info("Persisted " + allQueries.size() + " queries");
}
- log.info("Persisted " + allQueries.size() + " queries");
}
/*
@@ -2390,7 +2535,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public long getRunningQueriesCount() {
- return launchedQueries.size();
+ return launchedQueries.getQueriesCount();
+ }
+
+ @Override
+ public long getWaitingQueriesCount() {
+ return waitingQueries.getQueriesCount();
}
@Override
@@ -2542,7 +2692,76 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
String command = "add " + res.getType().toLowerCase() + " " + uri;
driver.execute(createResourceQuery(command, sessionHandle, driver));
- log.info("Added resource to hive driver for session "
- + sessionIdentifier + " cmd: " + command);
+ log.info("Added resource to hive driver for session " + sessionIdentifier + " cmd: " + command);
+ }
+
+ private boolean removeFromLaunchedQueries(final QueryContext finishedQuery) {
+
+ /* Check javadoc of QueryExecutionServiceImpl#removalFromLaunchedQueriesLock for reason for existence
+ of this lock. */
+
+ log.debug("Acquiring lock in removeFromLaunchedQueries");
+ removalFromLaunchedQueriesLock.lock();
+ boolean modified = false;
+
+ try {
+ modified = this.launchedQueries.remove(finishedQuery);
+ } finally {
+ removalFromLaunchedQueriesLock.unlock();
+ }
+
+ log.debug("launchedQueries.remove(finishedQuery) has returned [{}] for finished query with query id:[{}]", modified,
+ finishedQuery.getQueryHandleString());
+ return modified;
+ }
+
+ /**
+ * Caller of this method must make sure that this method is called inside a synchronized(queryContext) block
+ * for a safe copy from queryContext to an instance of {@link FinishedLensQuery}
+ *
+ * @param queryContext
+ */
+ private void processWaitingQueriesAsync(final QueryContext queryContext) {
+
+ final FinishedLensQuery finishedLensQuery = new FinishedLensQuery(queryContext);
+
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ logSegregationContext.setLogSegragationAndQueryId(finishedLensQuery.getHandle());
+ processWaitingQueries(finishedLensQuery);
+ } catch (final Throwable e) {
+ log.error("Error in processing waiting queries", e);
+ }
+ }
+ };
+
+ exceptionSafeSubmit(this.waitingQueriesSelectionSvc, r);
+ }
+
+ private void exceptionSafeSubmit(final ExecutorService svc, final Runnable r) {
+ try {
+ svc.submit(r);
+ } catch (final Throwable e) {
+ log.debug("Could not submit runnable:{}", e);
+ }
+ }
+
+ private void processWaitingQueries(final FinishedLensQuery finishedQuery) {
+
+ Set<QueryContext> eligibleWaitingQueries = this.waitingQueriesSelector
+ .selectQueries(finishedQuery, this.waitingQueries);
+
+ if (eligibleWaitingQueries.isEmpty()) {
+ log.debug("No queries eligible to move out of waiting state.");
+ return;
+ }
+
+ waitingQueries.removeAll(eligibleWaitingQueries);
+ queuedQueries.addAll(eligibleWaitingQueries);
+ if (log.isDebugEnabled()) {
+ log.debug("Added {} queries to queued queries", eligibleWaitingQueries.size());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
new file mode 100644
index 0000000..e3505bb
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollection.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.map.MultiValueMap;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ *
+ * Implementation which creates multiple in memory views of queries existing in lens system and owns responsibility of
+ * keeping all the views consistent with each other.
+ *
+ * @see EstimatedQueryCollection
+ *
+ */
+@Slf4j
+public class DefaultEstimatedQueryCollection implements EstimatedQueryCollection {
+
+ private final QueryCollection queries;
+ private final MultiValueMap queriesByDriver = MultiValueMap.decorate(new HashMap(), LinkedHashSet.class);
+
+ public DefaultEstimatedQueryCollection(@NonNull final QueryCollection queries) {
+ this.queries = queries;
+ }
+
+ @Override
+ public Set<QueryContext> getQueries(final LensDriver driver) {
+ final Collection<QueryContext> driverQueries = getQueriesCollectionForDriver(driver);
+ return Sets.newLinkedHashSet(driverQueries);
+ }
+
+ @Override
+ public int getQueriesCount(final LensDriver driver) {
+ return getQueriesCollectionForDriver(driver).size();
+ }
+
+ @Override
+ public QueryCost getTotalQueryCost(final String user) {
+
+ return getTotalQueryCost(this.queries.getQueries(user));
+ }
+
+ /**
+ *
+ * @param query
+ * @return
+ * @throws IllegalStateException if selected driver or selected driver query cost is not set for the query
+ */
+ @Override
+ public boolean add(QueryContext query) {
+ checkState(query);
+ this.queriesByDriver.put(query.getSelectedDriver(), query);
+ return this.queries.add(query);
+ }
+
+ /**
+ *
+ * @param queries
+ * @throws IllegalStateException if selected driver or selected driver query cost is not set for any of the queries
+ */
+ @Override
+ public boolean addAll(Set<QueryContext> queries) {
+
+ boolean modified = false;
+ for (QueryContext query : queries) {
+ modified |= add(query);
+ }
+ return modified;
+ }
+
+ /**
+ *
+ * @param query
+ * @return
+ */
+ @Override
+ public boolean remove(QueryContext query) {
+ this.queriesByDriver.remove(query.getSelectedDriver());
+ return this.queries.remove(query);
+ }
+
+ @Override
+ public boolean removeAll(Set<QueryContext> queries) {
+
+ boolean modified = false;
+ for (QueryContext query : queries) {
+ modified |= remove(query);
+ }
+ return modified;
+ }
+
+ @Override
+ public Set<QueryContext> getQueries() {
+ return this.queries.getQueries();
+ }
+
+ @Override
+ public Set<QueryContext> getQueries(String user) {
+ return this.queries.getQueries(user);
+ }
+
+ @Override
+ public int getQueriesCount() {
+ return this.queries.getQueriesCount();
+ }
+
+ @VisibleForTesting
+ void checkState(final QueryContext query) {
+ Preconditions.checkState(query.getSelectedDriver() != null);
+ Preconditions.checkState(query.getSelectedDriverQueryCost() != null);
+ }
+
+ private Collection<QueryContext> getQueriesCollectionForDriver(final LensDriver driver) {
+
+ final Collection<QueryContext> driverQueries = queriesByDriver.getCollection(driver);
+ return driverQueries != null ? driverQueries : CollectionUtils.EMPTY_COLLECTION;
+ }
+
+ private QueryCost getTotalQueryCost(final Collection<QueryContext> queries) {
+
+ if (queries.isEmpty()) {
+ return new FactPartitionBasedQueryCost(0);
+ }
+
+ QueryContext query0 = Iterables.get(queries, 0);
+ QueryCost totalQueryCost = query0.getSelectedDriverQueryCost();
+
+ for (QueryContext query : queries) {
+ QueryCost queryCost = query.getSelectedDriverQueryCost();
+ totalQueryCost = totalQueryCost.add(queryCost);
+ }
+ log.debug("Total Query Cost:{}", totalQueryCost);
+ return totalQueryCost;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java
new file mode 100644
index 0000000..f9e7701
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/DefaultQueryCollection.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.collect;
+
+import java.util.*;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.map.MultiValueMap;
+
+import com.google.common.collect.Sets;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Implementation which creates multiple in memory views of queries existing in lens system and owns responsibility of
+ * keeping all the views consistent with each other. This implementation is not thread-safe.
+ *
+ * @see QueryCollection
+ */
+@NoArgsConstructor
+@Slf4j
+public class DefaultQueryCollection implements QueryCollection {
+
+ private final Set<QueryContext> queries = Sets.newLinkedHashSet();
+ private final MultiValueMap queriesByUser = MultiValueMap.decorate(new HashMap(), LinkedHashSet.class);
+
+ public DefaultQueryCollection(@NonNull final Set<QueryContext> queries) {
+ addAll(queries);
+ }
+
+ @Override
+ public boolean add(final QueryContext query) {
+
+ queriesByUser.put(query.getSubmittedUser(), query);
+ return queries.add(query);
+ }
+
+ @Override
+ public boolean addAll(final Set<QueryContext> queries) {
+
+ boolean modified = false;
+ for (QueryContext query : queries) {
+ modified |= add(query);
+ }
+ return modified;
+ }
+
+ @Override
+ public boolean remove(final QueryContext query) {
+ queriesByUser.remove(query.getSubmittedUser(), query);
+ return queries.remove(query);
+ }
+
+ @Override
+ public boolean removeAll(Set<QueryContext> queries) {
+
+ boolean modified = false;
+ for (QueryContext query : queries) {
+ modified |= remove(query);
+ }
+ return modified;
+ }
+
+ @Override
+ public Set<QueryContext> getQueries() {
+ return Sets.newHashSet(queries);
+ }
+
+ @Override
+ public Set<QueryContext> getQueries(final String user) {
+ final Collection<QueryContext> userQueries = getQueriesCollectionForUser(user);
+ return Sets.newLinkedHashSet(userQueries);
+ }
+
+ @Override
+ public int getQueriesCount() {
+ return queries.size();
+ }
+
+ private Collection<QueryContext> getQueriesCollectionForUser(final String user) {
+
+ final Collection<QueryContext> userQueries = queriesByUser.getCollection(user);
+ return userQueries != null ? userQueries : CollectionUtils.EMPTY_COLLECTION;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/EstimatedQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/EstimatedQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/EstimatedQueryCollection.java
new file mode 100644
index 0000000..cffea0d
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/EstimatedQueryCollection.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+
+/**
+ *
+ * @see EstimatedImmutableQueryCollection
+ * @see MutableQueryCollection
+ */
+public interface EstimatedQueryCollection extends EstimatedImmutableQueryCollection, MutableQueryCollection {
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java
new file mode 100644
index 0000000..fee4120
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelector.java
@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.collect;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import lombok.NonNull;
+
+/**
+ * Selects queries eligible by all {@link WaitingQueriesSelectionPolicy} to move them out of waiting state.
+ *
+ */
+public class IntersectingWaitingQueriesSelector implements WaitingQueriesSelector {
+
+ private final ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies;
+
+ public IntersectingWaitingQueriesSelector(
+ @NonNull final ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies) {
+ this.selectionPolicies = selectionPolicies;
+ }
+
+ /**
+ * Selects queries eligible by all {@link WaitingQueriesSelectionPolicy} to move them out of waiting state.
+ *
+ * @see WaitingQueriesSelector#selectQueries(FinishedLensQuery, EstimatedImmutableQueryCollection)
+ *
+ * @param finishedQuery
+ * @param waitingQueries
+ * @return
+ */
+ @Override
+ public Set<QueryContext> selectQueries(final FinishedLensQuery finishedQuery,
+ final EstimatedImmutableQueryCollection waitingQueries) {
+
+ Set<WaitingQueriesSelectionPolicy> allSelectionPolicies = prepareAllSelectionPolicies(finishedQuery);
+
+ List<Set<QueryContext>> candiateQueriesSets = getAllCandidateQueriesSets(finishedQuery, waitingQueries,
+ allSelectionPolicies);
+
+ return findCommonQueries(candiateQueriesSets);
+ }
+
+ @VisibleForTesting
+ Set<WaitingQueriesSelectionPolicy> prepareAllSelectionPolicies(final FinishedLensQuery finishedQuery) {
+
+ /* Get the selection policies of driver on which this query was run */
+ ImmutableSet<WaitingQueriesSelectionPolicy> driverSelectionPolicies = finishedQuery.getDriverSelectionPolicies();
+
+ return Sets.union(this.selectionPolicies, driverSelectionPolicies);
+ }
+
+ private List<Set<QueryContext>> getAllCandidateQueriesSets(
+ final FinishedLensQuery finishedQuery, final EstimatedImmutableQueryCollection waitingQueries,
+ final Set<WaitingQueriesSelectionPolicy> allSelectionPolicies) {
+
+ List<Set<QueryContext>> candidateQueriesSets = Lists.newLinkedList();
+
+ for (final WaitingQueriesSelectionPolicy selectionPolicy : allSelectionPolicies) {
+
+ Set<QueryContext> candiateQueries = selectionPolicy.selectQueries(finishedQuery, waitingQueries);
+ candidateQueriesSets.add(candiateQueries);
+ }
+ return candidateQueriesSets;
+ }
+
+ @VisibleForTesting
+ Set<QueryContext> findCommonQueries(final List<Set<QueryContext>> candiateQueriesSets) {
+
+ Set<QueryContext> commonQueries = Sets.newLinkedHashSet();
+
+ if (!candiateQueriesSets.isEmpty()) {
+ commonQueries = Iterables.get(candiateQueriesSets, 0);
+
+ for (Set<QueryContext> candidateEligibleQueries : candiateQueriesSets) {
+ commonQueries.retainAll(candidateEligibleQueries);
+ }
+ }
+ return commonQueries;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/MutableQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/MutableQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/MutableQueryCollection.java
new file mode 100644
index 0000000..e6e777c
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/MutableQueryCollection.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ *
+ * {@link MutableQueryCollection} interface defines all mutable behaviours on queries existing in lens system.
+ *
+ * Responsibility of implementations of this interface is to make sure that if multiple views are created for the
+ * collection of queries, then they should remain consistent with each other after mutable behaviours are executed.
+ *
+ */
+
+public interface MutableQueryCollection {
+
+ /**
+ * add the given query to existing queries
+ *
+ * @param query
+ * @return
+ */
+ boolean add(final QueryContext query);
+
+ /**
+ *
+ * @param queries
+ * @return
+ */
+ boolean addAll(final Set<QueryContext> queries);
+
+ /**
+ * removes given query from existing queries
+ *
+ * @param query
+ * @return
+ */
+ boolean remove(final QueryContext query);
+
+ /**
+ *
+ * @param queries
+ */
+ boolean removeAll(final Set<QueryContext> queries);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/QueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/QueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/QueryCollection.java
new file mode 100644
index 0000000..0878ec9
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/QueryCollection.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.collect;
+
+import org.apache.lens.server.api.query.collect.ImmutableQueryCollection;
+
+/**
+ *
+ * {@link QueryCollection} interface defines all (immutable + mutable) behaviours on queries existing in lens system.
+ *
+ * @see ImmutableQueryCollection
+ * @see MutableQueryCollection
+ */
+public interface QueryCollection extends ImmutableQueryCollection, MutableQueryCollection {
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java
new file mode 100644
index 0000000..cdbd2ad
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeEstimatedQueryCollection.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import lombok.NonNull;
+
+/**
+ * Makes an implementation of {@link EstimatedQueryCollection} interface thread safe by wrapping all behaviours in
+ * synchronized method.
+ */
+public class ThreadSafeEstimatedQueryCollection implements EstimatedQueryCollection {
+
+ private final EstimatedQueryCollection estimatedQueries;
+
+ public ThreadSafeEstimatedQueryCollection(@NonNull final EstimatedQueryCollection estimatedQueries) {
+ this.estimatedQueries = estimatedQueries;
+ }
+
+ @Override
+ public synchronized Set<QueryContext> getQueries(LensDriver driver) {
+ return this.estimatedQueries.getQueries();
+ }
+
+ @Override
+ public synchronized int getQueriesCount(LensDriver driver) {
+ return this.estimatedQueries.getQueriesCount();
+ }
+
+ @Override
+ public synchronized QueryCost getTotalQueryCost(String user) {
+ return this.estimatedQueries.getTotalQueryCost(user);
+ }
+
+ @Override
+ public synchronized Set<QueryContext> getQueries() {
+ return this.estimatedQueries.getQueries();
+ }
+
+ @Override
+ public synchronized Set<QueryContext> getQueries(String user) {
+ return this.estimatedQueries.getQueries(user);
+ }
+
+ @Override
+ public synchronized int getQueriesCount() {
+ return this.estimatedQueries.getQueriesCount();
+ }
+
+ @Override
+ public synchronized boolean add(QueryContext query) {
+ return this.estimatedQueries.add(query);
+ }
+
+ @Override
+ public synchronized boolean addAll(Set<QueryContext> queries) {
+ return this.estimatedQueries.addAll(queries);
+ }
+
+ @Override
+ public synchronized boolean remove(QueryContext query) {
+ return this.estimatedQueries.remove(query);
+ }
+
+ @Override
+ public synchronized boolean removeAll(Set<QueryContext> queries) {
+ return this.estimatedQueries.removeAll(queries);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeQueryCollection.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeQueryCollection.java
new file mode 100644
index 0000000..7b43a38
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/ThreadSafeQueryCollection.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+import lombok.NonNull;
+
+/**
+ * Makes a {@link QueryCollection} implementation thread safe by synchronizing all behaviours.
+ *
+ * @see QueryCollection
+ *
+ */
+public class ThreadSafeQueryCollection implements QueryCollection {
+
+ private final QueryCollection queries;
+
+ public ThreadSafeQueryCollection(@NonNull QueryCollection queries) {
+ this.queries = queries;
+ }
+
+ @Override
+ public synchronized boolean add(QueryContext query) {
+ return this.queries.add(query);
+ }
+
+ @Override
+ public synchronized boolean addAll(Set<QueryContext> queries) {
+ return this.queries.addAll(queries);
+ }
+
+ @Override
+ public synchronized boolean remove(QueryContext query) {
+ return this.queries.remove(query);
+ }
+
+ @Override
+ public synchronized boolean removeAll(Set<QueryContext> queries) {
+ return this.queries.removeAll(queries);
+ }
+
+ @Override
+ public synchronized Set<QueryContext> getQueries() {
+ return this.queries.getQueries();
+ }
+
+ @Override
+ public synchronized Set<QueryContext> getQueries(final String user) {
+ return this.queries.getQueries(user);
+ }
+
+ @Override
+ public synchronized int getQueriesCount() {
+ return this.queries.getQueriesCount();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicy.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicy.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicy.java
new file mode 100644
index 0000000..1c29ca4
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicy.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+
+public class UserSpecificWaitingQueriesSelectionPolicy implements WaitingQueriesSelectionPolicy {
+
+
+ /**
+ *
+ * @param finishedQuery
+ * @param waitingQueries current waiting queries
+ * @return
+ */
+ @Override
+ public Set<QueryContext> selectQueries(final FinishedLensQuery finishedQuery,
+ final EstimatedImmutableQueryCollection waitingQueries) {
+
+ String user = finishedQuery.getSubmitter();
+ return waitingQueries.getQueries(user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyFactory.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyFactory.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyFactory.java
new file mode 100644
index 0000000..185b70b
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class UserSpecificWaitingQueriesSelectionPolicyFactory
+ implements ConfigBasedObjectCreationFactory<WaitingQueriesSelectionPolicy> {
+
+ @Override
+ public WaitingQueriesSelectionPolicy create(Configuration conf) {
+ return new UserSpecificWaitingQueriesSelectionPolicy();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/collect/WaitingQueriesSelector.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/collect/WaitingQueriesSelector.java b/lens-server/src/main/java/org/apache/lens/server/query/collect/WaitingQueriesSelector.java
new file mode 100644
index 0000000..d51c192
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/collect/WaitingQueriesSelector.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+
+/**
+ * Selects a subset of waiting queries eligible to move out of waiting state.
+ *
+ */
+public interface WaitingQueriesSelector {
+
+ /**
+ *
+ * @param finishedQuery
+ * @param waitingQueries
+ * @return Set of waiting queries eligible to move out of waiting state. Empty set is returned when no query is
+ * eligible. null is never returned. Multiple iterations over the returned set are guaranteed to be in the
+ * same order.
+ */
+ Set<QueryContext> selectQueries(
+ final FinishedLensQuery finishedQuery, final EstimatedImmutableQueryCollection waitingQueries);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java b/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java
new file mode 100644
index 0000000..2decf42
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.constraint;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import lombok.NonNull;
+
+/**
+ *
+ * This {@link QueryLaunchingConstraintsChecker} enforces that a candidate query will be allowed to launch only if
+ * all {@link QueryLaunchingConstraint}s of lens server and all {@link QueryLaunchingConstraint}s of driver selected
+ * for query allow the query to be launched.
+ *
+ */
+public class DefaultQueryLaunchingConstraintsChecker implements QueryLaunchingConstraintsChecker {
+
+ private final ImmutableSet<QueryLaunchingConstraint> lensQueryConstraints;
+
+ public DefaultQueryLaunchingConstraintsChecker(
+ @NonNull final ImmutableSet<QueryLaunchingConstraint> lensQueryConstraints) {
+ this.lensQueryConstraints = lensQueryConstraints;
+ }
+
+ @Override
+ public boolean canLaunch(final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries) {
+
+ Set<QueryLaunchingConstraint> allConstraints = prepareAllConstraints(candidateQuery);
+
+ for (QueryLaunchingConstraint queryConstraint : allConstraints) {
+ if (!queryConstraint.allowsLaunchOf(candidateQuery, launchedQueries)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @VisibleForTesting
+ Set<QueryLaunchingConstraint> prepareAllConstraints(final QueryContext candidateQuery) {
+
+ ImmutableSet<QueryLaunchingConstraint> driverConstraints = candidateQuery.getSelectedDriverQueryConstraints();
+ return Sets.union(this.lensQueryConstraints, driverConstraints);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/constraint/QueryLaunchingConstraintsChecker.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/constraint/QueryLaunchingConstraintsChecker.java b/lens-server/src/main/java/org/apache/lens/server/query/constraint/QueryLaunchingConstraintsChecker.java
new file mode 100644
index 0000000..a95ab48
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/constraint/QueryLaunchingConstraintsChecker.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.constraint;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+
+/**
+ *
+ * Checks whether the candidate query can be allowed to launch in the current state of launched queries.
+ *
+ */
+public interface QueryLaunchingConstraintsChecker {
+
+ boolean canLaunch(final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java b/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java
new file mode 100644
index 0000000..0a8d4c3
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.constraint;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import com.google.common.base.Optional;
+import lombok.EqualsAndHashCode;
+import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@EqualsAndHashCode
+public class TotalQueryCostCeilingConstraint implements QueryLaunchingConstraint {
+
+ /**
+ * Per user total query cost ceiling for launching a new query.
+ */
+ private final Optional<QueryCost> totalQueryCostCeilingPerUser;
+
+ public TotalQueryCostCeilingConstraint(@NonNull final Optional<QueryCost> totalQueryCostCeilingPerUser) {
+ this.totalQueryCostCeilingPerUser = totalQueryCostCeilingPerUser;
+ }
+
+ /**
+ *
+ * This constraint allows a query to be launched by the user,
+ *
+ * if total query cost of launched queries of
+ * the user is less than or equal to the total query cost ceiling per user
+ *
+ * OR
+ *
+ * the total query cost ceiling per user is not present.
+ *
+ * @param candidateQuery The query which is the next candidate to be launched.
+ * @param launchedQueries Current launched queries
+ * @return
+ */
+ @Override
+ public boolean allowsLaunchOf(
+ final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries) {
+
+ if (!totalQueryCostCeilingPerUser.isPresent()) {
+ return true;
+ }
+
+ final String currentUser = candidateQuery.getSubmittedUser();
+ QueryCost totalQueryCostForCurrentUser = launchedQueries.getTotalQueryCost(currentUser);
+
+ boolean canLaunch = (totalQueryCostForCurrentUser.compareTo(totalQueryCostCeilingPerUser.get()) <= 0);
+ log.debug("canLaunch:{}", canLaunch);
+ return canLaunch;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactory.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactory.java b/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactory.java
new file mode 100644
index 0000000..03ba024
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.constraint;
+
+import static org.apache.lens.server.api.LensConfConstants.TOTAL_QUERY_COST_CEILING_PER_USER_KEY;
+
+import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.base.Optional;
+
+public class TotalQueryCostCeilingConstraintFactory
+ implements ConfigBasedObjectCreationFactory<QueryLaunchingConstraint> {
+
+ @Override
+ public QueryLaunchingConstraint create(Configuration conf) {
+
+ final Double costCeiling = Double.parseDouble(conf.get(TOTAL_QUERY_COST_CEILING_PER_USER_KEY));
+ Optional<QueryCost> totalQueryCostCeilingPerUser = Optional.absent();
+
+ if (costCeiling >= 0.0) {
+ totalQueryCostCeilingPerUser = Optional.<QueryCost>of(new FactPartitionBasedQueryCost(costCeiling));
+ }
+ return new TotalQueryCostCeilingConstraint(totalQueryCostCeilingPerUser);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/util/FairPriorityBlockingQueue.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/util/FairPriorityBlockingQueue.java b/lens-server/src/main/java/org/apache/lens/server/util/FairPriorityBlockingQueue.java
new file mode 100644
index 0000000..1175f65
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/util/FairPriorityBlockingQueue.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.util;
+
+import java.util.Collection;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ *
+ * A priority blocking queue in which an element is not allowed to be removed from head while a bulk addAll operation
+ * is ongoing. This allows all elements added through addAll to get equal chance in scheduling.
+ *
+ */
+public class FairPriorityBlockingQueue<E> {
+
+ private final PriorityBlockingQueue<E> priorityBlockingQueue = new PriorityBlockingQueue<E>();
+ private final Object fairnessLock = new Object();
+ private final ReentrantLock conditionalWaitLock = new ReentrantLock();
+ private final Condition notEmpty = conditionalWaitLock.newCondition();
+
+ /**
+ *
+ * take is implemented by using poll and a fairnessLock to synchronize removal from head of queue with bulk addAll
+ * operation.
+ *
+ * @return
+ * @throws InterruptedException
+ */
+ public E take() throws InterruptedException {
+
+ E e;
+
+ for (;;) {
+
+ synchronized (fairnessLock) {
+ e = priorityBlockingQueue.poll();
+ }
+
+ if (e != null) {
+ return e;
+ }
+
+ waitUntilNotEmpty();
+ }
+ }
+
+ public boolean remove(final Object o) {
+ return priorityBlockingQueue.remove(o);
+ }
+
+ public void addAll(final Collection<? extends E> c) {
+ synchronized (fairnessLock) {
+ priorityBlockingQueue.addAll(c);
+ }
+ signalNotEmpty();
+ }
+
+ public boolean add(final E e) {
+
+ boolean modified = priorityBlockingQueue.add(e);
+ signalNotEmpty();
+ return modified;
+ }
+
+ public int size() {
+ return priorityBlockingQueue.size();
+ }
+
+ private void waitUntilNotEmpty() throws InterruptedException {
+
+ conditionalWaitLock.lock();
+ try {
+ notEmpty.await();
+ } finally {
+ conditionalWaitLock.unlock();
+ }
+ }
+
+ private void signalNotEmpty() {
+
+ conditionalWaitLock.lock();
+ try {
+ notEmpty.signal();
+ } finally {
+ conditionalWaitLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java b/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
index 0ee1c04..9c386a6 100644
--- a/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
+++ b/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
@@ -34,6 +34,7 @@ import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.ResultSetHandler;
import org.apache.hadoop.conf.Configuration;
+
/**
* The Class UtilityMethods.
*/
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index d26beb7..6af8d10 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -716,4 +716,41 @@
<value>3600</value>
<description>Interval at which lens session expiry service runs</description>
</property>
+
+ <property>
+ <name>lens.server.scheduling.queue.poll.interval.millisec</name>
+ <value>2000</value>
+ <description>The interval at which submission thread will poll scheduling queue to fetch the next query for
+ submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without
+ sleeping. The interval has to be given in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>lens.server.query.launching.constraint.factories</name>
+ <value>org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory</value>
+ <description>Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an
+ implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation
+ of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all
+ constraints pass.</description>
+ </property>
+
+ <property>
+ <name>lens.server.total.query.cost.ceiling.per.user</name>
+ <value>-1.0</value>
+ <description>A query submitted by user will be launched only if total query cost of all current
+ launched queries of user is less than or equal to total query cost ceiling defined by this property.
+ This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using
+ org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in
+ lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total
+ query cost of launched queries submitted by a user.</description>
+ </property>
+
+ <property>
+ <name>lens.server.waiting.queries.selection.policy.factories</name>
+ <value>org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory</value>
+ <description>Factories used to instantiate waiting queries selection policies. Every factory should
+ be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an
+ implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.</description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
index 75a5c0f..62e9954 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensTestUtil.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.server;
+import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import java.io.File;
@@ -50,7 +51,6 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
-import org.testng.Assert;
import lombok.extern.slf4j.Slf4j;
@@ -94,7 +94,6 @@ public final class LensTestUtil {
final QueryHandle handle = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
-
// wait till the query finishes
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
@@ -104,12 +103,13 @@ public final class LensTestUtil {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- assertTrue(ctx.getSubmissionTime() > 0);
- assertTrue(ctx.getLaunchTime() > 0);
- assertTrue(ctx.getDriverStartTime() > 0);
- assertTrue(ctx.getDriverFinishTime() > 0);
- assertTrue(ctx.getFinishTime() > 0);
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ final String debugHelpMsg = "Query Handle:"+ctx.getQueryHandleString();
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, debugHelpMsg);
+ assertTrue(ctx.getSubmissionTime() > 0, debugHelpMsg);
+ assertTrue(ctx.getLaunchTime() > 0, debugHelpMsg);
+ assertTrue(ctx.getDriverStartTime() > 0, debugHelpMsg);
+ assertTrue(ctx.getDriverFinishTime() > 0, debugHelpMsg);
+ assertTrue(ctx.getFinishTime() > 0, debugHelpMsg);
}
public static void createTable(String tblName, WebTarget parent, LensSessionHandle lensSessionId)
@@ -145,7 +145,7 @@ public final class LensTestUtil {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
}
/**
* Load data.
@@ -214,7 +214,7 @@ public final class LensTestUtil {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 7e486ae..4b9962a 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -253,7 +253,8 @@ public class TestQueryService extends LensJerseyTest {
assertEquals(ctx.getDriverFinishTime(), 0);
assertTrue(ctx.getFinishTime() > 0);
assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
- assertTrue(metricsSvc.getTotalFailedQueries() >= failedQueries + 1);
+ /* Commented and jira ticket raised for correction: https://issues.apache.org/jira/browse/LENS-685
+ assertTrue(metricsSvc.getTotalFailedQueries() >= failedQueries + 1);*/
}
// test with execute async post, get all queries, get query context,
@@ -689,16 +690,17 @@ public class TestQueryService extends LensJerseyTest {
while (!stat.finished()) {
lensQuery = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
stat = lensQuery.getStatus();
+ /* Commented and jira ticket raised for correction: https://issues.apache.org/jira/browse/LENS-683
switch (stat.getStatus()) {
case RUNNING:
assertEquals(metricsSvc.getRunningQueries(), runningQueries + 1,
- "Asserting queries for " + lensQuery.getQueryHandle());
+ "Asserting queries for " + ctx.getQueryHandle());
break;
case QUEUED:
assertEquals(metricsSvc.getQueuedQueries(), queuedQueries + 1);
break;
default: // nothing
- }
+ }*/
Thread.sleep(1000);
}
assertTrue(lensQuery.getSubmissionTime() > 0);
[24/50] [abbrv] incubator-lens git commit: LENS-676 : Pass ClI's
extra arguments to Bootstrap
Posted by sh...@apache.org.
LENS-676 : Pass ClI's extra arguments to Bootstrap
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c65fa3c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c65fa3c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c65fa3c2
Branch: refs/heads/LENS-581
Commit: c65fa3c2fb1a4f193a8bd8e6c68b05b1a75a8fa1
Parents: 7de74eb
Author: Rajat Khadelwal <pr...@apache.org>
Authored: Tue Jul 21 14:08:33 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Jul 21 14:08:33 2015 +0530
----------------------------------------------------------------------
tools/scripts/lens-cli.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c65fa3c2/tools/scripts/lens-cli.sh
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-cli.sh b/tools/scripts/lens-cli.sh
index 8b490e8..51bcdca 100644
--- a/tools/scripts/lens-cli.sh
+++ b/tools/scripts/lens-cli.sh
@@ -111,6 +111,6 @@ fi
# finally, invoke the appropriate command
exec ${JAVA_BIN} ${JAVA_PROPERTIES} -cp ${LENSCPPATH} \
- "$CLI_JAR" $*
+ "$CLI_JAR" $args
exit 0
[22/50] [abbrv] incubator-lens git commit: LENS-652 : Add Update
partition api and cli command
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 18b6c87..077396e 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -19,6 +19,8 @@
package org.apache.lens.cube.metadata;
+import static org.testng.Assert.assertEquals;
+
import java.util.*;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
@@ -33,10 +35,7 @@ import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.serdeConstants;
@@ -345,24 +344,24 @@ public class TestCubeMetastoreClient {
public void testStorage() throws Exception {
Storage hdfsStorage = new HDFSStorage(c1);
client.createStorage(hdfsStorage);
- Assert.assertEquals(1, client.getAllStorages().size());
+ assertEquals(1, client.getAllStorages().size());
Storage hdfsStorage2 = new HDFSStorage(c2);
client.createStorage(hdfsStorage2);
- Assert.assertEquals(2, client.getAllStorages().size());
+ assertEquals(2, client.getAllStorages().size());
Storage hdfsStorage3 = new HDFSStorage(c3);
client.createStorage(hdfsStorage3);
- Assert.assertEquals(3, client.getAllStorages().size());
+ assertEquals(3, client.getAllStorages().size());
Storage hdfsStorage4 = new HDFSStorage(c4);
client.createStorage(hdfsStorage4);
- Assert.assertEquals(4, client.getAllStorages().size());
+ assertEquals(4, client.getAllStorages().size());
- Assert.assertEquals(hdfsStorage, client.getStorage(c1));
- Assert.assertEquals(hdfsStorage2, client.getStorage(c2));
- Assert.assertEquals(hdfsStorage3, client.getStorage(c3));
- Assert.assertEquals(hdfsStorage4, client.getStorage(c4));
+ assertEquals(hdfsStorage, client.getStorage(c1));
+ assertEquals(hdfsStorage2, client.getStorage(c2));
+ assertEquals(hdfsStorage3, client.getStorage(c3));
+ assertEquals(hdfsStorage4, client.getStorage(c4));
}
@Test(priority = 1)
@@ -372,7 +371,7 @@ public class TestCubeMetastoreClient {
client.createDimension(stateDim);
client.createDimension(countryDim);
- Assert.assertEquals(client.getAllDimensions().size(), 4);
+ assertEquals(client.getAllDimensions().size(), 4);
Assert.assertTrue(client.tableExists(cityDim.getName()));
Assert.assertTrue(client.tableExists(stateDim.getName()));
Assert.assertTrue(client.tableExists(countryDim.getName()));
@@ -384,12 +383,12 @@ public class TestCubeMetastoreClient {
// validate expression in citydim
Dimension city = client.getDimension(cityDim.getName());
- Assert.assertEquals(dimExpressions.size(), city.getExpressions().size());
- Assert.assertEquals(dimExpressions.size(), city.getExpressionNames().size());
+ assertEquals(dimExpressions.size(), city.getExpressions().size());
+ assertEquals(dimExpressions.size(), city.getExpressionNames().size());
Assert.assertNotNull(city.getExpressionByName("stateAndCountry"));
Assert.assertNotNull(city.getExpressionByName("cityaddress"));
- Assert.assertEquals(city.getExpressionByName("cityaddress").getDescription(), "city with state and city and zip");
- Assert.assertEquals(city.getExpressionByName("cityaddress").getDisplayString(), "City Address");
+ assertEquals(city.getExpressionByName("cityaddress").getDescription(), "city with state and city and zip");
+ assertEquals(city.getExpressionByName("cityaddress").getDisplayString(), "City Address");
ExprColumn stateCountryExpr = new ExprColumn(new FieldSchema("stateAndCountry", "String",
"state and country together with hiphen as separator"), "State and Country",
@@ -412,24 +411,24 @@ public class TestCubeMetastoreClient {
city.removeExpression("cityAddress");
city = client.getDimension(cityDim.getName());
- Assert.assertEquals(1, city.getExpressions().size());
+ assertEquals(1, city.getExpressions().size());
ExprColumn stateAndCountryActual = city.getExpressionByName("stateAndCountry");
Assert.assertNotNull(stateAndCountryActual.getExpressions());
- Assert.assertEquals(2, stateAndCountryActual.getExpressions().size());
+ assertEquals(2, stateAndCountryActual.getExpressions().size());
Assert.assertTrue(stateAndCountryActual.getExpressions().contains("concat(statedim.name, \"-\", countrydim.name)"));
Assert.assertTrue(stateAndCountryActual.getExpressions()
.contains("concat(countrydim.name, \"-\", countrydim.name)"));
Assert.assertNotNull(city.getExpressionByName("stateAndCountry"));
- Assert.assertEquals(city.getExpressionByName("stateAndCountry").getExpr(),
+ assertEquals(city.getExpressionByName("stateAndCountry").getExpr(),
"concat(statedim.name, \"-\", countrydim.name)");
stateAndCountryActual.removeExpression("concat(countrydim.name, \"-\", countrydim.name)");
city.alterExpression(stateAndCountryActual);
client.alterDimension(city.getName(), city);
Dimension cityAltered = client.getDimension(city.getName());
- Assert.assertEquals(1, cityAltered.getExpressionByName("stateAndCountry").getExpressions().size());
+ assertEquals(1, cityAltered.getExpressionByName("stateAndCountry").getExpressions().size());
List<TableReference> chain = new ArrayList<TableReference>();
@@ -463,51 +462,51 @@ public class TestCubeMetastoreClient {
Dimension altered = client.getDimension(zipDim.getName());
- Assert.assertEquals(toAlter, altered);
+ assertEquals(toAlter, altered);
Assert.assertNotNull(altered.getAttributeByName("newZipDim"));
Assert.assertNotNull(altered.getAttributeByName("newRefDim"));
Assert.assertNotNull(altered.getAttributeByName("f2"));
Assert.assertNotNull(altered.getAttributeByName("stateid"));
Assert.assertNull(altered.getAttributeByName("f1"));
- Assert.assertEquals(1, altered.getExpressions().size());
+ assertEquals(1, altered.getExpressions().size());
Assert.assertNotNull(altered.getExpressionByName("formattedcode"));
- Assert.assertEquals(altered.getExpressionByName("formattedcode").getExpr(), "format_number(code, \"#,###,###\")");
+ assertEquals(altered.getExpressionByName("formattedcode").getExpr(), "format_number(code, \"#,###,###\")");
CubeDimAttribute newzipdim = altered.getAttributeByName("newZipDim");
Assert.assertTrue(newzipdim instanceof BaseDimAttribute);
- Assert.assertEquals(((BaseDimAttribute) newzipdim).getType(), "int");
- Assert.assertEquals((((BaseDimAttribute) newzipdim).getNumOfDistinctValues().get()), Long.valueOf(1000));
+ assertEquals(((BaseDimAttribute) newzipdim).getType(), "int");
+ assertEquals((((BaseDimAttribute) newzipdim).getNumOfDistinctValues().get()), Long.valueOf(1000));
CubeDimAttribute newrefdim = altered.getAttributeByName("newRefDim");
Assert.assertTrue(newrefdim instanceof ReferencedDimAtrribute);
- Assert.assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().size(), 1);
- Assert.assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().get(0).getDestTable(), cityDim.getName());
- Assert.assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().get(0).getDestColumn(), "id");
+ assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().size(), 1);
+ assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().get(0).getDestTable(), cityDim.getName());
+ assertEquals(((ReferencedDimAtrribute) newrefdim).getReferences().get(0).getDestColumn(), "id");
CubeDimAttribute f2 = altered.getAttributeByName("f2");
Assert.assertTrue(f2 instanceof BaseDimAttribute);
- Assert.assertEquals(((BaseDimAttribute) f2).getType(), "varchar");
+ assertEquals(((BaseDimAttribute) f2).getType(), "varchar");
CubeDimAttribute stateid = altered.getAttributeByName("stateid");
Assert.assertTrue(stateid instanceof ReferencedDimAtrribute);
- Assert.assertEquals(((ReferencedDimAtrribute) stateid).getReferences().size(), 1);
- Assert.assertEquals(((ReferencedDimAtrribute) stateid).getReferences().get(0).getDestTable(), stateDim.getName());
- Assert.assertEquals(((ReferencedDimAtrribute) stateid).getReferences().get(0).getDestColumn(), "id");
+ assertEquals(((ReferencedDimAtrribute) stateid).getReferences().size(), 1);
+ assertEquals(((ReferencedDimAtrribute) stateid).getReferences().get(0).getDestTable(), stateDim.getName());
+ assertEquals(((ReferencedDimAtrribute) stateid).getReferences().get(0).getDestColumn(), "id");
- Assert.assertEquals(altered.getProperties().get("alter.prop"), "altered");
+ assertEquals(altered.getProperties().get("alter.prop"), "altered");
- Assert.assertEquals(altered.getChainByName("stateFromZip"), zipState);
+ assertEquals(altered.getChainByName("stateFromZip"), zipState);
- Assert.assertEquals(altered.getJoinChains().size(), 1);
+ assertEquals(altered.getJoinChains().size(), 1);
JoinChain zipchain = altered.getChainByName("stateFromZip");
- Assert.assertEquals(zipchain.getDisplayString(), "Zip State");
- Assert.assertEquals(zipchain.getDescription(), "zip State desc");
- Assert.assertEquals(zipchain.getPaths().size(), 1);
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().size(), 4);
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(0).toString(), "zipdim.cityid");
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(1).toString(), "citydim.id");
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(2).toString(), "citydim.stateid");
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(3).toString(), "statedim.id");
+ assertEquals(zipchain.getDisplayString(), "Zip State");
+ assertEquals(zipchain.getDescription(), "zip State desc");
+ assertEquals(zipchain.getPaths().size(), 1);
+ assertEquals(zipchain.getPaths().get(0).getReferences().size(), 4);
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(0).toString(), "zipdim.cityid");
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(1).toString(), "citydim.id");
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(2).toString(), "citydim.stateid");
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(3).toString(), "statedim.id");
}
private void validateDim(Dimension udim, Set<CubeDimAttribute> attrs, String basedim, String referdim)
@@ -518,14 +517,14 @@ public class TestCubeMetastoreClient {
Dimension dim = new Dimension(dimTbl);
Assert.assertTrue(udim.equals(dim));
Assert.assertTrue(udim.equals(client.getDimension(udim.getName())));
- Assert.assertEquals(dim.getAttributes().size(), attrs.size());
+ assertEquals(dim.getAttributes().size(), attrs.size());
Assert.assertNotNull(dim.getAttributeByName(basedim));
Assert.assertTrue(dim.getAttributeByName(basedim) instanceof BaseDimAttribute);
if (referdim != null) {
Assert.assertNotNull(dim.getAttributeByName(referdim));
Assert.assertTrue(dim.getAttributeByName(referdim) instanceof ReferencedDimAtrribute);
}
- Assert.assertEquals(udim.getAttributeNames().size() + udim.getExpressionNames().size(), dim.getAllFieldNames()
+ assertEquals(udim.getAttributeNames().size() + udim.getExpressionNames().size(), dim.getAllFieldNames()
.size());
}
@@ -540,33 +539,33 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(cube.equals(cube2));
Assert.assertFalse(cube2.isDerivedCube());
Assert.assertTrue(cube2.getTimedDimensions().isEmpty());
- Assert.assertEquals(cubeMeasures.size(), cube2.getMeasureNames().size());
+ assertEquals(cubeMeasures.size(), cube2.getMeasureNames().size());
// +8 is for hierarchical dimension
- Assert.assertEquals(cubeDimensions.size() + 8, cube2.getDimAttributeNames().size());
- Assert.assertEquals(cubeMeasures.size(), cube2.getMeasures().size());
- Assert.assertEquals(cubeExpressions.size(), cube2.getExpressions().size());
- Assert.assertEquals(cubeExpressions.size(), cube2.getExpressionNames().size());
- Assert.assertEquals(cubeDimensions.size(), cube2.getDimAttributes().size());
- Assert.assertEquals(cubeDimensions.size() + 8 + cubeMeasures.size() + cubeExpressions.size(), cube2
+ assertEquals(cubeDimensions.size() + 8, cube2.getDimAttributeNames().size());
+ assertEquals(cubeMeasures.size(), cube2.getMeasures().size());
+ assertEquals(cubeExpressions.size(), cube2.getExpressions().size());
+ assertEquals(cubeExpressions.size(), cube2.getExpressionNames().size());
+ assertEquals(cubeDimensions.size(), cube2.getDimAttributes().size());
+ assertEquals(cubeDimensions.size() + 8 + cubeMeasures.size() + cubeExpressions.size(), cube2
.getAllFieldNames().size());
Assert.assertNotNull(cube2.getMeasureByName("msr4"));
- Assert.assertEquals(cube2.getMeasureByName("msr4").getDescription(), "fourth measure");
- Assert.assertEquals(cube2.getMeasureByName("msr4").getDisplayString(), "Measure4");
+ assertEquals(cube2.getMeasureByName("msr4").getDescription(), "fourth measure");
+ assertEquals(cube2.getMeasureByName("msr4").getDisplayString(), "Measure4");
Assert.assertNotNull(cube2.getDimAttributeByName("location"));
- Assert.assertEquals(cube2.getDimAttributeByName("location").getDescription(), "location hierarchy");
+ assertEquals(cube2.getDimAttributeByName("location").getDescription(), "location hierarchy");
Assert.assertNotNull(cube2.getDimAttributeByName("dim1"));
- Assert.assertEquals(cube2.getDimAttributeByName("dim1").getDescription(), "basedim");
+ assertEquals(cube2.getDimAttributeByName("dim1").getDescription(), "basedim");
Assert.assertNull(cube2.getDimAttributeByName("dim1").getDisplayString());
Assert.assertNotNull(cube2.getDimAttributeByName("dim2"));
- Assert.assertEquals(cube2.getDimAttributeByName("dim2").getDescription(), "ref dim");
- Assert.assertEquals(cube2.getDimAttributeByName("dim2").getDisplayString(), "Dim2 refer");
+ assertEquals(cube2.getDimAttributeByName("dim2").getDescription(), "ref dim");
+ assertEquals(cube2.getDimAttributeByName("dim2").getDisplayString(), "Dim2 refer");
Assert.assertNotNull(cube2.getExpressionByName("msr5"));
- Assert.assertEquals(cube2.getExpressionByName("msr5").getDescription(), "fifth measure");
- Assert.assertEquals(cube2.getExpressionByName("msr5").getDisplayString(), "Avg msr5");
+ assertEquals(cube2.getExpressionByName("msr5").getDescription(), "fifth measure");
+ assertEquals(cube2.getExpressionByName("msr5").getDisplayString(), "Avg msr5");
Assert.assertNotNull(cube2.getExpressionByName("booleancut"));
- Assert.assertEquals(cube2.getExpressionByName("booleancut").getDescription(), "a boolean expression");
- Assert.assertEquals(cube2.getExpressionByName("booleancut").getDisplayString(), "Boolean Cut");
- Assert.assertEquals(cube2.getExpressionByName("booleancut").getExpressions().size(), 2);
+ assertEquals(cube2.getExpressionByName("booleancut").getDescription(), "a boolean expression");
+ assertEquals(cube2.getExpressionByName("booleancut").getDisplayString(), "Boolean Cut");
+ assertEquals(cube2.getExpressionByName("booleancut").getExpressions().size(), 2);
// Validate expression can contain delimiter character
List<String> booleanCutExprs = new ArrayList<String>(cube2.getExpressionByName("booleancut").getExpressions());
Assert.assertTrue(booleanCutExprs.contains("dim1 | dim2 AND dim2 = 'XYZ'"));
@@ -575,32 +574,32 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(cube2.getJoinChainNames().contains("cityfromzip"));
Assert.assertTrue(cube2.getJoinChainNames().contains("city"));
Assert.assertFalse(cube2.getJoinChains().isEmpty());
- Assert.assertEquals(cube2.getJoinChains().size(), 2);
+ assertEquals(cube2.getJoinChains().size(), 2);
JoinChain zipchain = cube2.getChainByName("cityfromzip");
- Assert.assertEquals(zipchain.getDisplayString(), "Zip City");
- Assert.assertEquals(zipchain.getDescription(), "zip city desc");
- Assert.assertEquals(zipchain.getPaths().size(), 2);
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().size(), 4);
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(0).toString(), "testmetastorecube.zipcode");
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(1).toString(), "zipdim.zipcode");
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(2).toString(), "zipdim.cityid");
- Assert.assertEquals(zipchain.getPaths().get(0).getReferences().get(3).toString(), "citydim.id");
- Assert.assertEquals(zipchain.getPaths().get(1).getReferences().size(), 4);
- Assert.assertEquals(zipchain.getPaths().get(1).getReferences().get(0).toString(), "testmetastorecube.zipcode2");
- Assert.assertEquals(zipchain.getPaths().get(1).getReferences().get(1).toString(), "zipdim.zipcode");
- Assert.assertEquals(zipchain.getPaths().get(1).getReferences().get(2).toString(), "zipdim.cityid");
- Assert.assertEquals(zipchain.getPaths().get(1).getReferences().get(3).toString(), "citydim.id");
+ assertEquals(zipchain.getDisplayString(), "Zip City");
+ assertEquals(zipchain.getDescription(), "zip city desc");
+ assertEquals(zipchain.getPaths().size(), 2);
+ assertEquals(zipchain.getPaths().get(0).getReferences().size(), 4);
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(0).toString(), "testmetastorecube.zipcode");
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(1).toString(), "zipdim.zipcode");
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(2).toString(), "zipdim.cityid");
+ assertEquals(zipchain.getPaths().get(0).getReferences().get(3).toString(), "citydim.id");
+ assertEquals(zipchain.getPaths().get(1).getReferences().size(), 4);
+ assertEquals(zipchain.getPaths().get(1).getReferences().get(0).toString(), "testmetastorecube.zipcode2");
+ assertEquals(zipchain.getPaths().get(1).getReferences().get(1).toString(), "zipdim.zipcode");
+ assertEquals(zipchain.getPaths().get(1).getReferences().get(2).toString(), "zipdim.cityid");
+ assertEquals(zipchain.getPaths().get(1).getReferences().get(3).toString(), "citydim.id");
JoinChain citychain = cube2.getChainByName("city");
- Assert.assertEquals(citychain.getDisplayString(), "Cube City");
- Assert.assertEquals(citychain.getDescription(), "cube city desc");
- Assert.assertEquals(citychain.getPaths().size(), 1);
- Assert.assertEquals(citychain.getPaths().get(0).getReferences().size(), 2);
- Assert.assertEquals(citychain.getPaths().get(0).getReferences().get(0).toString(), "testmetastorecube.cityid");
- Assert.assertEquals(citychain.getPaths().get(0).getReferences().get(1).toString(), "citydim.id");
+ assertEquals(citychain.getDisplayString(), "Cube City");
+ assertEquals(citychain.getDescription(), "cube city desc");
+ assertEquals(citychain.getPaths().size(), 1);
+ assertEquals(citychain.getPaths().get(0).getReferences().size(), 2);
+ assertEquals(citychain.getPaths().get(0).getReferences().get(0).toString(), "testmetastorecube.cityid");
+ assertEquals(citychain.getPaths().get(0).getReferences().get(1).toString(), "citydim.id");
Assert.assertNotNull(cube2.getDimAttributeByName("zipcityname"));
- Assert.assertEquals(((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname")).getChainName(),
+ assertEquals(((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname")).getChainName(),
"cityfromzip");
- Assert.assertEquals(((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname")).getRefColumn(),
+ assertEquals(((ReferencedDimAtrribute) cube2.getDimAttributeByName("zipcityname")).getRefColumn(),
"name");
client.createDerivedCube(CUBE_NAME, DERIVED_CUBE_NAME, measures, dimensions, new HashMap<String, String>(), 0L);
@@ -611,10 +610,10 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(derivedCube.equals(dcube2));
Assert.assertTrue(dcube2.isDerivedCube());
Assert.assertTrue(dcube2.getTimedDimensions().isEmpty());
- Assert.assertEquals(measures.size(), dcube2.getMeasureNames().size());
- Assert.assertEquals(dimensions.size(), dcube2.getDimAttributeNames().size());
- Assert.assertEquals(measures.size(), dcube2.getMeasures().size());
- Assert.assertEquals(dimensions.size(), dcube2.getDimAttributes().size());
+ assertEquals(measures.size(), dcube2.getMeasureNames().size());
+ assertEquals(dimensions.size(), dcube2.getDimAttributeNames().size());
+ assertEquals(measures.size(), dcube2.getMeasures().size());
+ assertEquals(dimensions.size(), dcube2.getDimAttributes().size());
Assert.assertNotNull(dcube2.getMeasureByName("msr3"));
Assert.assertNull(dcube2.getMeasureByName("msr4"));
Assert.assertNull(dcube2.getDimAttributeByName("location"));
@@ -631,10 +630,10 @@ public class TestCubeMetastoreClient {
Assert.assertFalse(cubeWithProps.getTimedDimensions().isEmpty());
Assert.assertTrue(cubeWithProps.getTimedDimensions().contains("dt"));
Assert.assertTrue(cubeWithProps.getTimedDimensions().contains("mydate"));
- Assert.assertEquals(cubeMeasures.size(), cube2.getMeasureNames().size());
- Assert.assertEquals(cubeDimensions.size() + 8, cube2.getDimAttributeNames().size());
- Assert.assertEquals(cubeMeasures.size(), cube2.getMeasures().size());
- Assert.assertEquals(cubeDimensions.size(), cube2.getDimAttributes().size());
+ assertEquals(cubeMeasures.size(), cube2.getMeasureNames().size());
+ assertEquals(cubeDimensions.size() + 8, cube2.getDimAttributeNames().size());
+ assertEquals(cubeMeasures.size(), cube2.getMeasures().size());
+ assertEquals(cubeDimensions.size(), cube2.getDimAttributes().size());
Assert.assertNotNull(cube2.getMeasureByName("msr4"));
Assert.assertNotNull(cube2.getDimAttributeByName("location"));
Assert.assertFalse(cube2.allFieldsQueriable());
@@ -648,7 +647,7 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(derivedCubeWithProps.equals(dcube2));
Assert.assertTrue(dcube2.isDerivedCube());
Assert.assertNotNull(derivedCubeWithProps.getProperties().get("cube.custom.prop"));
- Assert.assertEquals(derivedCubeWithProps.getProperties().get("cube.custom.prop"), "myval");
+ assertEquals(derivedCubeWithProps.getProperties().get("cube.custom.prop"), "myval");
Assert.assertNull(dcube2.getMeasureByName("msr4"));
Assert.assertNotNull(dcube2.getMeasureByName("msr3"));
Assert.assertNull(dcube2.getDimAttributeByName("location"));
@@ -683,13 +682,13 @@ public class TestCubeMetastoreClient {
Assert.assertNotNull(toAlter.getMeasureByName("testAddMsr1"));
Assert.assertNotNull(toAlter.getMeasureByName("msr3"));
- Assert.assertEquals(toAlter.getMeasureByName("msr3").getDisplayString(), "Measure3Altered");
- Assert.assertEquals(toAlter.getMeasureByName("msr3").getDescription(), "third altered measure");
+ assertEquals(toAlter.getMeasureByName("msr3").getDisplayString(), "Measure3Altered");
+ assertEquals(toAlter.getMeasureByName("msr3").getDescription(), "third altered measure");
Assert.assertNull(toAlter.getMeasureByName("msr4"));
Assert.assertNotNull(toAlter.getDimAttributeByName("testAddDim1"));
- Assert.assertEquals(toAlter.getDimAttributeByName("testAddDim1").getDescription(), "dim to add");
+ assertEquals(toAlter.getDimAttributeByName("testAddDim1").getDescription(), "dim to add");
Assert.assertNotNull(toAlter.getDimAttributeByName("dim1"));
- Assert.assertEquals(toAlter.getDimAttributeByName("dim1").getDescription(), "basedim altered");
+ assertEquals(toAlter.getDimAttributeByName("dim1").getDescription(), "basedim altered");
Assert.assertNull(toAlter.getDimAttributeByName("location2"));
client.alterCube(cubeName, toAlter);
@@ -698,29 +697,29 @@ public class TestCubeMetastoreClient {
Cube altered = new Cube(alteredHiveTbl);
- Assert.assertEquals(toAlter, altered);
+ assertEquals(toAlter, altered);
Assert.assertNotNull(altered.getMeasureByName("testAddMsr1"));
CubeMeasure addedMsr = altered.getMeasureByName("testAddMsr1");
- Assert.assertEquals(addedMsr.getType(), "int");
+ assertEquals(addedMsr.getType(), "int");
Assert.assertNotNull(altered.getDimAttributeByName("testAddDim1"));
BaseDimAttribute addedDim = (BaseDimAttribute) altered.getDimAttributeByName("testAddDim1");
- Assert.assertEquals(addedDim.getType(), "string");
- Assert.assertEquals(addedDim.getDescription(), "dim to add");
+ assertEquals(addedDim.getType(), "string");
+ assertEquals(addedDim.getDescription(), "dim to add");
Assert.assertTrue(altered.getTimedDimensions().contains("zt"));
- Assert.assertEquals(altered.getMeasureByName("msr3").getDisplayString(), "Measure3Altered");
- Assert.assertEquals(altered.getMeasureByName("msr3").getDescription(), "third altered measure");
+ assertEquals(altered.getMeasureByName("msr3").getDisplayString(), "Measure3Altered");
+ assertEquals(altered.getMeasureByName("msr3").getDescription(), "third altered measure");
Assert.assertNotNull(altered.getDimAttributeByName("dim1"));
- Assert.assertEquals(altered.getDimAttributeByName("dim1").getDescription(), "basedim altered");
+ assertEquals(altered.getDimAttributeByName("dim1").getDescription(), "basedim altered");
Assert.assertNull(altered.getDimAttributeByName("location2"));
Assert.assertNull(altered.getChainByName("cityFromZip"));
- Assert.assertEquals(altered.getChainByName("city").getDescription(), "cube city desc modified");
+ assertEquals(altered.getChainByName("city").getDescription(), "cube city desc modified");
toAlter.alterMeasure(new ColumnMeasure(new FieldSchema("testAddMsr1", "double", "testAddMeasure")));
client.alterCube(cubeName, toAlter);
altered = new Cube(Hive.get(conf).getTable(cubeName));
addedMsr = altered.getMeasureByName("testaddmsr1");
Assert.assertNotNull(addedMsr);
- Assert.assertEquals(addedMsr.getType(), "double");
+ assertEquals(addedMsr.getType(), "double");
Assert.assertTrue(client.getAllFacts(altered).isEmpty());
}
@@ -747,13 +746,13 @@ public class TestCubeMetastoreClient {
DerivedCube altered = (DerivedCube) client.getCube(name);
- Assert.assertEquals(toAlter, altered);
+ assertEquals(toAlter, altered);
Assert.assertNotNull(altered.getMeasureByName("msr4"));
CubeMeasure addedMsr = altered.getMeasureByName("msr4");
- Assert.assertEquals(addedMsr.getType(), "bigint");
+ assertEquals(addedMsr.getType(), "bigint");
Assert.assertNotNull(altered.getDimAttributeByName("dim1StartTime"));
BaseDimAttribute addedDim = (BaseDimAttribute) altered.getDimAttributeByName("dim1StartTime");
- Assert.assertEquals(addedDim.getType(), "string");
+ assertEquals(addedDim.getType(), "string");
Assert.assertNotNull(addedDim.getStartTime());
client.dropCube(name);
@@ -796,15 +795,15 @@ public class TestCubeMetastoreClient {
Table cubeTbl = client.getHiveTable(factName);
Assert.assertTrue(client.isFactTable(cubeTbl));
Assert.assertTrue(client.isFactTableForCube(cubeTbl, CUBE_NAME));
- Assert.assertEquals(client.getAllFacts(client.getCube(CUBE_NAME)).get(0).getName(), factName.toLowerCase());
- Assert.assertEquals(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(),
+ assertEquals(client.getAllFacts(client.getCube(CUBE_NAME)).get(0).getName(), factName.toLowerCase());
+ assertEquals(client.getAllFacts(client.getCube(DERIVED_CUBE_NAME)).get(0).getName(),
factName.toLowerCase());
CubeFactTable cubeFact2 = new CubeFactTable(cubeTbl);
Assert.assertTrue(cubeFact.equals(cubeFact2));
// Assert for storage tables
for (String entry : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, entry);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry);
Assert.assertTrue(client.tableExists(storageTableName));
}
@@ -822,10 +821,10 @@ public class TestCubeMetastoreClient {
FieldSchema newcol = new FieldSchema("newcol", "int", "new col for part");
cubeFact.alterColumn(newcol);
client.alterCubeFactTable(cubeFact.getName(), cubeFact, storageTables);
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, c1);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c1);
+ assertEquals(client.getAllParts(storageTableName).size(), 1);
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(parts.size(), 0);
+ assertEquals(parts.size(), 0);
Map<String, Date> timeParts2 = new HashMap<String, Date>();
timeParts2.put(TestCubeMetastoreClient.getDatePartitionKey(), nowPlus1);
@@ -834,7 +833,7 @@ public class TestCubeMetastoreClient {
partSpec2.setInputFormat(SequenceFileInputFormat.class.getCanonicalName());
partSpec2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
client.addPartition(partSpec2, c1);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 2);
+ assertEquals(client.getAllParts(storageTableName).size(), 2);
Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts,
new HashMap<String, String>()));
Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts2,
@@ -842,10 +841,10 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(parts.size(), 0);
+ assertEquals(parts.size(), 0);
client.dropPartition(cubeFact.getName(), c1, timeParts2, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
+ assertEquals(client.getAllParts(storageTableName).size(), 1);
Assert.assertTrue(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts,
new HashMap<String, String>()));
Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts2,
@@ -853,10 +852,10 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(parts.size(), 0);
+ assertEquals(parts.size(), 0);
client.dropPartition(cubeFact.getName(), c1, timeParts, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts,
new HashMap<String, String>()));
Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts2,
@@ -941,9 +940,9 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(contains);
// alter storage table desc
- String c1TableName = MetastoreUtil.getFactStorageTableName(factName, c1);
+ String c1TableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c1);
Table c1Table = client.getTable(c1TableName);
- Assert.assertEquals(c1Table.getInputFormatClass().getCanonicalName(),
+ assertEquals(c1Table.getInputFormatClass().getCanonicalName(),
TextInputFormat.class.getCanonicalName());
s1 = new StorageTableDesc();
s1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName());
@@ -958,9 +957,9 @@ public class TestCubeMetastoreClient {
CubeFactTable altered2 = client.getCubeFact(factName);
Assert.assertTrue(client.tableExists(c1TableName));
Table alteredC1Table = client.getTable(c1TableName);
- Assert.assertEquals(alteredC1Table.getInputFormatClass().getCanonicalName(),
+ assertEquals(alteredC1Table.getInputFormatClass().getCanonicalName(),
SequenceFileInputFormat.class.getCanonicalName());
- Assert.assertEquals(alteredC1Table.getSerdeParam(serdeConstants.FIELD_DELIM), ":");
+ assertEquals(alteredC1Table.getSerdeParam(serdeConstants.FIELD_DELIM), ":");
boolean storageTableColsAltered = false;
for (FieldSchema column : alteredC1Table.getAllCols()) {
@@ -973,7 +972,7 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(altered2.getStorages().contains("C4"));
Assert.assertTrue(altered2.getUpdatePeriods().get("C4").equals(updates));
- String c4TableName = MetastoreUtil.getFactStorageTableName(factName, c4);
+ String c4TableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c4);
Assert.assertTrue(client.tableExists(c4TableName));
// add storage
@@ -981,10 +980,10 @@ public class TestCubeMetastoreClient {
CubeFactTable altered3 = client.getCubeFact(factName);
Assert.assertTrue(altered3.getStorages().contains("C3"));
Assert.assertTrue(altered3.getUpdatePeriods().get("C3").equals(updates));
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, c3);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c3);
Assert.assertTrue(client.tableExists(storageTableName));
client.dropStorageFromFact(factName, c2);
- storageTableName = MetastoreUtil.getFactStorageTableName(factName, c2);
+ storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, c2);
Assert.assertFalse(client.tableExists(storageTableName));
List<CubeFactTable> cubeFacts = client.getAllFacts(client.getCube(CUBE_NAME));
List<String> cubeFactNames = new ArrayList<String>();
@@ -999,8 +998,8 @@ public class TestCubeMetastoreClient {
}
Assert.assertTrue(cubeFactNames.contains(factName.toLowerCase()));
client.dropFact(factName, true);
- Assert.assertFalse(client.tableExists(MetastoreUtil.getFactStorageTableName(factName, c1)));
- Assert.assertFalse(client.tableExists(MetastoreUtil.getFactStorageTableName(factName, c3)));
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(factName, c1)));
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(factName, c3)));
Assert.assertFalse(client.tableExists(factName));
cubeFacts = client.getAllFacts(cube);
cubeFactNames = new ArrayList<String>();
@@ -1056,7 +1055,7 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String entry : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, entry);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry);
Assert.assertTrue(client.tableExists(storageTableName));
}
@@ -1075,16 +1074,16 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName()));
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- String storageTableName = MetastoreUtil.getFactStorageTableName(cubeFact.getName(), c1);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFact.getName(), c1);
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(parts.size(), 0);
+ assertEquals(parts.size(), 0);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
+ assertEquals(client.getAllParts(storageTableName).size(), 1);
parts = client.getPartitionsByFilter(storageTableName, testDtPart.getName() + "='latest'");
- Assert.assertEquals(parts.size(), 0);
+ assertEquals(parts.size(), 0);
client.dropPartition(cubeFact.getName(), c1, timeParts, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts,
new HashMap<String, String>()));
Assert.assertFalse(client.latestPartitionExists(cubeFact.getName(), c1,
@@ -1146,12 +1145,12 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String entry : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, entry);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry);
Assert.assertTrue(client.tableExists(storageTableName));
}
- String c1TableName = MetastoreUtil.getFactStorageTableName(cubeFact.getName(), c1);
- String c2TableName = MetastoreUtil.getFactStorageTableName(cubeFact.getName(), c2);
+ String c1TableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFact.getName(), c1);
+ String c2TableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFact.getName(), c2);
Table c1Table = client.getHiveTable(c1TableName);
Table c2Table = client.getHiveTable(c2TableName);
@@ -1206,11 +1205,11 @@ public class TestCubeMetastoreClient {
PartitionTimeline timeline2Temp = client.partitionTimelineCache.get(factName, c2, UpdatePeriod.HOURLY,
getDatePartitionKey());
- Assert.assertEquals(timeline1Temp.getClass(), EndsAndHolesPartitionTimeline.class);
- Assert.assertEquals(timeline2Temp.getClass(), StoreAllPartitionTimeline.class);
+ assertEquals(timeline1Temp.getClass(), EndsAndHolesPartitionTimeline.class);
+ assertEquals(timeline2Temp.getClass(), StoreAllPartitionTimeline.class);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 3);
- Assert.assertEquals(client.getAllParts(c2TableName).size(), 3);
+ assertEquals(client.getAllParts(c1TableName).size(), 3);
+ assertEquals(client.getAllParts(c2TableName).size(), 3);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
@@ -1244,8 +1243,8 @@ public class TestCubeMetastoreClient {
client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1);
client.addPartitions(Arrays.asList(partSpec4, partSpec5), c2);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 5);
- Assert.assertEquals(client.getAllParts(c2TableName).size(), 5);
+ assertEquals(client.getAllParts(c1TableName).size(), 5);
+ assertEquals(client.getAllParts(c2TableName).size(), 5);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
@@ -1268,8 +1267,8 @@ public class TestCubeMetastoreClient {
client.addPartition(partSpec6, c1);
client.addPartition(partSpec6, c2);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 6);
- Assert.assertEquals(client.getAllParts(c2TableName).size(), 6);
+ assertEquals(client.getAllParts(c1TableName).size(), 6);
+ assertEquals(client.getAllParts(c2TableName).size(), 6);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
@@ -1287,8 +1286,27 @@ public class TestCubeMetastoreClient {
client.addPartition(partSpec7, c1);
client.addPartition(partSpec7, c2);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 7);
- Assert.assertEquals(client.getAllParts(c2TableName).size(), 7);
+
+ List<Partition> c1Parts = client.getAllParts(c1TableName);
+ List<Partition> c2Parts = client.getAllParts(c2TableName);
+
+ assertEquals(c1Parts.size(), 7);
+ assertEquals(c2Parts.size(), 7);
+
+ // Test update partition
+ Random random = new Random();
+ for (Partition partition : c1Parts) {
+ partition.setLocation("blah");
+ partition.setBucketCount(random.nextInt());
+ client.updatePartition(factName, c1, partition);
+ }
+ assertSamePartitions(client.getAllParts(c1TableName), c1Parts);
+ for (Partition partition : c2Parts) {
+ partition.setLocation("blah");
+ partition.setBucketCount(random.nextInt());
+ }
+ client.updatePartitions(factName, c2, c2Parts);
+ assertSamePartitions(client.getAllParts(c2TableName), c2Parts);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
@@ -1298,23 +1316,23 @@ public class TestCubeMetastoreClient {
assertNoPartitionNamedLatest(c1TableName, partColNames);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
- Assert.assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get(
+ assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get(
MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
- Assert.assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get(
+ assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get(
MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
// alter tables and see timeline still exists
client.alterCubeFactTable(factName, cubeFact, storageTables);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
- Assert.assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get(
+ assertEquals(Hive.get(client.getConf()).getTable(c1TableName).getParameters().get(
MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
- Assert.assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get(
+ assertEquals(Hive.get(client.getConf()).getTable(c2TableName).getParameters().get(
MetastoreUtil.getPartitionTimelineCachePresenceKey()), "true");
client.dropPartition(cubeFact.getName(), c1, timeParts5, null, UpdatePeriod.HOURLY);
client.dropPartition(cubeFact.getName(), c2, timeParts5, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 6);
+ assertEquals(client.getAllParts(c1TableName).size(), 6);
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName()));
@@ -1330,7 +1348,7 @@ public class TestCubeMetastoreClient {
client.dropPartition(cubeFact.getName(), c1, timeParts7, null, UpdatePeriod.HOURLY);
client.dropPartition(cubeFact.getName(), c2, timeParts7, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 5);
+ assertEquals(client.getAllParts(c1TableName).size(), 5);
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName()));
@@ -1345,7 +1363,7 @@ public class TestCubeMetastoreClient {
client.dropPartition(cubeFact.getName(), c1, timeParts2, nonTimeSpec, UpdatePeriod.HOURLY);
client.dropPartition(cubeFact.getName(), c2, timeParts2, nonTimeSpec, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 4);
+ assertEquals(client.getAllParts(c1TableName).size(), 4);
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1, itPart.getName()));
@@ -1359,7 +1377,7 @@ public class TestCubeMetastoreClient {
client.dropPartition(cubeFact.getName(), c1, timeParts4, null, UpdatePeriod.HOURLY);
client.dropPartition(cubeFact.getName(), c2, timeParts4, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 3);
+ assertEquals(client.getAllParts(c1TableName).size(), 3);
assertNoPartitionNamedLatest(c1TableName, partColNames);
assertSameTimelines(factName, storages, UpdatePeriod.HOURLY, partColNames);
@@ -1388,14 +1406,27 @@ public class TestCubeMetastoreClient {
}
+ private void assertSamePartitions(List<Partition> parts1, List<Partition> parts2) {
+ assertEquals(parts1.size(), parts2.size());
+ for (int i = 0; i < parts1.size(); i++) {
+ assertEquals(parts1.get(i).toString(), parts2.get(i).toString(), "toString differs at element " + i);
+ assertEquals(parts1.get(i).getPath(), parts2.get(i).getPath(), "path differs at element " + i);
+ assertEquals(parts1.get(i).getLocation(), parts2.get(i).getLocation(), "location differs at element " + i);
+ assertEquals(parts1.get(i).getBucketCount(), parts2.get(i).getBucketCount(),
+ "bucket count differs at element " + i);
+ assertEquals(parts1.get(i).getCompleteName(), parts2.get(i).getCompleteName(),
+ "complete name differs at element " + i);
+ }
+ }
+
private void assertTimeline(EndsAndHolesPartitionTimeline endsAndHolesPartitionTimeline,
StoreAllPartitionTimeline storeAllPartitionTimeline, UpdatePeriod updatePeriod,
Date first, Date latest, Date... holes) throws LensException {
TimePartition firstPart = TimePartition.of(updatePeriod, first);
TimePartition latestPart = TimePartition.of(updatePeriod, latest);
- Assert.assertEquals(endsAndHolesPartitionTimeline.getFirst(), firstPart);
- Assert.assertEquals(endsAndHolesPartitionTimeline.getLatest(), TimePartition.of(updatePeriod, latest));
- Assert.assertEquals(endsAndHolesPartitionTimeline.getHoles().size(), holes.length);
+ assertEquals(endsAndHolesPartitionTimeline.getFirst(), firstPart);
+ assertEquals(endsAndHolesPartitionTimeline.getLatest(), TimePartition.of(updatePeriod, latest));
+ assertEquals(endsAndHolesPartitionTimeline.getHoles().size(), holes.length);
for (Date date : holes) {
Assert.assertTrue(endsAndHolesPartitionTimeline.getHoles().contains(TimePartition.of(updatePeriod, date)));
}
@@ -1407,7 +1438,7 @@ public class TestCubeMetastoreClient {
for (Date holeDate : holes) {
partitions.remove(TimePartition.of(updatePeriod, holeDate));
}
- Assert.assertEquals(storeAllPartitionTimeline.getAllPartitions(), partitions);
+ assertEquals(storeAllPartitionTimeline.getAllPartitions(), partitions);
}
private void assertSameTimelines(String factName, String[] storages, UpdatePeriod updatePeriod, String[] partCols)
@@ -1430,7 +1461,7 @@ public class TestCubeMetastoreClient {
Iterator<TimePartition> iter2 = timeline2.iterator();
while (iter1.hasNext()) {
Assert.assertTrue(iter2.hasNext());
- Assert.assertEquals(iter1.next(), iter2.next());
+ assertEquals(iter1.next(), iter2.next());
}
Assert.assertFalse(iter2.hasNext());
}
@@ -1449,7 +1480,7 @@ public class TestCubeMetastoreClient {
private void assertNoPartitionNamedLatest(String storageTableName, String... partCols) throws HiveException {
for (String p : partCols) {
- Assert.assertEquals(client.getPartitionsByFilter(storageTableName, p + "='latest'").size(), 0);
+ assertEquals(client.getPartitionsByFilter(storageTableName, p + "='latest'").size(), 0);
}
}
@@ -1538,7 +1569,7 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String entry : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, entry);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry);
Assert.assertTrue(client.tableExists(storageTableName));
}
@@ -1551,17 +1582,17 @@ public class TestCubeMetastoreClient {
new HashMap<String, String>()));
Assert.assertTrue(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- String storageTableName = MetastoreUtil.getFactStorageTableName(cubeFact.getName(), c1);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFact.getName(), c1);
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(parts.size(), 0);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
+ assertEquals(parts.size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 1);
client.dropPartition(cubeFact.getName(), c1, timeParts, null, UpdatePeriod.HOURLY);
Assert.assertFalse(client.factPartitionExists(cubeFact.getName(), c1, UpdatePeriod.HOURLY, timeParts,
new HashMap<String, String>()));
Assert.assertFalse(client.latestPartitionExists(cubeFact.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
}
@Test(priority = 2)
@@ -1616,7 +1647,7 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String entry : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factNameWithPart, entry);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factNameWithPart, entry);
Assert.assertTrue(client.tableExists(storageTableName));
}
@@ -1633,17 +1664,17 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(client.latestPartitionExists(cubeFactWithParts.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
Assert.assertFalse(client.latestPartitionExists(cubeFactWithParts.getName(), c1, factPartColumns.get(0).getName()));
- String storageTableName = MetastoreUtil.getFactStorageTableName(cubeFactWithParts.getName(), c1);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFactWithParts.getName(), c1);
+ assertEquals(client.getAllParts(storageTableName).size(), 1);
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(parts.size(), 0);
+ assertEquals(parts.size(), 0);
client.dropPartition(cubeFactWithParts.getName(), c1, timeParts, partSpec, UpdatePeriod.HOURLY);
Assert.assertFalse(client.factPartitionExists(cubeFactWithParts.getName(), c1, UpdatePeriod.HOURLY, timeParts,
partSpec));
Assert.assertFalse(client.latestPartitionExists(cubeFactWithParts.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
}
@Test(priority = 2)
@@ -1698,7 +1729,7 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String entry : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factNameWithPart, entry);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factNameWithPart, entry);
Assert.assertTrue(client.tableExists(storageTableName));
}
@@ -1721,8 +1752,8 @@ public class TestCubeMetastoreClient {
TestCubeMetastoreClient.getDatePartitionKey()));
Assert.assertTrue(client.latestPartitionExists(cubeFactWithParts.getName(), c1, testDtPart.getName()));
Assert.assertFalse(client.latestPartitionExists(cubeFactWithParts.getName(), c1, factPartColumns.get(0).getName()));
- String storageTableName = MetastoreUtil.getFactStorageTableName(cubeFactWithParts.getName(), c1);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFactWithParts.getName(), c1);
+ assertEquals(client.getAllParts(storageTableName).size(), 1);
assertNoPartitionNamedLatest(storageTableName, "dt", testDtPart.getName());
@@ -1732,7 +1763,7 @@ public class TestCubeMetastoreClient {
Assert.assertFalse(client.latestPartitionExists(cubeFactWithParts.getName(), c1, testDtPart.getName()));
Assert.assertFalse(client.latestPartitionExists(cubeFactWithParts.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
}
@Test(priority = 2)
@@ -1792,7 +1823,7 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String entry : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, entry);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, entry);
Assert.assertTrue(client.tableExists(storageTableName));
}
@@ -1808,8 +1839,8 @@ public class TestCubeMetastoreClient {
partSpec));
Assert.assertTrue(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- String storageTableName = MetastoreUtil.getFactStorageTableName(cubeFactWithTwoStorages.getName(), c1);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 1);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFactWithTwoStorages.getName(), c1);
+ assertEquals(client.getAllParts(storageTableName).size(), 1);
assertNoPartitionNamedLatest(storageTableName, "dt");
@@ -1820,8 +1851,8 @@ public class TestCubeMetastoreClient {
new HashMap<String, String>()));
Assert.assertTrue(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c2,
TestCubeMetastoreClient.getDatePartitionKey()));
- String storageTableName2 = MetastoreUtil.getFactStorageTableName(cubeFactWithTwoStorages.getName(), c2);
- Assert.assertEquals(client.getAllParts(storageTableName2).size(), 1);
+ String storageTableName2 = MetastoreUtil.getFactOrDimtableStorageTableName(cubeFactWithTwoStorages.getName(), c2);
+ assertEquals(client.getAllParts(storageTableName2).size(), 1);
assertNoPartitionNamedLatest(storageTableName2, "dt");
@@ -1830,14 +1861,14 @@ public class TestCubeMetastoreClient {
timeParts, partSpec));
Assert.assertFalse(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
client.dropPartition(cubeFactWithTwoStorages.getName(), c2, timeParts, null, UpdatePeriod.HOURLY);
Assert.assertFalse(client.factPartitionExists(cubeFactWithTwoStorages.getName(), c2, UpdatePeriod.HOURLY,
timeParts, new HashMap<String, String>()));
Assert.assertFalse(client.latestPartitionExists(cubeFactWithTwoStorages.getName(), c2,
TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName2).size(), 0);
+ assertEquals(client.getAllParts(storageTableName2).size(), 0);
}
@Test(priority = 2)
@@ -1897,20 +1928,19 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts));
Assert
.assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, TestCubeMetastoreClient.getDatePartitionKey()));
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, c1);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 2);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, c1);
+ assertEquals(client.getAllParts(storageTableName).size(), 2);
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(1, parts.size());
- Assert
- .assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName());
- Assert.assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
+ assertEquals(1, parts.size());
+ assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName());
+ assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
UpdatePeriod.HOURLY.format().format(now));
client.dropPartition(cubeDim.getName(), c1, timeParts, null, UpdatePeriod.HOURLY);
Assert.assertFalse(client.dimPartitionExists(cubeDim.getName(), c1, timeParts));
Assert.assertFalse(client.latestPartitionExists(cubeDim.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
}
@Test(priority = 2)
@@ -1962,14 +1992,14 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String storage : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, storage);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, storage);
Assert.assertTrue(client.tableExists(storageTableName));
}
FieldSchema newcol = new FieldSchema("newcol", "int", "new col for part");
// test partition
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, c1);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, c1);
Assert.assertFalse(client.dimTableLatestPartitionExists(storageTableName));
Map<String, Date> timeParts = new HashMap<String, Date>();
timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), now);
@@ -1978,14 +2008,13 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts));
Assert
.assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 2);
+ assertEquals(client.getAllParts(storageTableName).size(), 2);
Assert.assertTrue(client.dimTableLatestPartitionExists(storageTableName));
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(1, parts.size());
- Assert
- .assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName());
+ assertEquals(1, parts.size());
+ assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName());
Assert.assertFalse(parts.get(0).getCols().contains(newcol));
- Assert.assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
+ assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
UpdatePeriod.HOURLY.format().format(now));
// Partition with different schema
@@ -1999,17 +2028,17 @@ public class TestCubeMetastoreClient {
sPartSpec2.setInputFormat(SequenceFileInputFormat.class.getCanonicalName());
sPartSpec2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
client.addPartition(sPartSpec2, c1);
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 3);
+ assertEquals(client.getAllParts(storageTableName).size(), 3);
Assert.assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts));
Assert.assertTrue(client.dimPartitionExists(cubeDim.getName(), c1, timeParts2));
Assert
.assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, TestCubeMetastoreClient.getDatePartitionKey()));
parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(1, parts.size());
- Assert.assertEquals(SequenceFileInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass()
+ assertEquals(1, parts.size());
+ assertEquals(SequenceFileInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass()
.getCanonicalName());
Assert.assertTrue(parts.get(0).getCols().contains(newcol));
- Assert.assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
+ assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
UpdatePeriod.HOURLY.format().format(nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts2, null, UpdatePeriod.HOURLY);
@@ -2019,19 +2048,18 @@ public class TestCubeMetastoreClient {
.assertTrue(client.latestPartitionExists(cubeDim.getName(), c1, TestCubeMetastoreClient.getDatePartitionKey()));
Assert.assertTrue(client.dimTableLatestPartitionExists(storageTableName));
parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(1, parts.size());
- Assert
- .assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName());
- Assert.assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
+ assertEquals(1, parts.size());
+ assertEquals(TextInputFormat.class.getCanonicalName(), parts.get(0).getInputFormatClass().getCanonicalName());
+ assertEquals(parts.get(0).getParameters().get(MetastoreUtil.getLatestPartTimestampKey("dt")),
UpdatePeriod.HOURLY.format().format(now));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 2);
+ assertEquals(client.getAllParts(storageTableName).size(), 2);
client.dropPartition(cubeDim.getName(), c1, timeParts, null, UpdatePeriod.HOURLY);
Assert.assertFalse(client.dimPartitionExists(cubeDim.getName(), c1, timeParts));
Assert.assertFalse(client.dimPartitionExists(cubeDim.getName(), c1, timeParts2));
Assert.assertFalse(client.latestPartitionExists(cubeDim.getName(), c1,
TestCubeMetastoreClient.getDatePartitionKey()));
- Assert.assertEquals(client.getAllParts(storageTableName).size(), 0);
+ assertEquals(client.getAllParts(storageTableName).size(), 0);
Assert.assertFalse(client.dimTableLatestPartitionExists(storageTableName));
client.addPartition(sPartSpec2, c1);
@@ -2068,7 +2096,7 @@ public class TestCubeMetastoreClient {
client.createCubeDimensionTable(countryDim.getName(), dimName, dimColumns, 0L, storageNames, null, storageTables);
// test partition
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, c3);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, c3);
Assert.assertFalse(client.dimTableLatestPartitionExists(storageTableName));
Map<String, Date> expectedLatestValues = Maps.newHashMap();
Map<String, Date> timeParts = new HashMap<String, Date>();
@@ -2127,9 +2155,9 @@ public class TestCubeMetastoreClient {
private void assertLatestForRegions(String storageTableName, Map<String, Date> expectedLatestValues)
throws HiveException, LensException {
List<Partition> parts = client.getPartitionsByFilter(storageTableName, "dt='latest'");
- Assert.assertEquals(parts.size(), expectedLatestValues.size());
+ assertEquals(parts.size(), expectedLatestValues.size());
for (Partition part : parts) {
- Assert.assertEquals(MetastoreUtil.getLatestTimeStampOfDimtable(part, getDatePartitionKey()),
+ assertEquals(MetastoreUtil.getLatestTimeStampOfDimtable(part, getDatePartitionKey()),
TimePartition.of(UpdatePeriod.HOURLY, expectedLatestValues.get(part.getSpec().get("region"))).getDate());
}
}
@@ -2191,7 +2219,7 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String storage : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, storage);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, storage);
Assert.assertTrue(client.tableExists(storageTableName));
}
@@ -2217,10 +2245,10 @@ public class TestCubeMetastoreClient {
UpdatePeriod.HOURLY);
client.addPartitions(Arrays.asList(partSpec1, partSpec2, partSpec3), c1);
- String c1TableName = MetastoreUtil.getDimStorageTableName(cubeDim.getName(), c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
+ String c1TableName = MetastoreUtil.getFactOrDimtableStorageTableName(cubeDim.getName(), c1);
+ assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
+ assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, now, nowPlus1));
Map<String, Date> timeParts4 = new HashMap<String, Date>();
@@ -2240,8 +2268,8 @@ public class TestCubeMetastoreClient {
client.addPartitions(Arrays.asList(partSpec4, partSpec5), c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
+ assertEquals(client.getAllParts(c1TableName).size(), 10);
+ assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, nowPlus1, nowPlus1, nowPlus1));
Map<String, Date> timeParts6 = new HashMap<String, Date>();
timeParts6.put(TestCubeMetastoreClient.getDatePartitionKey(), nowMinus2);
@@ -2252,7 +2280,7 @@ public class TestCubeMetastoreClient {
client.addPartition(partSpec6, c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 11);
+ assertEquals(client.getAllParts(c1TableName).size(), 11);
Map<String, Date> timeParts7 = new HashMap<String, Date>();
@@ -2263,36 +2291,36 @@ public class TestCubeMetastoreClient {
UpdatePeriod.HOURLY);
client.addPartition(partSpec7, c1);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 12);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
+ assertEquals(client.getAllParts(c1TableName).size(), 12);
+ assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, nowPlus1, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts5, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 11);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
+ assertEquals(client.getAllParts(c1TableName).size(), 11);
+ assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts7, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 10);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
+ assertEquals(client.getAllParts(c1TableName).size(), 10);
+ assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, nowPlus1));
client.dropPartition(cubeDim.getName(), c1, timeParts2, nonTimeSpec, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 9);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
+ assertEquals(client.getAllParts(c1TableName).size(), 9);
+ assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, nowPlus1, now));
client.dropPartition(cubeDim.getName(), c1, timeParts4, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 8);
- Assert.assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
+ assertEquals(client.getAllParts(c1TableName).size(), 8);
+ assertEquals(getLatestValues(c1TableName, UpdatePeriod.HOURLY, partColNames, null),
toPartitionArray(UpdatePeriod.HOURLY, now, now, now));
client.dropPartition(cubeDim.getName(), c1, timeParts3, nonTimeSpec, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 5);
+ assertEquals(client.getAllParts(c1TableName).size(), 5);
client.dropPartition(cubeDim.getName(), c1, timeParts6, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 4);
+ assertEquals(client.getAllParts(c1TableName).size(), 4);
client.dropPartition(cubeDim.getName(), c1, timeParts1, null, UpdatePeriod.HOURLY);
- Assert.assertEquals(client.getAllParts(c1TableName).size(), 0);
+ assertEquals(client.getAllParts(c1TableName).size(), 0);
assertNoPartitionNamedLatest(c1TableName, partColNames);
}
@@ -2362,9 +2390,9 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(typeChanged);
// alter storage table desc
- String c1TableName = MetastoreUtil.getDimStorageTableName(dimTblName, c1);
+ String c1TableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, c1);
Table c1Table = client.getTable(c1TableName);
- Assert.assertEquals(c1Table.getInputFormatClass().getCanonicalName(),
+ assertEquals(c1Table.getInputFormatClass().getCanonicalName(),
TextInputFormat.class.getCanonicalName());
s1 = new StorageTableDesc();
s1.setInputFormat(SequenceFileInputFormat.class.getCanonicalName());
@@ -2378,7 +2406,7 @@ public class TestCubeMetastoreClient {
CubeDimensionTable altered2 = client.getDimensionTable(dimTblName);
Assert.assertTrue(client.tableExists(c1TableName));
Table alteredC1Table = client.getTable(c1TableName);
- Assert.assertEquals(alteredC1Table.getInputFormatClass().getCanonicalName(),
+ assertEquals(alteredC1Table.getInputFormatClass().getCanonicalName(),
SequenceFileInputFormat.class.getCanonicalName());
boolean storageTblColAltered = false;
for (FieldSchema column : alteredC1Table.getAllCols()) {
@@ -2388,10 +2416,10 @@ public class TestCubeMetastoreClient {
}
}
Assert.assertTrue(storageTblColAltered);
- String c4TableName = MetastoreUtil.getDimStorageTableName(dimTblName, c4);
+ String c4TableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, c4);
Assert.assertTrue(client.tableExists(c4TableName));
Table c4Table = client.getTable(c4TableName);
- Assert.assertEquals(c4Table.getInputFormatClass().getCanonicalName(),
+ assertEquals(c4Table.getInputFormatClass().getCanonicalName(),
SequenceFileInputFormat.class.getCanonicalName());
Assert.assertTrue(altered2.getStorages().contains("C4"));
Assert.assertFalse(altered2.hasStorageSnapshots("C4"));
@@ -2401,16 +2429,16 @@ public class TestCubeMetastoreClient {
s2.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
client.addStorage(dimTable, c2, null, s2);
client.addStorage(dimTable, c3, UpdatePeriod.DAILY, s1);
- Assert.assertTrue(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c2)));
- Assert.assertTrue(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c3)));
+ Assert.assertTrue(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, c2)));
+ Assert.assertTrue(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, c3)));
CubeDimensionTable altered3 = client.getDimensionTable(dimTblName);
Assert.assertFalse(altered3.hasStorageSnapshots("C2"));
Assert.assertTrue(altered3.hasStorageSnapshots("C3"));
client.dropStorageFromDim(dimTblName, "C1");
- Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c1)));
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, c1)));
client.dropDimensionTable(dimTblName, true);
- Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c2)));
- Assert.assertFalse(client.tableExists(MetastoreUtil.getDimStorageTableName(dimTblName, c3)));
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, c2)));
+ Assert.assertFalse(client.tableExists(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, c3)));
Assert.assertFalse(client.tableExists(dimTblName));
// alter storage tables
}
@@ -2458,7 +2486,7 @@ public class TestCubeMetastoreClient {
// Assert for storage tables
for (String storageName : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimName, storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, storageName);
Assert.assertTrue(client.tableExists(storageTableName));
Assert.assertTrue(!client.getHiveTable(storageTableName).isPartitioned());
}
@@ -2515,9 +2543,9 @@ public class TestCubeMetastoreClient {
Assert.assertTrue(cubeDim.equals(cubeDim2));
// Assert for storage tables
- String storageTableName1 = MetastoreUtil.getDimStorageTableName(dimName, c1);
+ String storageTableName1 = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, c1);
Assert.assertTrue(client.tableExists(storageTableName1));
- String storageTableName2 = MetastoreUtil.getDimStorageTableName(dimName, c2);
+ String storageTableName2 = MetastoreUtil.getFactOrDimtableStorageTableName(dimName, c2);
Assert.assertTrue(client.tableExists(storageTableName2));
Assert.assertTrue(!client.getHiveTable(storageTableName2).isPartitioned());
}
@@ -2526,8 +2554,8 @@ public class TestCubeMetastoreClient {
public void testCaching() throws HiveException, ParseException {
client = CubeMetastoreClient.getInstance(conf);
CubeMetastoreClient client2 = CubeMetastoreClient.getInstance(new HiveConf(TestCubeMetastoreClient.class));
- Assert.assertEquals(5, client.getAllCubes().size());
- Assert.assertEquals(5, client2.getAllCubes().size());
+ assertEquals(5, client.getAllCubes().size());
+ assertEquals(5, client2.getAllCubes().size());
defineCube("testcache1", "testcache2", "derived1", "derived2");
client.createCube("testcache1", cubeMeasures, cubeDimensions);
@@ -2536,18 +2564,18 @@ public class TestCubeMetastoreClient {
client.createDerivedCube("testcache2", "derived2", measures, dimensions, CUBE_PROPERTIES, 0L);
Assert.assertNotNull(client.getCube("testcache1"));
Assert.assertNotNull(client2.getCube("testcache1"));
- Assert.assertEquals(9, client.getAllCubes().size());
- Assert.assertEquals(9, client2.getAllCubes().size());
+ assertEquals(9, client.getAllCubes().size());
+ assertEquals(9, client2.getAllCubes().size());
client2 = CubeMetastoreClient.getInstance(conf);
- Assert.assertEquals(9, client.getAllCubes().size());
- Assert.assertEquals(9, client2.getAllCubes().size());
+ assertEquals(9, client.getAllCubes().size());
+ assertEquals(9, client2.getAllCubes().size());
conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, false);
client = CubeMetastoreClient.getInstance(conf);
client2 = CubeMetastoreClient.getInstance(conf);
- Assert.assertEquals(9, client.getAllCubes().size());
- Assert.assertEquals(9, client2.getAllCubes().size());
+ assertEquals(9, client.getAllCubes().size());
+ assertEquals(9, client2.getAllCubes().size());
defineCube("testcache3", "testcache4", "dervied3", "derived4");
client.createCube("testcache3", cubeMeasures, cubeDimensions);
client.createCube("testcache4", cubeMeasures, cubeDimensions, CUBE_PROPERTIES);
@@ -2555,8 +2583,8 @@ public class TestCubeMetastoreClient {
client.createDerivedCube("testcache4", "derived4", measures, dimensions, CUBE_PROPERTIES, 0L);
Assert.assertNotNull(client.getCube("testcache3"));
Assert.assertNotNull(client2.getCube("testcache3"));
- Assert.assertEquals(13, client.getAllCubes().size());
- Assert.assertEquals(13, client2.getAllCubes().size());
+ assertEquals(13, client.getAllCubes().size());
+ assertEquals(13, client2.getAllCubes().size());
conf.setBoolean(MetastoreConstants.METASTORE_ENABLE_CACHING, true);
client = CubeMetastoreClient.getInstance(conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index f939858..e0338f5 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -1119,10 +1119,10 @@ public class CubeTestSetup {
List<StoragePartitionDesc> storagePartitionDescs = Lists.newArrayList();
List<String> partitions = Lists.newArrayList();
StoreAllPartitionTimeline ttdStoreAll =
- new StoreAllPartitionTimeline(MetastoreUtil.getFactStorageTableName(fact.getName(), c4), HOURLY,
+ new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY,
"ttd");
StoreAllPartitionTimeline ttd2StoreAll =
- new StoreAllPartitionTimeline(MetastoreUtil.getFactStorageTableName(fact.getName(), c4), HOURLY,
+ new StoreAllPartitionTimeline(MetastoreUtil.getFactOrDimtableStorageTableName(fact.getName(), c4), HOURLY,
"ttd2");
while (!(temp.after(NOW))) {
Map<String, Date> timeParts = new HashMap<String, Date>();
@@ -1170,7 +1170,7 @@ public class CubeTestSetup {
assertNotNull(storageName);
assertNotNull(updatePeriod);
assertNotNull(timeDim);
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName);
List<PartitionTimeline> timelines = client.getTimelines(factName, storageName, updatePeriod.name(), timeDim);
assertEquals(timelines.size(), 1);
PartitionTimeline actualTimeline = timelines.get(0);
@@ -1178,14 +1178,14 @@ public class CubeTestSetup {
assertEquals(client.getTable(storageTableName).getParameters()
.get(MetastoreUtil.getPartitionTimelineStorageClassKey(updatePeriod,
timeDim)), expectedTimeline.getClass().getCanonicalName());
- expectedTimeline.init(client.getTable(MetastoreUtil.getFactStorageTableName(factName, storageName)));
+ expectedTimeline.init(client.getTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName)));
assertEquals(actualTimeline, expectedTimeline);
}
private void assertTimeline(CubeMetastoreClient client, String factName, String storageName,
UpdatePeriod updatePeriod, String timeDim, Class<? extends PartitionTimeline> partitionTimelineClass)
throws Exception {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName);
PartitionTimeline expectedTimeline = partitionTimelineClass.getConstructor(
String.class, UpdatePeriod.class, String.class)
.newInstance(storageTableName, updatePeriod, timeDim);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-driver-hive/src/main/resources/hivedriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml
index 85a3f89..300ce3f 100644
--- a/lens-driver-hive/src/main/resources/hivedriver-default.xml
+++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml
@@ -35,6 +35,14 @@
</property>
<property>
+ <name>lens.driver.hive.query.hook.class</name>
+ <value>org.apache.lens.server.api.driver.NoOpDriverQueryHook</value>
+ <description>The query hook class for hive driver. By default hook is No op. To add a hook, you should look
+ at the default implementation and from there it'll be easy to derive what value can be added through a new hook
+ </description>
+ </property>
+
+ <property>
<name>lens.driver.hive.hs2.connection.expiry.delay</name>
<value>600000</value>
<description>The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2</description>
@@ -133,22 +141,9 @@
user more control.
</description>
</property>
-
- <property>
- <name>lens.driver.hive.priority.partition.weight.monthly</name>
- <value>0.5</value>
- <description>Weight of monthly partition in cost calculation</description>
- </property>
-
- <property>
- <name>lens.driver.hive.priority.partition.weight.daily</name>
- <value>0.75</value>
- <description>Weight of monthly partition in cost calculation</description>
- </property>
-
<property>
- <name>lens.driver.hive.priority.partition.weight.hourly</name>
- <value>1.0</value>
- <description>Weight of monthly partition in cost calculation</description>
+ <name>lens.driver.hive.cost.calculator.class</name>
+ <value>org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator</value>
+ <description>Cost calculator class. By default calculating cost through fact partitions.</description>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index 6226521..bd31f82 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -355,7 +355,7 @@ public interface CubeMetastoreService {
* @throws LensException
*/
void addPartitionToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName,
- XPartition partition) throws LensException;
+ XPartition partition) throws LensException;
/**
* Add partitions to dimension table on a storage.
@@ -363,10 +363,11 @@ public interface CubeMetastoreService {
* @param sessionid The sessionid
* @param dimTblName The dimension table name
* @param storageName The storage name
- * @param partitions {@link org.apache.lens.api.metastore.XPartitionList} * @throws LensException
+ * @param partitions {@link XPartitionList}
+ * @throws LensException
*/
void addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName,
- XPartitionList partitions) throws LensException;
+ XPartitionList partitions) throws LensException;
/**
* Get fact table given by name
@@ -487,11 +488,11 @@ public interface CubeMetastoreService {
* @param sessionid The sessionid
* @param fact The fact table name
* @param storageName The storage name
- * @param partition {@link org.apache.lens.api.metastore.XPartition}
+ * @param partition {@link XPartition}
* @throws LensException
*/
void addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
- XPartition partition) throws LensException;
+ XPartition partition) throws LensException;
/**
* Add partitions to fact on a storage
@@ -499,11 +500,11 @@ public interface CubeMetastoreService {
* @param sessionid The sessionid
* @param fact The fact table name
* @param storageName The storage name
- * @param partitions {@link org.apache.lens.api.metastore.XPartitionList}
+ * @param partitions {@link XPartitionList}
* @throws LensException
*/
void addPartitionsToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
- XPartitionList partitions) throws LensException;
+ XPartitionList partitions) throws LensException;
/**
* Drop partition from storage with spec specified as comma separated string
@@ -558,4 +559,10 @@ public interface CubeMetastoreService {
String updatePeriod, String timeDimension) throws LensException, HiveException;
XJoinChains getAllJoinChains(LensSessionHandle sessionid, String table) throws LensException;
+
+ void updatePartition(LensSessionHandle sessionid, String tblName, String storageName,
+ XPartition partition) throws LensException;
+
+ void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName,
+ XPartitionList partitions) throws LensException;
}
[47/50] [abbrv] incubator-lens git commit: Revert "Lens-581 :
Rafactored all API to new api (Vikas Singh via sharad)"
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MetaStoreClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MetaStoreClient.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MetaStoreClient.java
deleted file mode 100644
index ed73fcf..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MetaStoreClient.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.dao;
-
-import java.util.Date;
-import java.util.List;
-
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.ml.api.*;
-import org.apache.lens.server.api.error.LensException;
-
-public interface MetaStoreClient {
- /**
- * @param name
- * @param dataTable
- * @param dataBase
- * @return
- * @throws LensException
- */
- DataSet createDataSet(String name, String dataTable, String dataBase) throws LensException;
-
- /**
- * Creates a data set
- *
- * @param name
- * @return
- */
- DataSet getDataSet(String name) throws LensException;
-
- /**
- * @param name
- * @param algo
- * @param algoSpec
- * @param features
- * @param label
- * @return same Id if
- * @throws LensException
- */
- Model createModel(String name, String algo, AlgoSpec algoSpec,
- List<Feature> features, Feature label) throws LensException;
-
- /**
- * Retrieves Model
- *
- * @param modelId
- * @return
- * @throws LensException If model not present or meta store error
- */
- Model getModel(String modelId) throws LensException;
-
- /**
- * creates model instance
- *
- * @param startTime
- * @param finishTime
- * @param status
- * @param lensSessionHandle
- * @param modelId
- * @param dataSet
- * @param path
- * @param evaluationId
- * @return
- * @throws LensException
- */
- ModelInstance createModelInstance(Date startTime, Date finishTime, Status status, LensSessionHandle lensSessionHandle,
- String modelId, String dataSet, String path, String evaluationId)
- throws LensException;
-
- /**
- * Updates the model instance
- *
- * @param modelInstance
- * @throws LensException If modelInstance not already present in DB or meta store error.
- */
- void updateModelInstance(ModelInstance modelInstance) throws LensException;
-
- /**
- * Return list of all ModelInstances in meta store having Status other than COMPLETED, FAILED, CANCELLED
- *
- * @return
- * @throws LensException On meta store error.
- */
- List<ModelInstance> getIncompleteModelInstances() throws LensException;
-
- /**
- * Return list of all Evaluation in meta store having Status other than COMPLETED, FAILED, CANCELLED
- *
- * @return
- * @throws LensException On meta store error.
- */
- List<Evaluation> getIncompleteEvaluations() throws LensException;
-
- /**
- * Return list of all Prediction in meta store having Status other than COMPLETED, FAILED, CANCELLED
- *
- * @return
- * @throws LensException On meta store error.
- */
- List<Prediction> getIncompletePredictions() throws LensException;
-
- /**
- * Returns all ModelInstances for modelId
- *
- * @param modelId
- * @return
- * @throws LensException If modelId is not present or meta store error.
- */
- List<ModelInstance> getModelInstances(String modelId) throws LensException;
-
- /**
- * Returns all Evaluations for modelInstanceId
- *
- * @param modelInstanceId
- * @return
- * @throws LensException
- */
- List<Evaluation> getEvaluations(String modelInstanceId) throws LensException;
-
- /**
- * Returns all Prediction for modelInstanceId
- *
- * @param modelInstanceId
- * @return
- * @throws LensException
- */
- List<Prediction> getPredictions(String modelInstanceId) throws LensException;
-
- /**
- * @param modelInstanceId
- * @return
- * @throws LensException If modelInstanceId is not present
- */
- ModelInstance getModelInstance(String modelInstanceId) throws LensException;
-
- /**
- * Creates Prediction
- *
- * @param startTime
- * @param finishTime
- * @param status
- * @param lensSessionHandle
- * @param modelInstanceId
- * @param inputDataSet
- * @param outputDataSet
- * @return
- * @throws LensException
- */
- Prediction createPrediction(Date startTime, Date finishTime, Status status, LensSessionHandle lensSessionHandle,
- String modelInstanceId, String inputDataSet, String outputDataSet) throws LensException;
-
- /**
- * gets Prediction
- *
- * @param predictionId
- * @return
- * @throws LensException If prediction Id is not present or meta store error
- */
- Prediction getPrediction(String predictionId) throws LensException;
-
- /**
- * Updates Evaluation
- *
- * @param evaluation
- * @throws LensException If evaluation not already present in DB or meta store error.
- */
- void updateEvaluation(Evaluation evaluation) throws LensException;
-
- /**
- * Creates Evaluation
- *
- * @param startTime
- * @param finishTime
- * @param status
- * @param lensSessionHandle
- * @param modelInstanceId
- * @param inputDataSetName
- * @return
- * @throws LensException
- */
- Evaluation createEvaluation(Date startTime, Date finishTime, Status status, LensSessionHandle lensSessionHandle,
- String
- modelInstanceId, String inputDataSetName) throws LensException;
-
- /**
- * gets evaluation
- *
- * @param evaluationId
- * @return
- * @throws LensException If evaluationId is not present in meta store.
- */
- Evaluation getEvaluation(String evaluationId) throws LensException;
-
- /**
- * Updates prediction
- *
- * @param prediction
- * @throws LensException If evaluation not already present in DB or meta store error.
- */
- void updatePrediction(Prediction prediction) throws LensException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/BatchPredictSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/BatchPredictSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/BatchPredictSpec.java
deleted file mode 100644
index 1c9a4ce..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/BatchPredictSpec.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.impl;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.lens.ml.api.Feature;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Table;
-
-import lombok.Getter;
-
-/**
- * BatchPredictSpec class. Contains table specification for input table for prediction in batch mode. Returns the
- * HIVE query which can be used to run the prediction job.
- */
-public class BatchPredictSpec {
-
- /**
- * The Constant LOG.
- */
- public static final Log LOG = LogFactory.getLog(BatchPredictSpec.class);
-
- /**
- * The db.
- */
- private String db;
-
- /**
- * The table containing input data.
- */
- private String inputTable;
-
- // TODO use partition condition
- /**
- * The partition filter.
- */
- private String partitionFilter;
-
- /**
- * The feature columns.
- */
- private List<Feature> featureColumns;
-
- /**
- * The output column.
- */
- private String outputColumn;
-
- /**
- * The output table.
- */
- private String outputTable;
-
- /**
- * The conf.
- */
- private transient HiveConf conf;
-
- /**
- * The algorithm.
- */
- private String algorithm;
-
- /**
- * The model id.
- */
- private String modelID;
-
- /**
- * The modelInstanceIds
- */
- private String modelInstanceId;
-
- @Getter
- private boolean outputTableExists;
-
- /**
- * A unique testId which is predictionId
- */
- @Getter
- private String testID;
-
- private HashMap<String, FieldSchema> columnNameToFieldSchema;
-
- /**
- * New builder.
- *
- * @return the table testing spec builder
- */
- public static TableTestingSpecBuilder newBuilder() {
- return new TableTestingSpecBuilder();
- }
-
- /**
- * Validate.
- *
- * @return true, if successful
- */
- public boolean validate() {
- List<FieldSchema> columns;
- try {
- Hive metastoreClient = Hive.get(conf);
- Table tbl = (db == null) ? metastoreClient.getTable(inputTable) : metastoreClient.getTable(db, inputTable);
- columns = tbl.getAllCols();
- columnNameToFieldSchema = new HashMap<String, FieldSchema>();
-
- for (FieldSchema fieldSchema : columns) {
- columnNameToFieldSchema.put(fieldSchema.getName(), fieldSchema);
- }
-
- // Check if output table exists
- Table outTbl = metastoreClient.getTable(db == null ? "default" : db, outputTable, false);
- outputTableExists = (outTbl != null);
- } catch (HiveException exc) {
- LOG.error("Error getting table info " + toString(), exc);
- return false;
- }
-
- // Check if labeled column and feature columns are contained in the table
- List<String> testTableColumns = new ArrayList<String>(columns.size());
- for (FieldSchema column : columns) {
- testTableColumns.add(column.getName());
- }
-
- List<String> inputColumnNames = new ArrayList();
- for (Feature feature : featureColumns) {
- inputColumnNames.add(feature.getDataColumn());
- }
-
- if (!testTableColumns.containsAll(inputColumnNames)) {
- LOG.info("Invalid feature columns: " + inputColumnNames + ". Actual columns in table:" + testTableColumns);
- return false;
- }
-
-
- if (StringUtils.isBlank(outputColumn)) {
- LOG.info("Output column is required");
- return false;
- }
-
- if (StringUtils.isBlank(outputTable)) {
- LOG.info("Output table is required");
- return false;
- }
- return true;
- }
-
- public String getTestQuery() {
- if (!validate()) {
- return null;
- }
-
- // We always insert a dynamic partition
- StringBuilder q = new StringBuilder("INSERT OVERWRITE TABLE " + outputTable + " PARTITION (part_testid='" + testID
- + "') SELECT ");
- List<String> featureNameList = new ArrayList();
- List<String> featureMapBuilder = new ArrayList();
- for (Feature feature : featureColumns) {
- featureNameList.add(feature.getDataColumn());
- featureMapBuilder.add("'" + feature.getDataColumn() + "'");
- featureMapBuilder.add(feature.getDataColumn());
- }
- String featureCols = StringUtils.join(featureNameList, ",");
- String featureMapString = StringUtils.join(featureMapBuilder, ",");
- q.append(featureCols).append(",").append("predict(").append("'").append(algorithm)
- .append("', ").append("'").append(modelID).append("', ").append("'").append(modelInstanceId).append("', ")
- .append(featureMapString).append(") ").append(outputColumn)
- .append(" FROM ").append(inputTable);
-
- return q.toString();
- }
-
- public String getCreateOutputTableQuery() {
- StringBuilder createTableQuery = new StringBuilder("CREATE TABLE IF NOT EXISTS ").append(outputTable).append("(");
- // Output table contains feature columns, label column, output column
- List<String> outputTableColumns = new ArrayList<String>();
- for (Feature featureCol : featureColumns) {
- outputTableColumns.add(featureCol.getDataColumn() + " "
- + columnNameToFieldSchema.get(featureCol.getDataColumn()).getType());
- }
-
-
- outputTableColumns.add(outputColumn + " string");
-
- createTableQuery.append(StringUtils.join(outputTableColumns, ", "));
-
- // Append partition column
- createTableQuery.append(") PARTITIONED BY (part_testid string)");
-
- return createTableQuery.toString();
- }
-
- /**
- * The Class TableTestingSpecBuilder.
- */
- public static class TableTestingSpecBuilder {
-
- /**
- * The spec.
- */
- private final BatchPredictSpec spec;
-
- /**
- * Instantiates a new table testing spec builder.
- */
- public TableTestingSpecBuilder() {
- spec = new BatchPredictSpec();
- }
-
- /**
- * Database.
- *
- * @param database the database
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder database(String database) {
- spec.db = database;
- return this;
- }
-
- /**
- * Set the input table
- *
- * @param table the table
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder inputTable(String table) {
- spec.inputTable = table;
- return this;
- }
-
- /**
- * Partition filter for input table
- *
- * @param partFilter the part filter
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder partitionFilter(String partFilter) {
- spec.partitionFilter = partFilter;
- return this;
- }
-
- /**
- * Feature columns.
- *
- * @param featureColumns the feature columns
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder featureColumns(List<Feature> featureColumns) {
- spec.featureColumns = featureColumns;
- return this;
- }
-
- /**
- * Output column.
- *
- * @param outputColumn the output column
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder outputColumn(String outputColumn) {
- spec.outputColumn = outputColumn;
- return this;
- }
-
- /**
- * Output table.
- *
- * @param table the table
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder outputTable(String table) {
- spec.outputTable = table;
- return this;
- }
-
- /**
- * Hive conf.
- *
- * @param conf the conf
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder hiveConf(HiveConf conf) {
- spec.conf = conf;
- return this;
- }
-
- /**
- * Algorithm.
- *
- * @param algorithm the algorithm
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder algorithm(String algorithm) {
- spec.algorithm = algorithm;
- return this;
- }
-
- /**
- * Model id.
- *
- * @param modelID the model id
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder modelID(String modelID) {
- spec.modelID = modelID;
- return this;
- }
-
- /**
- * modelInstanceID
- *
- * @param modelInstanceId
- * @return the table testing spec builder
- */
- public TableTestingSpecBuilder modelInstanceID(String modelInstanceId) {
- spec.modelInstanceId = modelInstanceId;
- return this;
- }
-
- /**
- * Builds the.
- *
- * @return the table testing spec
- */
- public BatchPredictSpec build() {
- return spec;
- }
-
- /**
- * Set the unique test id
- *
- * @param testID
- * @return
- */
- public TableTestingSpecBuilder testID(String testID) {
- spec.testID = testID;
- return this;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
index 5e41fc3..60a4008 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
@@ -19,11 +19,8 @@
package org.apache.lens.ml.impl;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.lens.ml.algo.api.TrainedModel;
-import org.apache.lens.ml.api.MLConfConstants;
+import org.apache.lens.ml.algo.api.MLModel;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -43,36 +40,31 @@ import org.apache.hadoop.mapred.JobConf;
/**
* Generic UDF to laod ML Models saved in HDFS and apply the model on list of columns passed as argument.
- * The feature list is expected to be key value pair. i.e. feature_name, feature_value
*/
@Description(name = "predict",
value = "_FUNC_(algorithm, modelID, features...) - Run prediction algorithm with given "
+ "algorithm name, model ID and input feature columns")
public final class HiveMLUDF extends GenericUDF {
+ private HiveMLUDF() {
+ }
- /**
- * The Constant LOG.
- */
+ /** The Constant UDF_NAME. */
+ public static final String UDF_NAME = "predict";
+
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(HiveMLUDF.class);
- /**
- * The conf.
- */
+
+ /** The conf. */
private JobConf conf;
- /**
- * The soi.
- */
+
+ /** The soi. */
private StringObjectInspector soi;
- /**
- * The doi.
- */
+
+ /** The doi. */
private LazyDoubleObjectInspector doi;
- /**
- * The model.
- */
- private TrainedModel model;
- private HiveMLUDF() {
- }
+ /** The model. */
+ private MLModel model;
/**
* Currently we only support double as the return value.
@@ -83,22 +75,12 @@ public final class HiveMLUDF extends GenericUDF {
*/
@Override
public ObjectInspector initialize(ObjectInspector[] objectInspectors) throws UDFArgumentException {
- // We require algo name, model id, modelInstance id and at least one feature name value pair
- String usage = "algo_name model_id, modelInstance_id [feature_name, feature_value]+ .";
- if (objectInspectors.length < 5) {
- throw new UDFArgumentLengthException(
- "Algo name, model ID, modelInstance ID and at least one feature name value pair should be passed to "
- + MLConfConstants.UDF_NAME + ". " + usage);
- }
-
- int numberOfFeatures = objectInspectors.length;
- if (numberOfFeatures % 2 == 0) {
- throw new UDFArgumentException(
- "The feature list should be even in length since it's key value pair. i.e. feature_name, feature_value" + ". "
- + usage);
+ // We require algo name, model id and at least one feature
+ if (objectInspectors.length < 3) {
+ throw new UDFArgumentLengthException("Algo name, model ID and at least one feature should be passed to "
+ + UDF_NAME);
}
-
- LOG.info(MLConfConstants.UDF_NAME + " initialized");
+ LOG.info(UDF_NAME + " initialized");
return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
}
@@ -112,36 +94,22 @@ public final class HiveMLUDF extends GenericUDF {
public Object evaluate(DeferredObject[] deferredObjects) throws HiveException {
String algorithm = soi.getPrimitiveJavaObject(deferredObjects[0].get());
String modelId = soi.getPrimitiveJavaObject(deferredObjects[1].get());
- String modelInstanceId = soi.getPrimitiveJavaObject(deferredObjects[2].get());
- Map<String, String> features = new HashMap();
-
- for (int i = 3; i < deferredObjects.length; i += 2) {
- try {
- String key = soi.getPrimitiveJavaObject(deferredObjects[i].get());
- LazyDouble lazyDouble = (LazyDouble) deferredObjects[i + 1].get();
- Double value = (lazyDouble == null) ? 0d : doi.get(lazyDouble);
- LOG.debug("key: " + key + ", value " + value);
- features.put(key, String.valueOf(value));
- } catch (Exception e) {
- LOG.error("Error Parsing feature pair");
- throw new HiveException(e.getMessage());
- }
+
+ Double[] features = new Double[deferredObjects.length - 2];
+ for (int i = 2; i < deferredObjects.length; i++) {
+ LazyDouble lazyDouble = (LazyDouble) deferredObjects[i].get();
+ features[i - 2] = (lazyDouble == null) ? 0d : doi.get(lazyDouble);
}
try {
if (model == null) {
- model = ModelLoader.loadModel(conf, algorithm, modelId, modelInstanceId);
+ model = ModelLoader.loadModel(conf, algorithm, modelId);
}
} catch (IOException e) {
throw new HiveException(e);
}
- try {
- Object object = model.predict(features);
- return object;
- } catch (Exception e) {
- throw new HiveException(e);
- }
+ return model.predict(features);
}
/*
@@ -151,7 +119,7 @@ public final class HiveMLUDF extends GenericUDF {
*/
@Override
public String getDisplayString(String[] strings) {
- return MLConfConstants.UDF_NAME;
+ return UDF_NAME;
}
/*
@@ -165,7 +133,6 @@ public final class HiveMLUDF extends GenericUDF {
conf = context.getJobConf();
soi = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
doi = LazyPrimitiveObjectInspectorFactory.LAZY_DOUBLE_OBJECT_INSPECTOR;
- LOG.info(
- MLConfConstants.UDF_NAME + " configured. Model base dir path: " + conf.get(ModelLoader.MODEL_PATH_BASE_DIR));
+ LOG.info(UDF_NAME + " configured. Model base dir path: " + conf.get(ModelLoader.MODEL_PATH_BASE_DIR));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
index c749ab3..5938f8a 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/LensMLImpl.java
@@ -19,76 +19,73 @@
package org.apache.lens.ml.impl;
import java.io.IOException;
+import java.io.ObjectOutputStream;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.GenericType;
+import javax.ws.rs.core.MediaType;
+
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.ml.algo.api.Algorithm;
+import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.api.result.LensAPIResult;
+import org.apache.lens.ml.algo.api.MLAlgo;
import org.apache.lens.ml.algo.api.MLDriver;
-import org.apache.lens.ml.algo.api.TrainedModel;
+import org.apache.lens.ml.algo.api.MLModel;
+import org.apache.lens.ml.algo.spark.BaseSparkAlgo;
import org.apache.lens.ml.algo.spark.SparkMLDriver;
-import org.apache.lens.ml.api.*;
-import org.apache.lens.ml.dao.InMemoryMetaStoreClient;
-import org.apache.lens.ml.dao.MetaStoreClient;
+import org.apache.lens.ml.api.LensML;
+import org.apache.lens.ml.api.MLTestReport;
+import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.spark.api.java.JavaSparkContext;
-public class LensMLImpl implements LensML {
+import org.glassfish.jersey.media.multipart.FormDataBodyPart;
+import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
- /**
- * The Constant LOG.
- */
- public static final Log LOG = LogFactory.getLog(LensMLImpl.class);
+import lombok.extern.slf4j.Slf4j;
- /**
- * Check if the predict UDF has been registered for a user
- */
- private final Map<LensSessionHandle, Boolean> predictUdfStatus;
+/**
+ * The Class LensMLImpl.
+ */
+@Slf4j
+public class LensMLImpl implements LensML {
- /**
- * The drivers.
- */
+ /** The drivers. */
protected List<MLDriver> drivers;
- /**
- * The metaStoreClient
- */
- MetaStoreClient metaStoreClient = new InMemoryMetaStoreClient();
-
- Map<String, Algo> algorithms = new HashMap<String, Algo>();
-
- /**
- * The conf.
- */
+ /** The conf. */
private HiveConf conf;
- /**
- * The spark context.
- */
+ /** The spark context. */
private JavaSparkContext sparkContext;
- /**
- * Background thread to periodically check if we need to clear expire status for a session
- */
+ /** Check if the predict UDF has been registered for a user */
+ private final Map<LensSessionHandle, Boolean> predictUdfStatus;
+ /** Background thread to periodically check if we need to clear expire status for a session */
private ScheduledExecutorService udfStatusExpirySvc;
/**
- * Life Cycle manager of Model Instance creation.
- */
- private MLProcessLifeCycleManager mlProcessLifeCycleManager;
-
- /**
* Instantiates a new lens ml impl.
*
* @param conf the conf
@@ -111,10 +108,155 @@ public class LensMLImpl implements LensML {
this.sparkContext = jsc;
}
+ public List<String> getAlgorithms() {
+ List<String> algos = new ArrayList<String>();
+ for (MLDriver driver : drivers) {
+ algos.addAll(driver.getAlgoNames());
+ }
+ return algos;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getAlgoForName(java.lang.String)
+ */
+ public MLAlgo getAlgoForName(String algorithm) throws LensException {
+ for (MLDriver driver : drivers) {
+ if (driver.isAlgoSupported(algorithm)) {
+ return driver.getAlgoInstance(algorithm);
+ }
+ }
+ throw new LensException("Algo not supported " + algorithm);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#train(java.lang.String, java.lang.String, java.lang.String[])
+ */
+ public String train(String table, String algorithm, String[] args) throws LensException {
+ MLAlgo algo = getAlgoForName(algorithm);
+
+ String modelId = UUID.randomUUID().toString();
+
+ log.info("Begin training model " + modelId + ", algo=" + algorithm + ", table=" + table + ", params="
+ + Arrays.toString(args));
+
+ String database = null;
+ if (SessionState.get() != null) {
+ database = SessionState.get().getCurrentDatabase();
+ } else {
+ database = "default";
+ }
+
+ MLModel model = algo.train(toLensConf(conf), database, table, modelId, args);
+
+ log.info("Done training model: " + modelId);
+
+ model.setCreatedAt(new Date());
+ model.setAlgoName(algorithm);
+
+ Path modelLocation = null;
+ try {
+ modelLocation = persistModel(model);
+ log.info("Model saved: " + modelId + ", algo: " + algorithm + ", path: " + modelLocation);
+ return model.getId();
+ } catch (IOException e) {
+ throw new LensException("Error saving model " + modelId + " for algo " + algorithm, e);
+ }
+ }
+
+ /**
+ * Gets the algo dir.
+ *
+ * @param algoName the algo name
+ * @return the algo dir
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private Path getAlgoDir(String algoName) throws IOException {
+ String modelSaveBaseDir = conf.get(ModelLoader.MODEL_PATH_BASE_DIR, ModelLoader.MODEL_PATH_BASE_DIR_DEFAULT);
+ return new Path(new Path(modelSaveBaseDir), algoName);
+ }
+
+ /**
+ * Persist model.
+ *
+ * @param model the model
+ * @return the path
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ private Path persistModel(MLModel model) throws IOException {
+ // Get model save path
+ Path algoDir = getAlgoDir(model.getAlgoName());
+ FileSystem fs = algoDir.getFileSystem(conf);
+
+ if (!fs.exists(algoDir)) {
+ fs.mkdirs(algoDir);
+ }
+
+ Path modelSavePath = new Path(algoDir, model.getId());
+ ObjectOutputStream outputStream = null;
+
+ try {
+ outputStream = new ObjectOutputStream(fs.create(modelSavePath, false));
+ outputStream.writeObject(model);
+ outputStream.flush();
+ } catch (IOException io) {
+ log.error("Error saving model " + model.getId() + " reason: " + io.getMessage(), io);
+ throw io;
+ } finally {
+ IOUtils.closeQuietly(outputStream);
+ }
+ return modelSavePath;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getModels(java.lang.String)
+ */
+ public List<String> getModels(String algorithm) throws LensException {
+ try {
+ Path algoDir = getAlgoDir(algorithm);
+ FileSystem fs = algoDir.getFileSystem(conf);
+ if (!fs.exists(algoDir)) {
+ return null;
+ }
+
+ List<String> models = new ArrayList<String>();
+
+ for (FileStatus stat : fs.listStatus(algoDir)) {
+ models.add(stat.getPath().getName());
+ }
+
+ if (models.isEmpty()) {
+ return null;
+ }
+
+ return models;
+ } catch (IOException ioex) {
+ throw new LensException(ioex);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getModel(java.lang.String, java.lang.String)
+ */
+ public MLModel getModel(String algorithm, String modelId) throws LensException {
+ try {
+ return ModelLoader.loadModel(conf, algorithm, modelId);
+ } catch (IOException e) {
+ throw new LensException(e);
+ }
+ }
+
/**
- * Initialises LensMLImpl. Registers drives, checks if there are previously interrupted MLprocess.
+ * Inits the.
*
- * @param hiveConf
+ * @param hiveConf the hive conf
*/
public synchronized void init(HiveConf hiveConf) {
this.conf = hiveConf;
@@ -126,7 +268,7 @@ public class LensMLImpl implements LensML {
throw new RuntimeException("No ML Drivers specified in conf");
}
- LOG.info("Loading drivers " + Arrays.toString(driverClasses));
+ log.info("Loading drivers " + Arrays.toString(driverClasses));
drivers = new ArrayList<MLDriver>(driverClasses.length);
for (String driverClass : driverClasses) {
@@ -134,12 +276,12 @@ public class LensMLImpl implements LensML {
try {
cls = Class.forName(driverClass);
} catch (ClassNotFoundException e) {
- LOG.error("Driver class not found " + driverClass);
+ log.error("Driver class not found " + driverClass, e);
continue;
}
if (!MLDriver.class.isAssignableFrom(cls)) {
- LOG.warn("Not a driver class " + driverClass);
+ log.warn("Not a driver class " + driverClass);
continue;
}
@@ -148,22 +290,21 @@ public class LensMLImpl implements LensML {
MLDriver driver = mlDriverClass.newInstance();
driver.init(toLensConf(conf));
drivers.add(driver);
- LOG.info("Added driver " + driverClass);
+ log.info("Added driver " + driverClass);
} catch (Exception e) {
- LOG.error("Failed to create driver " + driverClass + " reason: " + e.getMessage(), e);
+ log.error("Failed to create driver " + driverClass + " reason: " + e.getMessage(), e);
}
}
if (drivers.isEmpty()) {
throw new RuntimeException("No ML drivers loaded");
}
- mlProcessLifeCycleManager = new MLProcessLifeCycleManager(conf, metaStoreClient, drivers);
-
- mlProcessLifeCycleManager.init();
-
- LOG.info("Inited ML service");
+ log.info("Inited ML service");
}
+ /**
+ * Start.
+ */
public synchronized void start() {
for (MLDriver driver : drivers) {
try {
@@ -171,215 +312,362 @@ public class LensMLImpl implements LensML {
((SparkMLDriver) driver).useSparkContext(sparkContext);
}
driver.start();
- registerAlgorithms(driver);
} catch (LensException e) {
- LOG.error("Failed to start driver " + driver, e);
+ log.error("Failed to start driver " + driver, e);
}
}
- mlProcessLifeCycleManager.start();
-
udfStatusExpirySvc = Executors.newSingleThreadScheduledExecutor();
udfStatusExpirySvc.scheduleAtFixedRate(new UDFStatusExpiryRunnable(), 60, 60, TimeUnit.SECONDS);
- LOG.info("Started ML service");
+ log.info("Started ML service");
}
+ /**
+ * Stop.
+ */
public synchronized void stop() {
for (MLDriver driver : drivers) {
try {
driver.stop();
} catch (LensException e) {
- LOG.error("Failed to stop driver " + driver, e);
+ log.error("Failed to stop driver " + driver, e);
}
}
drivers.clear();
udfStatusExpirySvc.shutdownNow();
-
- mlProcessLifeCycleManager.stop();
-
- LOG.info("Stopped ML service");
+ log.info("Stopped ML service");
}
- @Override
- public List<Algo> getAlgos() {
- List<Algo> allAlgos = new ArrayList<Algo>();
- allAlgos.addAll(algorithms.values());
- return allAlgos;
+ public synchronized HiveConf getHiveConf() {
+ return conf;
}
- @Override
- public Algo getAlgo(String name) throws LensException {
- return algorithms.get(name);
+ /**
+ * Clear models.
+ */
+ public void clearModels() {
+ ModelLoader.clearCache();
}
- @Override
- public String createDataSet(String name, String dataTable, String dataBase) throws LensException {
- DataSet dataSet = metaStoreClient.createDataSet(name, dataTable, dataBase);
- return dataSet.getName();
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getModelPath(java.lang.String, java.lang.String)
+ */
+ public String getModelPath(String algorithm, String modelID) {
+ return ModelLoader.getModelLocation(conf, algorithm, modelID).toString();
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#testModel(org.apache.lens.api.LensSessionHandle, java.lang.String, java.lang.String,
+ * java.lang.String)
+ */
@Override
- public String createDataSetFromQuery(String name, String query) {
+ public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
+ String outputTable) throws LensException {
return null;
}
- @Override
- public DataSet getDataSet(String name) throws LensException {
- return metaStoreClient.getDataSet(name);
+ /**
+ * Test a model in embedded mode.
+ *
+ * @param sessionHandle the session handle
+ * @param table the table
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @param queryApiUrl the query api url
+ * @return the ML test report
+ * @throws LensException the lens exception
+ */
+ public MLTestReport testModelRemote(LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
+ String queryApiUrl, String outputTable) throws LensException {
+ return testModel(sessionHandle, table, algorithm, modelID, new RemoteQueryRunner(sessionHandle, queryApiUrl),
+ outputTable);
}
- @Override
- public String createModel(String name, String algoName, Map<String, String> algoParams, List<Feature> features,
- Feature label, LensSessionHandle lensSessionHandle) throws LensException {
- Algo algo = getAlgo(algoName);
- AlgoSpec algoSpec = new AlgoSpec(algo, algoParams);
- Model model = metaStoreClient.createModel(name, algoName, algoSpec, features, label);
- return model.getId();
- }
+ /**
+ * Evaluate a model. Evaluation is done on data selected table from an input table. The model is run as a UDF and its
+ * output is inserted into a table with a partition. Each evaluation is given a unique ID. The partition label is
+ * associated with this unique ID.
+ * <p></p>
+ * <p>
+ * This call also required a query runner. Query runner is responsible for executing the evaluation query against Lens
+ * server.
+ * </p>
+ *
+ * @param sessionHandle the session handle
+ * @param table the table
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @param queryRunner the query runner
+ * @param outputTable table where test output will be written
+ * @return the ML test report
+ * @throws LensException the lens exception
+ */
+ public MLTestReport testModel(final LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
+ QueryRunner queryRunner, String outputTable) throws LensException {
+ if (sessionHandle == null) {
+ throw new NullPointerException("Null session not allowed");
+ }
+ // check if algorithm exists
+ if (!getAlgorithms().contains(algorithm)) {
+ throw new LensException("No such algorithm " + algorithm);
+ }
- @Override
- public Model getModel(String modelId) throws LensException {
- return metaStoreClient.getModel(modelId);
- }
+ MLModel<?> model;
+ try {
+ model = ModelLoader.loadModel(conf, algorithm, modelID);
+ } catch (IOException e) {
+ throw new LensException(e);
+ }
- @Override
- public String trainModel(String modelId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- String id = UUID.randomUUID().toString();
-
- ModelInstance modelInstance = metaStoreClient.createModelInstance(new Date(), null, Status.SUBMITTED,
- lensSessionHandle,
- modelId, dataSetName, "", null);
- mlProcessLifeCycleManager.addProcess(modelInstance);
- return modelInstance.getId();
- }
+ if (model == null) {
+ throw new LensException("Model not found: " + modelID + " algorithm=" + algorithm);
+ }
- @Override
- public ModelInstance getModelInstance(String modelInstanceId) throws LensException {
- ModelInstance modelInstance = (ModelInstance) mlProcessLifeCycleManager.getMLProcess(modelInstanceId);
- if (modelInstance != null) {
- return modelInstance;
+ String database = null;
+
+ if (SessionState.get() != null) {
+ database = SessionState.get().getCurrentDatabase();
}
- return metaStoreClient.getModelInstance(modelInstanceId);
- }
- @Override
- public boolean cancelModelInstance(String modelInstanceId) {
- return mlProcessLifeCycleManager.cancelProcess(modelInstanceId);
- }
+ String testID = UUID.randomUUID().toString().replace("-", "_");
+ final String testTable = outputTable;
+ final String testResultColumn = "prediction_result";
- @Override
- public List<ModelInstance> getAllModelInstances(String modelId) {
- return null;
- }
+ // TODO support error metric UDAFs
+ TableTestingSpec spec = TableTestingSpec.newBuilder().hiveConf(conf)
+ .database(database == null ? "default" : database).inputTable(table).featureColumns(model.getFeatureColumns())
+ .outputColumn(testResultColumn).lableColumn(model.getLabelColumn()).algorithm(algorithm).modelID(modelID)
+ .outputTable(testTable).testID(testID).build();
- @Override
- public String evaluate(String modelInstanceId, String inputDataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
+ String testQuery = spec.getTestQuery();
+ if (testQuery == null) {
+ throw new LensException("Invalid test spec. " + "table=" + table + " algorithm=" + algorithm + " modelID="
+ + modelID);
+ }
- metaStoreClient.getDataSet(inputDataSetName);
+ if (!spec.isOutputTableExists()) {
+ log.info("Output table '" + testTable + "' does not exist for test algorithm = " + algorithm + " modelid="
+ + modelID + ", Creating table using query: " + spec.getCreateOutputTableQuery());
+ // create the output table
+ String createOutputTableQuery = spec.getCreateOutputTableQuery();
+ queryRunner.runQuery(createOutputTableQuery);
+ log.info("Table created " + testTable);
+ }
- Evaluation evaluation = metaStoreClient.createEvaluation(new Date(), null, Status.SUBMITTED, lensSessionHandle,
- modelInstanceId, inputDataSetName);
- mlProcessLifeCycleManager.addProcess(evaluation);
- return evaluation.getId();
+ // Check if ML UDF is registered in this session
+ registerPredictUdf(sessionHandle, queryRunner);
+
+ log.info("Running evaluation query " + testQuery);
+ queryRunner.setQueryName("model_test_" + modelID);
+ QueryHandle testQueryHandle = queryRunner.runQuery(testQuery);
+
+ MLTestReport testReport = new MLTestReport();
+ testReport.setReportID(testID);
+ testReport.setAlgorithm(algorithm);
+ testReport.setFeatureColumns(model.getFeatureColumns());
+ testReport.setLabelColumn(model.getLabelColumn());
+ testReport.setModelID(model.getId());
+ testReport.setOutputColumn(testResultColumn);
+ testReport.setOutputTable(testTable);
+ testReport.setTestTable(table);
+ testReport.setQueryID(testQueryHandle.toString());
+
+ // Save test report
+ persistTestReport(testReport);
+ log.info("Saved test report " + testReport.getReportID());
+ return testReport;
}
- @Override
- public Evaluation getEvaluation(String evalId) throws LensException {
- return metaStoreClient.getEvaluation(evalId);
+ /**
+ * Persist test report.
+ *
+ * @param testReport the test report
+ * @throws LensException the lens exception
+ */
+ private void persistTestReport(MLTestReport testReport) throws LensException {
+ log.info("saving test report " + testReport.getReportID());
+ try {
+ ModelLoader.saveTestReport(conf, testReport);
+ log.info("Saved report " + testReport.getReportID());
+ } catch (IOException e) {
+ log.error("Error saving report " + testReport.getReportID() + " reason: " + e.getMessage(), e);
+ }
}
- @Override
- public boolean cancelEvaluation(String evalId) {
- return mlProcessLifeCycleManager.cancelProcess(evalId);
- }
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getTestReports(java.lang.String)
+ */
+ public List<String> getTestReports(String algorithm) throws LensException {
+ Path reportBaseDir = new Path(conf.get(ModelLoader.TEST_REPORT_BASE_DIR, ModelLoader.TEST_REPORT_BASE_DIR_DEFAULT));
+ FileSystem fs = null;
- @Override
- public String predict(String modelInstanceId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- String id = UUID.randomUUID().toString();
- DataSet dataSet = metaStoreClient.getDataSet(dataSetName);
- String outputDataSetName = MLConfConstants.PREDICTION_OUTPUT_TABLE_PREFIX + id.replace("-", "_");
- metaStoreClient.createDataSet(outputDataSetName, outputDataSetName, dataSet.getDatabase());
- Prediction prediction = metaStoreClient.createPrediction(new Date(), null, Status.SUBMITTED, lensSessionHandle,
- modelInstanceId,
- dataSetName, outputDataSetName);
- mlProcessLifeCycleManager.addProcess(prediction);
- return prediction.getId();
+ try {
+ fs = reportBaseDir.getFileSystem(conf);
+ if (!fs.exists(reportBaseDir)) {
+ return null;
+ }
+
+ Path algoDir = new Path(reportBaseDir, algorithm);
+ if (!fs.exists(algoDir)) {
+ return null;
+ }
+
+ List<String> reports = new ArrayList<String>();
+ for (FileStatus stat : fs.listStatus(algoDir)) {
+ reports.add(stat.getPath().getName());
+ }
+ return reports;
+ } catch (IOException e) {
+ log.error("Error reading report list for " + algorithm, e);
+ return null;
+ }
}
- @Override
- public Prediction getPrediction(String predictionId) throws LensException {
- Prediction prediction = (Prediction) mlProcessLifeCycleManager.getMLProcess(predictionId);
- if (prediction != null) {
- return prediction;
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getTestReport(java.lang.String, java.lang.String)
+ */
+ public MLTestReport getTestReport(String algorithm, String reportID) throws LensException {
+ try {
+ return ModelLoader.loadReport(conf, algorithm, reportID);
+ } catch (IOException e) {
+ throw new LensException(e);
}
- return metaStoreClient.getPrediction(predictionId);
}
- @Override
- public boolean cancelPrediction(String predictionId) {
- return mlProcessLifeCycleManager.cancelProcess(predictionId);
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#predict(java.lang.String, java.lang.String, java.lang.Object[])
+ */
+ public Object predict(String algorithm, String modelID, Object[] features) throws LensException {
+ // Load the model instance
+ MLModel<?> model = getModel(algorithm, modelID);
+ return model.predict(features);
}
- @Override
- public String predict(String modelInstanceId, Map<String, String> featureVector) throws LensException {
- ModelInstance modelInstance;
- Model model;
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#deleteModel(java.lang.String, java.lang.String)
+ */
+ public void deleteModel(String algorithm, String modelID) throws LensException {
try {
- modelInstance = metaStoreClient.getModelInstance(modelInstanceId);
- if (modelInstance == null) {
- throw new LensException("Invalid modelInstance Id.");
- }
- if (modelInstance.getStatus() != Status.COMPLETED) {
- throw new LensException("Prediction is allowed only on modelInstances which has completed training "
- + "successfully. Current modelInstance status : " + modelInstance.getStatus());
- }
- model = metaStoreClient.getModel(modelInstance.getModelId());
- } catch (Exception e) {
- throw new LensException("Error Reading modelInstanceId :" + modelInstanceId, e);
+ ModelLoader.deleteModel(conf, algorithm, modelID);
+ log.info("DELETED model " + modelID + " algorithm=" + algorithm);
+ } catch (IOException e) {
+ log.error(
+ "Error deleting model file. algorithm=" + algorithm + " model=" + modelID + " reason: " + e.getMessage(), e);
+ throw new LensException("Unable to delete model " + modelID + " for algorithm " + algorithm, e);
}
+ }
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#deleteTestReport(java.lang.String, java.lang.String)
+ */
+ public void deleteTestReport(String algorithm, String reportID) throws LensException {
try {
- TrainedModel trainedModel = ModelLoader
- .loadModel(conf, model.getAlgoSpec().getAlgo().getName(), model.getId(), modelInstance.getId());
- Object trainingResult = trainedModel.predict(featureVector);
- return trainingResult.toString();
- } catch (Exception e) {
- throw new LensException("Error while training model for modelInstanceId: " + modelInstanceId, e);
+ ModelLoader.deleteTestReport(conf, algorithm, reportID);
+ log.info("DELETED report=" + reportID + " algorithm=" + algorithm);
+ } catch (IOException e) {
+ log.error("Error deleting report " + reportID + " algorithm=" + algorithm + " reason: " + e.getMessage(), e);
+ throw new LensException("Unable to delete report " + reportID + " for algorithm " + algorithm, e);
}
}
- /**
- * Register all available algorithms to cache.
+ /*
+ * (non-Javadoc)
*
- * @param driver
+ * @see org.apache.lens.ml.LensML#getAlgoParamDescription(java.lang.String)
*/
- void registerAlgorithms(MLDriver driver) {
- for (String algoName : driver.getAlgoNames()) {
- try {
- final Algorithm algorithm = driver.getAlgoInstance(algoName);
- algorithms.put(algorithm.getName(), new Algo() {
- @Override
- public String getName() {
- return algorithm.getName();
- }
+ public Map<String, String> getAlgoParamDescription(String algorithm) {
+ MLAlgo algo = null;
+ try {
+ algo = getAlgoForName(algorithm);
+ } catch (LensException e) {
+ log.error("Error getting algo description : " + algorithm, e);
+ return null;
+ }
+ if (algo instanceof BaseSparkAlgo) {
+ return ((BaseSparkAlgo) algo).getArgUsage();
+ }
+ return null;
+ }
- @Override
- public String getDescription() {
- return algorithm.getDescription();
- }
+ /**
+ * Submit model test query to a remote Lens server.
+ */
+ class RemoteQueryRunner extends QueryRunner {
+
+ /** The query api url. */
+ final String queryApiUrl;
+
+ /**
+ * Instantiates a new remote query runner.
+ *
+ * @param sessionHandle the session handle
+ * @param queryApiUrl the query api url
+ */
+ public RemoteQueryRunner(LensSessionHandle sessionHandle, String queryApiUrl) {
+ super(sessionHandle);
+ this.queryApiUrl = queryApiUrl;
+ }
- @Override
- public List<AlgoParam> getParams() {
- return algorithm.getParams();
- }
- });
- } catch (Exception e) {
- LOG.error("Couldn't register algorithm " + algoName);
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.TestQueryRunner#runQuery(java.lang.String)
+ */
+ @Override
+ public QueryHandle runQuery(String query) throws LensException {
+ // Create jersey client for query endpoint
+ Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).build();
+ WebTarget target = client.target(queryApiUrl);
+ final FormDataMultiPart mp = new FormDataMultiPart();
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), sessionHandle,
+ MediaType.APPLICATION_XML_TYPE));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), query));
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
+
+ LensConf lensConf = new LensConf();
+ lensConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, false + "");
+ lensConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false + "");
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), lensConf,
+ MediaType.APPLICATION_XML_TYPE));
+
+ final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
+
+ LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", sessionHandle).request()
+ .get(LensQuery.class);
+
+ QueryStatus stat = ctx.getStatus();
+ while (!stat.finished()) {
+ ctx = target.path(handle.toString()).queryParam("sessionid", sessionHandle).request().get(LensQuery.class);
+ stat = ctx.getStatus();
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ throw new LensException(e);
+ }
}
+
+ if (stat.getStatus() != QueryStatus.Status.SUCCESSFUL) {
+ throw new LensException("Query failed " + ctx.getQueryHandle().getHandleId() + " reason:"
+ + stat.getErrorMessage());
+ }
+
+ return ctx.getQueryHandle();
}
}
@@ -395,54 +683,29 @@ public class LensMLImpl implements LensML {
return lensConf;
}
- /**
- * Gets the algo dir.
- *
- * @param algoName the algo name
- * @return the algo dir
- * @throws java.io.IOException Signals that an I/O exception has occurred.
- */
- private Path getAlgoDir(String algoName) throws IOException {
- String modelSaveBaseDir = conf.get(ModelLoader.MODEL_PATH_BASE_DIR, ModelLoader.MODEL_PATH_BASE_DIR_DEFAULT);
- return new Path(new Path(modelSaveBaseDir), algoName);
- }
-
- /**
- * Registers predict UDF for a given LensSession
- *
- * @param sessionHandle
- * @param queryRunner
- * @throws LensException
- */
protected void registerPredictUdf(LensSessionHandle sessionHandle, QueryRunner queryRunner) throws LensException {
- if (isUdfRegistered(sessionHandle)) {
+ if (isUdfRegisterd(sessionHandle)) {
// Already registered, nothing to do
return;
}
- LOG.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
+ log.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
- String regUdfQuery = "CREATE TEMPORARY FUNCTION " + MLConfConstants.UDF_NAME + " AS '" + HiveMLUDF.class
+ String regUdfQuery = "CREATE TEMPORARY FUNCTION " + HiveMLUDF.UDF_NAME + " AS '" + HiveMLUDF.class
.getCanonicalName() + "'";
queryRunner.setQueryName("register_predict_udf_" + sessionHandle.getPublicId().toString());
QueryHandle udfQuery = queryRunner.runQuery(regUdfQuery);
- LOG.info("udf query handle is " + udfQuery);
+ log.info("udf query handle is " + udfQuery);
predictUdfStatus.put(sessionHandle, true);
- LOG.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
+ log.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
}
- /**
- * Checks if predict UDF is registered for a given LensSession
- *
- * @param sessionHandle
- * @return
- */
- protected boolean isUdfRegistered(LensSessionHandle sessionHandle) {
+ protected boolean isUdfRegisterd(LensSessionHandle sessionHandle) {
return predictUdfStatus.containsKey(sessionHandle);
}
/**
- * Clears predictUDFStatus for all closed LensSessions.
+ * Periodically check if sessions have been closed, and clear UDF registered status.
*/
private class UDFStatusExpiryRunnable implements Runnable {
public void run() {
@@ -452,12 +715,12 @@ public class LensMLImpl implements LensML {
List<LensSessionHandle> sessions = new ArrayList<LensSessionHandle>(predictUdfStatus.keySet());
for (LensSessionHandle sessionHandle : sessions) {
if (!sessionService.isOpen(sessionHandle)) {
- LOG.info("Session closed, removing UDF status: " + sessionHandle);
+ log.info("Session closed, removing UDF status: " + sessionHandle);
predictUdfStatus.remove(sessionHandle);
}
}
} catch (Exception exc) {
- LOG.warn("Error clearing UDF statuses", exc);
+ log.warn("Error clearing UDF statuses", exc);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLProcessLifeCycleManager.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLProcessLifeCycleManager.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLProcessLifeCycleManager.java
deleted file mode 100644
index f7b297e..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLProcessLifeCycleManager.java
+++ /dev/null
@@ -1,759 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.impl;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.*;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.query.LensQuery;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-import org.apache.lens.ml.algo.api.Algorithm;
-import org.apache.lens.ml.algo.api.MLDriver;
-import org.apache.lens.ml.algo.api.TrainedModel;
-import org.apache.lens.ml.api.*;
-import org.apache.lens.ml.dao.MetaStoreClient;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.QueryExecutionService;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.session.SessionState;
-
-import lombok.Getter;
-import lombok.Setter;
-
-/**
- * MLProcessLifeCycleManager class. Responsible for Life Cycle management of a MLProcess i.e. ModelInstance,
- * Prediction, Evaluation.
- */
-public class MLProcessLifeCycleManager {
-
- /**
- * The Constant LOG.
- */
- public static final Log LOG = LogFactory.getLog(MLProcessLifeCycleManager.class);
- /**
- * Prefix for all MLProcess worker threads.
- */
- private static final String ML_PROCESS_LIFECYCLE_THREAD_PREFIX = "MLProcess-";
- /**
- * Check if the predict UDF has been registered for a user
- */
- private final Map<LensSessionHandle, Boolean> predictUdfStatus;
- /**
- * Runnable for thread responsible for purging completed MLProcesses. and killing threads which have exceeded
- * maximum life time.
- */
- ProcessPurger processPurgerRunnable = new ProcessPurger();
- Thread processPurger = new Thread(processPurgerRunnable, "MLProcessPurger");
- /**
- * Runnable for thread responsible for submitting incoming MLProcesses to the Queue.
- */
- MLProcessSubmitter mlProcessSubmitterRunnable = new MLProcessSubmitter();
- Thread mlProcessSubmitter = new Thread(mlProcessSubmitterRunnable, "MLProcessSubmitter");
- /**
- * boolean for checking whether this LifeCycle is running or not.
- */
- boolean stopped;
- /**
- * The meta store client.
- */
- MetaStoreClient metaStoreClient;
-
- /**
- * All registered drivers.
- */
- List<MLDriver> drivers;
-
- /**
- * Map for storing MLProcesses which are submitted, or executing. Once a process is finished purger thread will
- * remove it from this Map after configured time or if a process exceeds it's maximum life time.
- */
- ConcurrentMap<String, MLProcessContext> allProcesses = new ConcurrentHashMap<String, MLProcessContext>();
- private HiveConf conf;
- /**
- * All accepted MLProcesses are put into this queue. MLProcessSubmitter thread waits on this queue. It fetches
- * MLProcesses from here starts its execution.
- */
- private BlockingQueue<MLProcessContext> submittedQueue = new LinkedBlockingQueue<MLProcessContext>();
- /**
- * Executor pool for MLProcess worker threads. i.e. EvaluationCreator, ModelCreator, PredictionCreator.
- */
- private ExecutorService executorPool;
-
- public MLProcessLifeCycleManager(HiveConf conf, MetaStoreClient metaStoreClient, List<MLDriver> drivers) {
- this.conf = conf;
- this.metaStoreClient = metaStoreClient;
- this.predictUdfStatus = new ConcurrentHashMap<LensSessionHandle, Boolean>();
- this.drivers = drivers;
- }
-
- /**
- * Initializes MLProcess. Restores previous incomplete MLProcesses
- */
- public void init() {
-
- try {
- for (MLProcess process : metaStoreClient.getIncompleteEvaluations()) {
- MLProcessContext mlProcessContext = new MLProcessContext(process);
- submittedQueue.add(mlProcessContext);
- }
- LOG.info("Restored old incomplete Evaluations.");
- } catch (Exception e) {
- LOG.error("Error while restoring previous incomplete Evaluations.");
- }
-
- try {
- for (MLProcess process : metaStoreClient.getIncompleteModelInstances()) {
- MLProcessContext mlProcessContext = new MLProcessContext(process);
- submittedQueue.add(mlProcessContext);
- LOG.info("Restored old incomplete ModelInstances.");
- }
- } catch (Exception e) {
- LOG.error("Error while restoring previous incomplete ModelInstance.");
- }
-
- try {
- for (MLProcess process : metaStoreClient.getIncompletePredictions()) {
- MLProcessContext mlProcessContext = new MLProcessContext(process);
- submittedQueue.add(mlProcessContext);
- LOG.info("Restored old incomplete Predictions.");
- }
- } catch (Exception e) {
- LOG.error("Error while restoring previous incomplete Predictions.");
- }
- LOG.info("Initialized MLProcessLifeCycle");
- }
-
- /**
- * Starts the MLProcessLifeCycle Manager
- */
- public void start() {
- stopped = false;
- startExecutorPool();
- mlProcessSubmitter.start();
- processPurger.start();
- LOG.info("Started MLProcessLifeCycle");
- }
-
- /**
- * Stop teh ML Process Life Cycle Manager
- */
- public void stop() {
- executorPool.shutdown();
- stopped = true;
- LOG.info("Stopped MLProcessLifeCycle");
- }
-
- public MLProcess getMLProcess(String id) {
- if (allProcesses.containsKey(id)) {
- return allProcesses.get(id).getMlProcess();
- }
- return null;
- }
-
- private void startExecutorPool() {
- int minPoolSize =
- conf.getInt(MLConfConstants.EXECUTOR_POOL_MIN_THREADS, MLConfConstants.DEFAULT_EXECUTOR_POOL_MIN_THREADS);
- int maxPoolSize = conf.getInt(MLConfConstants.EXECUTOR_POOL_MAX_THREADS, MLConfConstants
- .DEFAULT_EXECUTOR_POOL_MAX_THREADS);
-
- final ThreadFactory defaultFactory = Executors.defaultThreadFactory();
- final AtomicInteger thId = new AtomicInteger();
-
- ThreadFactory threadFactory = new ThreadFactory() {
- @Override
- public Thread newThread(Runnable r) {
- Thread th = defaultFactory.newThread(r);
- th.setName(ML_PROCESS_LIFECYCLE_THREAD_PREFIX + thId.incrementAndGet());
- return th;
- }
- };
-
- LOG.debug("starting executor pool");
- ThreadPoolExecutor executorPool =
- new ThreadPoolExecutor(minPoolSize, maxPoolSize, MLConfConstants.DEFAULT_CREATOR_POOL_KEEP_ALIVE_MILLIS,
- TimeUnit.MILLISECONDS,
- new LinkedBlockingQueue<Runnable>(), threadFactory);
- this.executorPool = executorPool;
- }
-
- /**
- * Accepts MLProcesses and adds them to submitted queue.
- *
- * @param mlProcess
- * @throws LensException
- */
- public void addProcess(MLProcess mlProcess) throws LensException {
- MLProcessContext mlProcessContext = new MLProcessContext(mlProcess);
- allProcesses.put(mlProcess.getId(), mlProcessContext);
- submittedQueue.add(mlProcessContext);
- LOG.debug("MLProcess submitted, Id: " + mlProcess.getId());
- }
-
- /**
- * Cancelles the execution of MLProcess by killing the Executor Thread.
- *
- * @param processId
- * @return
- */
- public boolean cancelProcess(String processId) {
- MLProcessContext mlProcessContext = allProcesses.get(processId);
- if (mlProcessContext != null) {
- setProcessStatusAndFinishTime(Status.CANCELLED, mlProcessContext);
- } else {
- return false;
- }
- if (mlProcessContext.getThread() != null) {
- if (!mlProcessContext.getThread().isDone()) {
- try {
- mlProcessContext.getThread().cancel(true);
- } catch (Exception e) {
- LOG.error("Error in cancelling MLProcess, Id: " + processId);
- return false;
- }
- LOG.debug("MLProcess cancelled, Id: " + processId);
- }
- }
- return true;
- }
-
- /**
- * Sets MLProcess status. Also updates finish time.
- *
- * @param status
- * @param ctx
- */
- public void setProcessStatusAndFinishTime(Status status, MLProcessContext ctx) {
- synchronized (ctx) {
- ctx.getMlProcess().setStatus(status);
- ctx.getMlProcess().setFinishTime(new Date());
- }
- }
-
- /**
- * Registers the Predict UDF for a given Lens Session.
- *
- * @param sessionHandle
- * @param queryRunner
- * @throws LensException
- */
- protected void registerPredictUdf(LensSessionHandle sessionHandle, QueryRunner queryRunner) throws LensException {
- if (isUdfRegistered(sessionHandle)) {
- // Already registered, nothing to do
- return;
- }
-
- LOG.info("Registering UDF for session " + sessionHandle.getPublicId().toString());
-
- String regUdfQuery = "CREATE TEMPORARY FUNCTION " + MLConfConstants.UDF_NAME + " AS '" + HiveMLUDF.class
- .getCanonicalName() + "'";
- queryRunner.setQueryName("register_predict_udf_" + sessionHandle.getPublicId().toString());
- QueryHandle udfQuery = queryRunner.runQuery(regUdfQuery);
- LOG.info("udf query handle is " + udfQuery);
- predictUdfStatus.put(sessionHandle, true);
- LOG.info("Predict UDF registered for session " + sessionHandle.getPublicId().toString());
- }
-
- protected boolean isUdfRegistered(LensSessionHandle sessionHandle) {
- return predictUdfStatus.containsKey(sessionHandle);
- }
-
- /**
- * An ML process is finished if it has status among FAILED, COMPLETED or CANCELLED.
- *
- * @param mlProcess
- * @return
- */
- boolean isFinished(MLProcess mlProcess) {
- Status status = mlProcess.getStatus();
- if (status == Status.FAILED || status == Status.COMPLETED || status == Status.CANCELLED) {
- return true;
- }
- return false;
- }
-
- /**
- * Returns the Algorithm registered in driver for the name.
- *
- * @param name
- * @return
- * @throws LensException
- */
- public Algorithm getAlgoForName(String name) throws LensException {
- for (MLDriver driver : drivers) {
- if (driver.isAlgoSupported(name)) {
- Algorithm algorithm = driver.getAlgoInstance(name);
- algorithm.configure(toLensConf(conf));
- return algorithm;
- }
- }
- throw new LensException("Algorithm not supported " + name);
- }
-
- private LensConf toLensConf(HiveConf conf) {
- LensConf lensConf = new LensConf();
- lensConf.getProperties().putAll(conf.getValByRegex(".*"));
- return lensConf;
- }
-
- /**
- * MLProcessContext class
- */
- public class MLProcessContext {
- @Getter
- @Setter
- MLProcess mlProcess;
- @Getter
- @Setter
- Future thread;
-
- public MLProcessContext(MLProcess mlProcess) {
- this.mlProcess = mlProcess;
- }
-
- }
-
- /**
- * MLProcessSubmitter thread. Responsible for starting one of the executor threads based on the type of MLProcess.
- */
- class MLProcessSubmitter implements Runnable {
- @Override
- public void run() {
-
- LOG.info("Started Submitter Thread.");
- try {
- while (!stopped) {
- MLProcessContext ctx = submittedQueue.take();
- synchronized (ctx) {
- // Only accept the process with SUBMITTED status. they might be cancelled.
- MLProcess mlProcess = ctx.getMlProcess();
- Runnable creatorThread = null;
- if (mlProcess instanceof ModelInstance) {
- creatorThread = new ModelExecutor(ctx.getMlProcess().getId());
- } else if (mlProcess instanceof Evaluation) {
- creatorThread = new EvaluationExecutor(ctx.getMlProcess().getId());
- } else if (mlProcess instanceof Prediction) {
- creatorThread = new PredictionExecutor(ctx.getMlProcess().getId());
- }
- if (creatorThread != null) {
- Future future = executorPool.submit(creatorThread);
- ctx.setThread(future);
- }
-
- }
- }
- } catch (InterruptedException ex) {
- LOG.error("Submitter has been interrupted, exiting" + ex.getMessage());
- return;
- } catch (Exception e) {
- LOG.error("Error in submitter", e);
- }
- LOG.info("Submitter exited");
- }
- }
-
- /**
- * Worker Thread for running an Evaluation process. It generates the target Hive Query which uses the predict udf
- * for predicting. Makes sure the outputTable is present, UDF is registered for current session. Finally runs the
- * generated hive query through Lens Server. On successful completion it sets the status of MLProcess to COMPLETED
- * otherwise FAILED.
- */
- private class EvaluationExecutor implements Runnable {
- String evaluationId;
-
- public EvaluationExecutor(String evaluationId) {
- this.evaluationId = evaluationId;
- }
-
- @Override
- public void run() {
- MLProcessContext ctx = null;
- try {
- ctx = allProcesses.get(evaluationId);
- if (ctx.getMlProcess().getStatus() == Status.SUBMITTED) {
- ctx.getMlProcess().setStatus(Status.RUNNING);
- String database = null;
- if (SessionState.get() != null) {
- database = SessionState.get().getCurrentDatabase();
- }
-
- Evaluation evaluation = (Evaluation) ctx.getMlProcess();
-
- LensSessionHandle sessionHandle = evaluation.getLensSessionHandle();
- if (sessionHandle == null) {
- LOG.error(
- "Error while running evaluation. Id: " + ctx.getMlProcess().getId() + ". Null session not allowed");
- return;
- }
-
- final String testResultColumn = "prediction_result";
-
- ModelInstance modelInstance = metaStoreClient.getModelInstance(evaluation.getModeInstanceId());
- Model model = metaStoreClient.getModel(modelInstance.getModelId());
-
-
- String outputTableName =
- (MLConfConstants.EVALUATION_OUTPUT_TABLE_PREFIX + evaluation.getId()).replace("-", "_");
- TableTestingSpec spec = TableTestingSpec.newBuilder().hiveConf(conf)
- .database(database == null ? "default" : database).inputTable(evaluation.getInputDataSetName())
- .featureColumns(model.getFeatureSpec())
- .outputColumn(testResultColumn).lableColumn(model.getLabelSpec())
- .algorithm(model.getAlgoSpec().getAlgo().getName()).modelID(model.getId())
- .modelInstanceID(modelInstance.getId())
- .outputTable(outputTableName).testID(evaluationId).build();
-
- String testQuery = spec.getTestQuery();
- if (testQuery == null) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error("Error while running evaluation. Id: " + ctx.getMlProcess().getId());
- return;
- } else {
-
- DirectQueryRunner queryRunner = new DirectQueryRunner(sessionHandle);
- if (!spec.isOutputTableExists()) {
- try {
- String createOutputTableQuery = spec.getCreateOutputTableQuery();
- LOG.info("Output table '" + outputTableName + "' does not exist for test algorithm = "
- + model.getAlgoSpec().getAlgo().getName() + " modelId="
- + model.getId() + " modelInstanceId= " + modelInstance.getId()
- + ", Creating table using query: " + spec.getCreateOutputTableQuery());
- queryRunner.runQuery(createOutputTableQuery);
- } catch (LensException e) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error("Error while running evaluation. Id: " + ctx.getMlProcess().getId()
- + ". Unable to create output table" + e.getMessage());
- return;
- }
-
- try {
- LOG.info("Registering UDF for prediction. Id" + ctx.getMlProcess().getId());
- registerPredictUdf(sessionHandle, queryRunner);
- } catch (LensException e) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error("Error while running evaluation. Id: " + ctx.getMlProcess().getId()
- + ". Unable to register predict UDF" + e.getMessage());
- return;
- }
-
- try {
- LOG.info("Running Prediction UDF" + ctx.getMlProcess().getId());
- queryRunner.runQuery(testQuery);
- } catch (LensException e) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error(
- "Error while running evaluation. Id: " + ctx.getMlProcess().getId() + ". Unable to run predict UDF"
- + e.getMessage());
- return;
- }
- setProcessStatusAndFinishTime(Status.COMPLETED, ctx);
- }
- }
- }
- } catch (Exception e) {
-
- if (ctx == null) {
- LOG.info("Error while running evaluation. Id: " + evaluationId);
- return;
- }
- LOG.info("Error while running evaluation. Id: " + evaluationId);
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- }
- LOG.info("exiting evaluation creator!");
-
- }
- }
-
- /**
- * Worker Thread for Creation of Model Instances. It launches the job for training a Model against the inputTable. On
- * successful completion it sets the status of MLProcess to COMPLETED otherwise FAILED.
- */
- private class ModelExecutor implements Runnable {
- String id;
-
- public ModelExecutor(String id) {
- this.id = id;
- }
-
- @Override
- public void run() {
- MLProcessContext ctx;
- try {
- ctx = allProcesses.get(id);
- } catch (NullPointerException ex) {
- LOG.error("");
- return;
- }
- try {
- ModelInstance modelInstance = (ModelInstance) ctx.getMlProcess();
- Model model = metaStoreClient.getModel(modelInstance.getModelId());
- DataSet dataSet = metaStoreClient.getDataSet(modelInstance.getDataSet());
-
- Algorithm algorithm = getAlgoForName(model.getAlgoSpec().getAlgo().getName());
- TrainedModel trainedModel;
- trainedModel = algorithm.train(model, dataSet);
-
-
- Path modelLocation = MLUtils.persistModel(trainedModel, model, modelInstance.getId());
- LOG.info("ModelInstance saved: " + modelInstance.getId() + ", algo: " + algorithm + ", path: "
- + modelLocation);
-
- setProcessStatusAndFinishTime(Status.COMPLETED, ctx);
-
- } catch (IOException ex) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error("Error saving modelInstance ID: " + ctx.getMlProcess().getId());
- } catch (LensException ex) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error("Error training modelInstance ID: " + ctx.getMlProcess().getId());
- } catch (Exception e) {
- LOG.error(e.getMessage());
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- }
- }
- }
-
- /**
- * Worker Thread for Batch Prediction process. It generates the target Hive Query which uses the predict udf
- * for prediction. Makes sure the outputTable is present, UDF is registered for current session. Finally runs the
- * generated hive query through Lens Server. On successful completion it sets the status of MLProcess to COMPLETED
- * otherwise FAILED.
- */
- private class PredictionExecutor implements Runnable {
- String predictionId;
-
- public PredictionExecutor(String predictionId) {
- this.predictionId = predictionId;
- }
-
- @Override
- public void run() {
- MLProcessContext ctx = null;
-
- try {
- ctx = allProcesses.get(predictionId);
-
- String database = null;
- if (SessionState.get() != null) {
- database = SessionState.get().getCurrentDatabase();
- }
- Prediction prediction = (Prediction) ctx.getMlProcess();
-
- LensSessionHandle sessionHandle = prediction.getLensSessionHandle();
- if (sessionHandle == null) {
- LOG.error("Error while running prediction. Id: " + ctx.getMlProcess().getId() + ". Null session not allowed");
- return;
- }
-
- ModelInstance modelInstance = metaStoreClient.getModelInstance(prediction.getModelInstanceId());
- Model model = metaStoreClient.getModel(modelInstance.getModelId());
-
- final String testResultColumn = "prediction_result";
-
- BatchPredictSpec spec = BatchPredictSpec.newBuilder().hiveConf(conf)
- .database(database == null ? "default" : database).inputTable(prediction.getInputDataSet())
- .featureColumns(model.getFeatureSpec())
- .outputColumn(testResultColumn).algorithm(model.getAlgoSpec().getAlgo().getName())
- .modelID(model.getId()).modelInstanceID(modelInstance.getId())
- .outputTable(prediction.getOutputDataSet()).testID(predictionId).build();
-
- String testQuery = spec.getTestQuery();
- if (testQuery == null) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error("Error while running prediction. Id: " + ctx.getMlProcess().getId());
- return;
- } else {
- DirectQueryRunner queryRunner = new DirectQueryRunner(sessionHandle);
- if (!spec.isOutputTableExists()) {
- try {
- String createOutputTableQuery = spec.getCreateOutputTableQuery();
- LOG.info("Output table '" + prediction.getOutputDataSet()
- + "' does not exist for predicting algorithm = " + model.getAlgoSpec().getAlgo().getName()
- + " modelId="
- + model.getId() + " modelInstanceId= " + modelInstance.getId()
- + ", Creating table using query: " + spec.getCreateOutputTableQuery());
- queryRunner.runQuery(createOutputTableQuery);
- } catch (LensException e) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error(
- "Error while running prediction. Id: " + ctx.getMlProcess().getId() + "Unable to create output table"
- + e.getMessage());
- return;
- }
-
- try {
- LOG.info("Registering UDF for prediction. Id" + ctx.getMlProcess().getId());
- registerPredictUdf(sessionHandle, queryRunner);
- } catch (LensException e) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error(
- "Error while running prediction. Id: " + ctx.getMlProcess().getId()
- + ". Unable to register prediction UDF" + e.getMessage());
- return;
- }
-
- try {
- LOG.info("Running Prediction UDF" + ctx.getMlProcess().getId());
- queryRunner.runQuery(testQuery);
- } catch (Exception e) {
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error(
- "Error while running prediction. Id: " + ctx.getMlProcess().getId() + ". Unable to run prediction UDF"
- + e.getMessage());
- return;
- }
- setProcessStatusAndFinishTime(Status.COMPLETED, ctx);
- }
- }
- } catch (Exception e) {
- if (ctx == null) {
- LOG.error("Error while running prediction. Id: " + predictionId + ", " + e.getMessage());
- return;
- }
- setProcessStatusAndFinishTime(Status.FAILED, ctx);
- LOG.error("Error while running prediction. Id: " + predictionId + ", " + e.getMessage());
- }
- LOG.info("exiting prediction creator!");
- }
-
- }
-
- /**
- * DirectQueryRunner class which runs query against the same lens server where ML Service is running.
- */
- private class DirectQueryRunner extends QueryRunner {
-
- /**
- * Instantiates a new direct query runner.
- *
- * @param sessionHandle the session handle
- */
- public DirectQueryRunner(LensSessionHandle sessionHandle) {
- super(sessionHandle);
- }
-
- /**
- * @param testQuery
- * @return
- * @throws LensException
- */
- @Override
- public QueryHandle runQuery(String testQuery) throws LensException {
- // Run the query in query executions service
- QueryExecutionService queryService;
- try {
- queryService = (QueryExecutionService) MLUtils.getServiceProvider().getService("query");
- } catch (Exception e) {
- throw new LensException("Error while getting Service Provider");
- }
-
- LensConf queryConf = new LensConf();
- queryConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, false + "");
- queryConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false + "");
-
- QueryHandle testQueryHandle = queryService.executeAsync(sessionHandle, testQuery, queryConf, queryName);
-
- // Wait for test query to complete
- LensQuery query = queryService.getQuery(sessionHandle, testQueryHandle);
-
- LOG.info("Submitted query " + testQueryHandle.getHandleId());
- while (!query.getStatus().finished()) {
- try {
- Thread.sleep(2000);
- } catch (InterruptedException e) {
- throw new LensException(e);
- }
-
- query = queryService.getQuery(sessionHandle, testQueryHandle);
- }
-
- if (query.getStatus().getStatus() != QueryStatus.Status.SUCCESSFUL) {
- throw new LensException("Failed to run test query: " + testQueryHandle.getHandleId() + " reason= "
- + query.getStatus().getErrorMessage());
- }
-
- return testQueryHandle;
- }
- }
-
- /**
- * Process Purger Thread. Removes processes from in memory cache after MLConfConstants.ML_PROCESS_CACHE_LIFE time.
- * Also kills a process if it exceeds MLConfConstants.ML_PROCESS_MAX_LIFE.
- */
- private class ProcessPurger implements Runnable {
- @Override
- public void run() {
- Set<String> keys = allProcesses.keySet();
- for (String key : keys) {
- MLProcessContext ctx = allProcesses.get(key);
- MLProcess mlProcess = ctx.getMlProcess();
- long maxQueryLife = conf.getLong(MLConfConstants.ML_PROCESS_MAX_LIFE, MLConfConstants
- .DEFAULT_ML_PROCESS_MAX_LIFE);
-
- if (isFinished(mlProcess)) {
- long cacheLife = conf.getLong(MLConfConstants.ML_PROCESS_CACHE_LIFE, MLConfConstants
- .DEFAULT_ML_PROCESS_CACHE_LIFE);
- if ((new Date().getTime() - mlProcess.getFinishTime().getTime()) > cacheLife) {
- try {
- updateMLProcess(mlProcess);
- } catch (Exception e) {
- LOG.error("Error while persisting MLProcess to meta store, Id: " + mlProcess.getId());
- }
- }
- } else if ((new Date().getTime() - mlProcess.getFinishTime().getTime()) > maxQueryLife) {
- // Kill the thread
- try {
- Future thread = ctx.getThread();
- if (!thread.isDone()) {
- thread.cancel(true);
- }
- mlProcess.setFinishTime(new Date());
- mlProcess.setStatus(Status.FAILED);
- updateMLProcess(mlProcess);
- } catch (LensException e) {
- LOG.error("Error while persisting MLProcess to meta store, Id: " + mlProcess.getId());
- } catch (Exception e) {
- LOG.error("Error while cancelling MLProcess, Id: " + mlProcess.getId());
- }
- }
- }
- }
-
- void updateMLProcess(MLProcess mlProcess) throws LensException {
- if (mlProcess instanceof Prediction) {
- metaStoreClient.updatePrediction((Prediction) mlProcess);
- } else if (mlProcess instanceof Evaluation) {
- metaStoreClient.updateEvaluation((Evaluation) mlProcess);
- } else if (mlProcess instanceof ModelInstance) {
- metaStoreClient.updateModelInstance((ModelInstance) mlProcess);
- }
- }
- }
-}
[23/50] [abbrv] incubator-lens git commit: LENS-652 : Add Update
partition api and cli command
Posted by sh...@apache.org.
LENS-652 : Add Update partition api and cli command
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/7de74eb3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/7de74eb3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/7de74eb3
Branch: refs/heads/LENS-581
Commit: 7de74eb3abc01d9e5c1afe55f97b309328d81d9e
Parents: d8e071c
Author: Rajat Khadelwal <pr...@apache.org>
Authored: Tue Jul 21 14:07:29 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Jul 21 14:07:29 2015 +0530
----------------------------------------------------------------------
lens-api/pom.xml | 2 +
.../lens/cli/commands/BaseLensCommand.java | 2 +-
.../commands/LensDimensionTableCommands.java | 35 +-
.../lens/cli/commands/LensFactCommands.java | 42 ++
.../cli/commands/PhysicalTableCrudCommand.java | 15 +
.../apache/lens/cli/LensCliApplicationTest.java | 2 +
.../cli/TestLensDimensionTableCommands.java | 107 ++--
.../apache/lens/cli/TestLensFactCommands.java | 42 +-
.../src/test/resources/dim1-local-parts.xml | 5 +-
.../src/test/resources/fact1-local-parts.xml | 4 +-
.../java/org/apache/lens/client/LensClient.java | 31 ++
.../apache/lens/client/LensMetadataClient.java | 95 ++++
.../lens/cube/metadata/CubeMetastoreClient.java | 107 +++-
.../apache/lens/cube/metadata/HDFSStorage.java | 16 +
.../lens/cube/metadata/MetastoreUtil.java | 6 +-
.../lens/cube/metadata/PartitionMetahook.java | 29 +-
.../org/apache/lens/cube/metadata/Storage.java | 47 +-
.../lens/cube/metadata/TimePartition.java | 20 +-
.../apache/lens/cube/metadata/UpdatePeriod.java | 19 +
.../lens/cube/parse/StorageTableResolver.java | 7 +-
.../cube/metadata/TestCubeMetastoreClient.java | 538 ++++++++++---------
.../apache/lens/cube/parse/CubeTestSetup.java | 10 +-
.../src/main/resources/hivedriver-default.xml | 27 +-
.../api/metastore/CubeMetastoreService.java | 21 +-
.../metastore/CubeMetastoreServiceImpl.java | 69 ++-
.../apache/lens/server/metastore/JAXBUtils.java | 65 ++-
.../server/metastore/MetastoreResource.java | 116 +++-
.../server/metastore/TestMetastoreService.java | 109 ++--
.../org/apache/lens/storage/db/DBStorage.java | 16 +
.../apache/lens/storage/db/TestDBStorage.java | 2 +-
src/site/apt/admin/hivedriver-config.apt | 12 +-
src/site/apt/user/cli.apt | 14 +-
32 files changed, 1166 insertions(+), 466 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index 507dd01..e1b763d 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -89,6 +89,8 @@
<arg>-extension</arg>
<arg>-Xinheritance</arg>
<arg>-XtoString</arg>
+ <arg>-Xequals</arg>
+ <arg>-XhashCode</arg>
</args>
<plugins>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
index 74b4e91..bdfae24 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
@@ -143,7 +143,7 @@ public class BaseLensCommand implements ExecutionProcessor {
.replaceAll("]", "\n").replaceAll(",", "").replaceAll("\"", "").replaceAll("\n\n", "\n");
}
- public String getValidPath(String path, boolean shouldBeDirectory, boolean shouldExist) {
+ public static String getValidPath(String path, boolean shouldBeDirectory, boolean shouldExist) {
path = path.replaceAll("/$", "");
if (path.startsWith("~")) {
path = path.replaceFirst("~", System.getProperty("user.home"));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
index 6a93393..5fd80a1 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDimensionTableCommands.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cli.commands;
+import java.io.File;
import java.util.List;
import org.apache.lens.api.APIResult;
@@ -31,6 +32,8 @@ import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
+import lombok.NonNull;
+
/**
* The Class LensDimensionTableCommands.
*/
@@ -189,7 +192,7 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
@CliCommand(value = "dimtable list partitions",
help = "get all partitions associated with dimtable <dimtable_name>, "
+ "storage <storage_name> filtered by <partition-filter>")
- public String getAllPartitionsOfDim(
+ public String getAllPartitionsOfDimtable(
@CliOption(key = {"", "dimtable_name"}, mandatory = true, help = "<dimtable_name>") String tableName,
@CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
@CliOption(key = {"", "filter"}, mandatory = false, help = "<partition-filter>") String filter) {
@@ -231,6 +234,16 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
@CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") String path) {
return addPartition(tableName, storageName, path);
}
+ @CliCommand(value = "dimtable update single-partition",
+ help = "update single partition to dimtable <dimtable_name>'s"
+ + " storage <storage_name>, reading spec from <partition-spec-path>"
+ + " The partition has to exist to be eligible for updation.")
+ public String updatePartitionOfDimtable(
+ @CliOption(key = {"", "dimtable_name"}, mandatory = true, help = "<dimtable_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") @NonNull final File path) {
+ return updatePartition(tableName, storageName, path);
+ }
/**
* Adds the partitions to dim table.
@@ -250,6 +263,16 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
@CliOption(key = {"", "path"}, mandatory = true, help = "<partition-list-spec-path>") String path) {
return addPartitions(tableName, storageName, path);
}
+ @CliCommand(value = "dimtable update partitions",
+ help = "update multiple partition to dimtable <dimtable_name>'s"
+ + " storage <storage_name>, reading partition list spec from <partition-list-spec-path>"
+ +" The partitions have to exist to be eligible for updation.")
+ public String updatePartitionsOfDimtable(
+ @CliOption(key = {"", "dimtable_name"}, mandatory = true, help = "<dimtable_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-list-spec-path>") String path) {
+ return updatePartitions(tableName, storageName, path);
+ }
@Override
protected XStorageTableElement readStorage(String tableName, String storage) {
@@ -298,6 +321,16 @@ public class LensDimensionTableCommands extends PhysicalTableCrudCommand<XDimens
}
@Override
+ protected APIResult doUpdatePartition(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionOfDim(tableName, storageName, validPath);
+ }
+
+ @Override
+ protected APIResult doUpdatePartitions(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionsOfDim(tableName, storageName, validPath);
+ }
+
+ @Override
public List<String> getAll() {
return getClient().getAllDimensionTables();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
index bdb9c38..5493dba 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensFactCommands.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cli.commands;
+import java.io.File;
import java.util.List;
import org.apache.lens.api.APIResult;
@@ -30,6 +31,8 @@ import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
+import lombok.NonNull;
+
/**
* The Class LensFactCommands.
*/
@@ -226,6 +229,24 @@ public class LensFactCommands extends PhysicalTableCrudCommand<XFactTable> {
@CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") String path) {
return addPartition(tableName, storageName, path);
}
+ /**
+ * updates the partition in the fact.
+ *
+ * @param tableName fact name
+ * @param storageName storage name
+ * @param path partition spec path
+ * @return the string
+ */
+ @CliCommand(value = "fact update single-partition",
+ help = "update single partition to fact <fact_name>'s"
+ + " storage <storage_name>, reading spec from <partition-spec-path>"
+ + " The partition has to exist to be eligible for updation.")
+ public String updatePartitionOfFact(
+ @CliOption(key = {"", "fact_name"}, mandatory = true, help = "<fact_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-spec-path>") @NonNull final File path) {
+ return updatePartition(tableName, storageName, path);
+ }
@CliCommand(value = "fact add partitions",
help = "add multiple partition to fact <fact_name>'s"
@@ -237,6 +258,17 @@ public class LensFactCommands extends PhysicalTableCrudCommand<XFactTable> {
return addPartitions(tableName, storageName, path);
}
+ @CliCommand(value = "fact update partitions",
+ help = "update multiple partition of fact <fact_name>'s"
+ + " storage <storage_name>, reading partition list spec from <partition-list-spec-path>"
+ +" The partitions have to exist to be eligible for updation.")
+ public String updatePartitionsOfFact(
+ @CliOption(key = {"", "fact_name"}, mandatory = true, help = "<fact_name>") String tableName,
+ @CliOption(key = {"", "storage_name"}, mandatory = true, help = "<storage_name>") String storageName,
+ @CliOption(key = {"", "path"}, mandatory = true, help = "<partition-list-spec-path>") String path) {
+ return updatePartitions(tableName, storageName, path);
+ }
+
@CliCommand(value = "fact timelines",
help = "get timelines for fact. Can optionally specify storage, update period and time dimension to filter by."
+ " Instead of time dimension, partition column can be directly passed as <time_dimension>")
@@ -294,6 +326,16 @@ public class LensFactCommands extends PhysicalTableCrudCommand<XFactTable> {
}
@Override
+ protected APIResult doUpdatePartition(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionOfFact(tableName, storageName, validPath);
+ }
+
+ @Override
+ protected APIResult doUpdatePartitions(String tableName, String storageName, String validPath) {
+ return getClient().updatePartitionsOfFact(tableName, storageName, validPath);
+ }
+
+ @Override
public List<String> getAll() {
return getClient().getAllFactTables();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
index a479c14..c505680 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/PhysicalTableCrudCommand.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cli.commands;
+import java.io.File;
import java.io.IOException;
import java.util.List;
@@ -84,10 +85,20 @@ public abstract class PhysicalTableCrudCommand<T> extends LensCRUDCommand<T> {
return doAddPartition(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
}
+ public String updatePartition(String tableName, String storageName, File path) {
+ // TODO: path.getPath() to be changed to path after LENS-634
+ return doUpdatePartition(tableName, storageName, getValidPath(path.getPath(), false, true)).toString()
+ .toLowerCase();
+ }
+
public String addPartitions(String tableName, String storageName, String path) {
return doAddPartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
}
+ public String updatePartitions(String tableName, String storageName, String path) {
+ return doUpdatePartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
+ }
+
public String dropPartitions(String tableName, String storageName, String filter) {
return doDropPartitions(tableName, storageName, filter).toString().toLowerCase();
}
@@ -111,4 +122,8 @@ public abstract class PhysicalTableCrudCommand<T> extends LensCRUDCommand<T> {
protected abstract APIResult doAddPartitions(String tableName, String storageName, String path);
protected abstract APIResult doDropPartitions(String tableName, String storageName, String filter);
+
+ protected abstract APIResult doUpdatePartition(String tableName, String storageName, String validPath);
+
+ protected abstract APIResult doUpdatePartitions(String tableName, String storageName, String validPath);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java b/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
index 2de0cad..8647ac3 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/LensCliApplicationTest.java
@@ -31,6 +31,8 @@ import org.testng.annotations.BeforeTest;
* The Class LensCliApplicationTest.
*/
public class LensCliApplicationTest extends LensAllApplicationJerseyTest {
+ public static final String SUCCESS_MESSAGE = "<?xml version=\"1.0\" encoding=\"utf-8\" "
+ + "standalone=\"yes\"?><result><status>succeeded</status><message></message></result>";
@Override
protected int getTestPort() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
index a87d0b7..56e962d 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensDimensionTableCommands.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.cli;
+import static org.testng.Assert.*;
+
import java.io.*;
import java.net.URISyntaxException;
import java.net.URL;
@@ -28,7 +30,6 @@ import org.apache.lens.cli.commands.LensDimensionCommands;
import org.apache.lens.cli.commands.LensDimensionTableCommands;
import org.apache.lens.client.LensClient;
-import org.testng.Assert;
import org.testng.annotations.Test;
import lombok.extern.slf4j.Slf4j;
@@ -104,8 +105,8 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
throws IOException {
LensDimensionTableCommands command = getCommand();
String dimList = command.showDimensionTables(null);
- Assert.assertEquals(dimList, "No dimensiontable found");
- Assert.assertEquals(command.showDimensionTables("test_dim"), "No dimensiontable found for test_dim");
+ assertEquals(dimList, "No dimensiontable found");
+ assertEquals(command.showDimensionTables("test_dim"), "No dimensiontable found for test_dim");
// add local storage before adding fact table
TestLensStorageCommands.addLocalStorage(storageName);
@@ -115,24 +116,24 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
command.createDimensionTable(new File(dimSpec.toURI()).getAbsolutePath());
} catch (Exception e) {
log.error("Unable to create dimtable", e);
- Assert.fail("Unable to create dimtable" + e.getMessage());
+ fail("Unable to create dimtable" + e.getMessage());
}
dimList = command.showDimensionTables(null);
- Assert.assertEquals(command.showDimensionTables("test_dim"), dimList);
+ assertEquals(command.showDimensionTables("test_dim"), dimList);
try {
- Assert.assertEquals(command.showDimensionTables("blah"), dimList);
- Assert.fail();
+ assertEquals(command.showDimensionTables("blah"), dimList);
+ fail();
} catch (NotFoundException e) {
log.info("blah is not a table", e);
}
try {
- Assert.assertEquals(command.showDimensionTables("dim_table2"), dimList);
- Assert.fail();
+ assertEquals(command.showDimensionTables("dim_table2"), dimList);
+ fail();
} catch (NotFoundException e) {
log.info("dim_table2 is a table, but not a dimension", e);
}
- Assert.assertTrue(dimList.contains(tableName), "dim_table table should be found");
+ assertTrue(dimList.contains(tableName), "dim_table table should be found");
}
/**
@@ -168,13 +169,13 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
String propString = "name : dim2.prop value : d2";
String propString1 = "name : dim2.prop value : d1";
String propString2 = "name : dim2.prop1 value : d2";
- Assert.assertTrue(desc.contains(propString));
+ assertTrue(desc.contains(propString));
command.updateDimensionTable("dim_table2", "/tmp/local-dim1.xml");
desc = command.describeDimensionTable("dim_table2");
log.debug(desc);
- Assert.assertTrue(desc.contains(propString1));
- Assert.assertTrue(desc.contains(propString2));
+ assertTrue(desc.contains(propString1));
+ assertTrue(desc.contains(propString2));
} finally {
newFile.delete();
@@ -189,16 +190,16 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
private static void testDimStorageActions() throws URISyntaxException {
LensDimensionTableCommands command = getCommand();
String result = command.getDimStorages("dim_table2");
- Assert.assertEquals(DIM_LOCAL, result);
+ assertEquals(DIM_LOCAL, result);
command.dropAllDimStorages("dim_table2");
result = command.getDimStorages("dim_table2");
- Assert.assertEquals(result, "No storage found for dim_table2");
+ assertEquals(result, "No storage found for dim_table2");
addLocalStorageToDim();
result = command.getDimStorages("dim_table2");
- Assert.assertNotEquals(result, "No storage found for dim_table2");
+ assertNotEquals(result, "No storage found for dim_table2");
command.dropStorageFromDim("dim_table2", DIM_LOCAL);
result = command.getDimStorages("dim_table2");
- Assert.assertEquals(result, "No storage found for dim_table2");
+ assertEquals(result, "No storage found for dim_table2");
addLocalStorageToDim();
}
@@ -213,46 +214,70 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
URL resource = TestLensDimensionTableCommands.class.getClassLoader().getResource("dim-local-storage-element.xml");
command.addNewDimStorage("dim_table2", new File(resource.toURI()).getAbsolutePath());
result = command.getDimStorages("dim_table2");
- Assert.assertEquals(DIM_LOCAL, result);
+ assertEquals(DIM_LOCAL, result);
result = command.getStorageFromDim("dim_table2", DIM_LOCAL);
String partString = "DAILY";
- Assert.assertTrue(result.contains(partString));
+ assertTrue(result.contains(partString));
}
/**
* Test dim partition actions.
*/
- private static void testDimPartitionActions() {
+ private static void testDimPartitionActions() throws URISyntaxException {
LensDimensionTableCommands command = getCommand();
- String result;
- result = command.getAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
- Assert.assertTrue(result.trim().isEmpty());
+ assertTrue(command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null).trim().isEmpty());
+ //TODO: remove getAbsolutePath()
+ String singlePartPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("dim1-local-part.xml").toURI()).getAbsolutePath();
+ String multiplePartsPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("dim1-local-parts.xml").toURI()).getAbsolutePath();
+
+ assertTrue(command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null).trim().isEmpty());
+
+ assertEquals(command.addPartitionToDimtable("dim_table2", DIM_LOCAL, singlePartPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionOfDimtable("dim_table2", DIM_LOCAL, new File(singlePartPath)), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+ assertEquals(command.addPartitionsToDimtable("dim_table2", DIM_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionsOfDimtable("dim_table2", DIM_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+
+ // Wrong files:
try {
- command.addPartitionToDimtable("dim_table2", DIM_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("dim1-local-part.xml").toURI()).getAbsolutePath());
+ command.addPartitionToDimtable("dim_table2", DIM_LOCAL, multiplePartsPath);
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
- Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
+ // pass
}
- verifyAndDeletePartition();
try {
- command.addPartitionsToDimtable("dim_table2", DIM_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("dim1-local-parts.xml").toURI()).getAbsolutePath());
+ command.updatePartitionOfDimtable("dim_table2", DIM_LOCAL, new File(multiplePartsPath));
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
- Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
+ // pass
+ }
+
+ try {
+ command.addPartitionsToDimtable("dim_table2", DIM_LOCAL, singlePartPath);
+ fail("Should fail");
+ } catch (Throwable t) {
+ // pass
+ }
+
+ try {
+ command.updatePartitionsOfDimtable("dim_table2", DIM_LOCAL, singlePartPath);
+ fail("Should fail");
+ } catch (Throwable t) {
+ // pass
}
- verifyAndDeletePartition();
}
- private static void verifyAndDeletePartition() {
- String result = command.getAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
+ private static void verifyAndDeletePartitions() {
+ String result = command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null);
String partString = "DAILY";
- Assert.assertTrue(result.contains(partString));
+ assertTrue(result.contains(partString));
command.dropAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
- result = command.getAllPartitionsOfDim("dim_table2", DIM_LOCAL, null);
- Assert.assertTrue(result.trim().isEmpty());
+ result = command.getAllPartitionsOfDimtable("dim_table2", DIM_LOCAL, null);
+ assertTrue(result.trim().isEmpty());
}
/**
@@ -269,7 +294,7 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
command.addPartitionToDimtable(tableName, storageName, new File(resource.toURI()).getAbsolutePath());
} catch (Throwable t) {
log.error("Unable to locate the storage part file for adding new storage to dim table dim_table2", t);
- Assert.fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
+ fail("Unable to locate the storage part file for adding new storage to dim table dim_table2");
}
}
@@ -279,10 +304,10 @@ public class TestLensDimensionTableCommands extends LensCliApplicationTest {
public static void dropDim1Table() {
LensDimensionTableCommands command = getCommand();
String dimList = command.showDimensionTables(null);
- Assert.assertEquals("dim_table2", dimList, "dim table should be found");
+ assertEquals("dim_table2", dimList, "dim table should be found");
command.dropDimensionTable("dim_table2", false);
dimList = command.showDimensionTables(null);
- Assert.assertEquals(dimList, "No dimensiontable found", "Dim tables should not be found");
+ assertEquals(dimList, "No dimensiontable found", "Dim tables should not be found");
TestLensStorageCommands.dropStorage(DIM_LOCAL);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
index 13bfbd2..fa342c8 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensFactCommands.java
@@ -222,40 +222,44 @@ public class TestLensFactCommands extends LensCliApplicationTest {
/**
* Test fact partition actions.
*/
- private void testFactPartitionActions() {
+ private void testFactPartitionActions() throws URISyntaxException {
LensFactCommands command = getCommand();
- String result;
verifyEmptyTimelines();
- result = command.getAllPartitionsOfFact("fact1", FACT_LOCAL, null);
- assertTrue(result.trim().isEmpty());
+ assertTrue(command.getAllPartitionsOfFact("fact1", FACT_LOCAL, null).trim().isEmpty());
+ String singlePartPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("fact1-local-part.xml").toURI()).getAbsolutePath();
+ String multiplePartsPath = new File(
+ TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath();
+ assertEquals(command.addPartitionToFact("fact1", FACT_LOCAL, singlePartPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionOfFact("fact1", FACT_LOCAL, new File(singlePartPath)), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+ assertEquals(command.addPartitionsToFact("fact1", FACT_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ assertEquals(command.updatePartitionsOfFact("fact1", FACT_LOCAL, multiplePartsPath), SUCCESS_MESSAGE);
+ verifyAndDeletePartitions();
+
+ // Wrong files:
try {
- command.addPartitionToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-part.xml").toURI()).getAbsolutePath());
+ command.addPartitionToFact("fact1", FACT_LOCAL, multiplePartsPath);
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to fact table fact1", t);
- fail("Unable to locate the storage part file for adding new storage to fact table fact1");
+ // pass
}
- verifyAndDeletePartitions();
try {
- command.addPartitionsToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath());
+ command.updatePartitionOfFact("fact1", FACT_LOCAL, new File(multiplePartsPath));
+ fail("Should fail");
} catch (Throwable t) {
- log.error("Unable to locate the storage part file for adding new storage to fact table fact1", t);
- fail("Unable to locate the storage part file for adding new storage to fact table fact1");
+ // pass
}
- verifyAndDeletePartitions();
- // Wrong files:
try {
- command.addPartitionToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-parts.xml").toURI()).getAbsolutePath());
+ command.addPartitionsToFact("fact1", FACT_LOCAL, singlePartPath);
fail("Should fail");
} catch (Throwable t) {
// pass
}
+
try {
- command.addPartitionsToFact("fact1", FACT_LOCAL, new File(
- TestLensFactCommands.class.getClassLoader().getResource("fact1-local-part.xml").toURI()).getAbsolutePath());
+ command.updatePartitionsOfFact("fact1", FACT_LOCAL, singlePartPath);
fail("Should fail");
} catch (Throwable t) {
// pass
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/resources/dim1-local-parts.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/dim1-local-parts.xml b/lens-cli/src/test/resources/dim1-local-parts.xml
index 87fbc6d..8edc5c7 100644
--- a/lens-cli/src/test/resources/dim1-local-parts.xml
+++ b/lens-cli/src/test/resources/dim1-local-parts.xml
@@ -21,10 +21,7 @@
-->
<x_partition_list xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
-
- <partition fact_or_dimension_table_name="dim_table2" location="/tmp/dim_table2" update_period="DAILY"
- xmlns="uri:lens:cube:0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <partition fact_or_dimension_table_name="dim_table2" location="/tmp/dim_table2" update_period="DAILY">
<time_partition_spec>
<part_spec_element key="dt" value="2014-03-27T12:00:00"/>
</time_partition_spec>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cli/src/test/resources/fact1-local-parts.xml
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/resources/fact1-local-parts.xml b/lens-cli/src/test/resources/fact1-local-parts.xml
index 6201320..9ca7d1f 100644
--- a/lens-cli/src/test/resources/fact1-local-parts.xml
+++ b/lens-cli/src/test/resources/fact1-local-parts.xml
@@ -21,9 +21,7 @@
-->
<x_partition_list xmlns="uri:lens:cube:0.1"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
- <partition fact_or_dimension_table_name="fact1" location="/tmp/fact1" update_period="HOURLY"
- xmlns="uri:lens:cube:0.1"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <partition fact_or_dimension_table_name="fact1" location="/tmp/fact1" update_period="HOURLY">
<time_partition_spec>
<part_spec_element key="dt" value="2014-03-27T12:00:00"/>
</time_partition_spec>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index b15b024..d7df1ba 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -530,6 +530,37 @@ public class LensClient {
public APIResult addPartitionsToDim(String table, String storage, String partsSpec) {
return mc.addPartitionsToDimensionTable(table, storage, partsSpec);
}
+ public APIResult updatePartitionOfFact(String table, String storage, String partSpec) {
+ return mc.updatePartitionOfFactTable(table, storage, partSpec);
+ }
+
+ public APIResult updatePartitionsOfFact(String table, String storage, String partsSpec) {
+ return mc.updatePartitionsOfFactTable(table, storage, partsSpec);
+ }
+
+ public APIResult updatePartitionOfFact(String table, String storage, XPartition xp) {
+ return mc.updatePartitionOfFactTable(table, storage, xp);
+ }
+
+ public APIResult updatePartitionsOfFact(String table, String storage, XPartitionList xpList) {
+ return mc.updatePartitionsOfFactTable(table, storage, xpList);
+ }
+
+ public APIResult updatePartitionOfDim(String table, String storage, String partSpec) {
+ return mc.updatePartitionOfDimensionTable(table, storage, partSpec);
+ }
+
+ public APIResult updatePartitionOfDim(String table, String storage, XPartition xp) {
+ return mc.updatePartitionOfDimensionTable(table, storage, xp);
+ }
+
+ public APIResult updatePartitionsOfDim(String table, String storage, XPartitionList xpList) {
+ return mc.updatePartitionsOfDimensionTable(table, storage, xpList);
+ }
+
+ public APIResult updatePartitionsOfDim(String table, String storage, String partsSpec) {
+ return mc.updatePartitionsOfDimensionTable(table, storage, partsSpec);
+ }
public QueryPrepareHandle prepare(String sql, String queryName) {
return statement.prepareQuery(sql, queryName);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index b4e0552..dba13ca 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -967,6 +967,101 @@ public class LensMetadataClient {
return new APIResult(Status.FAILED, "File not found");
}
}
+ public APIResult updatePartitionOfDimensionTable(String dimTblName, String storage,
+ XPartition partition) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("dimtables").path(dimTblName)
+ .path("storages").path(storage).path("partition")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartition(partition)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionOfDimensionTable(String dimTblName, String storage,
+ String partitionSpec) {
+ try {
+ return updatePartitionOfDimensionTable(dimTblName, storage, (XPartition) readFromXML(partitionSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
+
+ public APIResult updatePartitionsOfDimensionTable(String dimTblName, String storage,
+ XPartitionList partitions) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("dimtables").path(dimTblName)
+ .path("storages").path(storage).path("partitions")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartitionList(partitions)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionsOfDimensionTable(String dimTblName, String storage,
+ String partitionsSpec) {
+ try {
+ return updatePartitionsOfDimensionTable(dimTblName, storage, (XPartitionList) readFromXML(partitionsSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
+
+ public APIResult updatePartitionOfFactTable(String fact, String storage,
+ XPartition partition) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("facts").path(fact)
+ .path("storages").path(storage).path("partition")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartition(partition)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionOfFactTable(String fact, String storage,
+ String partitionSpec) {
+ try {
+ return updatePartitionOfFactTable(fact, storage, (XPartition) readFromXML(partitionSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
+
+ public APIResult updatePartitionsOfFactTable(String fact, String storage,
+ XPartitionList partitions) {
+ WebTarget target = getMetastoreWebTarget();
+ APIResult result = target.path("facts").path(fact)
+ .path("storages").path(storage).path("partitions")
+ .queryParam("sessionid", this.connection.getSessionHandle())
+ .request(MediaType.APPLICATION_XML)
+ .put(Entity.xml(objFact.createXPartitionList(partitions)), APIResult.class);
+ return result;
+ }
+
+ public APIResult updatePartitionsOfFactTable(String fact, String storage,
+ String partitionsSpec) {
+ try {
+ return updatePartitionsOfFactTable(fact, storage, (XPartitionList) readFromXML(partitionsSpec));
+ } catch (JAXBException e) {
+ LOG.info("Unmarshalling error:", e);
+ return new APIResult(Status.FAILED, "Unmarshalling failed");
+ } catch (IOException e) {
+ LOG.info("File error:", e);
+ return new APIResult(Status.FAILED, "File not found");
+ }
+ }
public Date getLatestDateOfCube(String cubeName, String timePartition) {
return getMetastoreWebTarget().path("cubes").path(cubeName).path("latestdate")
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index dc2d9f6..c8abca0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -34,10 +34,8 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
+import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.thrift.TException;
@@ -172,6 +170,67 @@ public class CubeMetastoreClient {
return ret;
}
+ public void updatePartition(String fact, String storageName, Partition partition)
+ throws HiveException, InvalidOperationException {
+ updatePartitions(fact, storageName, Collections.singletonList(partition));
+ }
+
+ public void updatePartitions(String factOrDimtableName, String storageName, List<Partition> partitions)
+ throws HiveException, InvalidOperationException {
+ List<Partition> partitionsToAlter = Lists.newArrayList();
+ partitionsToAlter.addAll(partitions);
+ partitionsToAlter.addAll(getAllLatestPartsEquivalentTo(factOrDimtableName, storageName, partitions));
+ getStorage(storageName).updatePartitions(getClient(), factOrDimtableName, partitionsToAlter);
+ }
+
+ private List<Partition> getAllLatestPartsEquivalentTo(String factOrDimtableName, String storageName,
+ List<Partition> partitions) throws HiveException {
+ if (isFactTable(factOrDimtableName)) {
+ return Lists.newArrayList();
+ }
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factOrDimtableName, storageName);
+ Table storageTable = getTable(storageTableName);
+ List<String> timePartCols = getTimePartColNamesOfTable(storageTable);
+ List<Partition> latestParts = Lists.newArrayList();
+ for (Partition partition : partitions) {
+ LinkedHashMap<String, String> partSpec = partition.getSpec();
+ LinkedHashMap<String, String> timePartSpec = Maps.newLinkedHashMap();
+ LinkedHashMap<String, String> nonTimePartSpec = Maps.newLinkedHashMap();
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
+ if (timePartCols.contains(entry.getKey())) {
+ timePartSpec.put(entry.getKey(), entry.getValue());
+ } else {
+ nonTimePartSpec.put(entry.getKey(), entry.getValue());
+ }
+ }
+ for (String timePartCol : timePartCols) {
+ Partition latestPart = getLatestPart(storageTableName, timePartCol, nonTimePartSpec);
+ if (latestPart != null) {
+ LinkedHashMap<String, String> latestPartSpec = latestPart.getSpec();
+ latestPartSpec.put(timePartCol, partSpec.get(timePartCol));
+ if (partSpec.equals(latestPartSpec)) {
+ latestPart.getParameters().putAll(partition.getParameters());
+ latestPart.getParameters().put(MetastoreUtil.getLatestPartTimestampKey(timePartCol),
+ partSpec.get(timePartCol));
+ latestPart.getTPartition().getSd().getSerdeInfo().getParameters().putAll(
+ partition.getTPartition().getSd().getSerdeInfo().getParameters());
+ latestPart.setLocation(partition.getLocation());
+ latestPart.setInputFormatClass(partition.getInputFormatClass());
+ latestPart.setOutputFormatClass(partition.getOutputFormatClass());
+ // the following is a fix because hive has a bug: https://issues.apache.org/jira/browse/HIVE-11278.
+ latestPart.getTPartition().getSd().setOutputFormat(
+ HiveFileFormatUtils.getOutputFormatSubstitute(partition.getOutputFormatClass(), false).getName());
+ latestPart.getTPartition().getSd().getSerdeInfo()
+ .setSerializationLib(partition.getTPartition().getSd().getSerdeInfo().getSerializationLib());
+ latestParts.add(latestPart);
+ }
+ }
+ }
+ }
+ return latestParts;
+ }
+
+
/**
* In-memory storage of {@link PartitionTimeline} objects for each valid
* storagetable-updateperiod-partitioncolumn tuple. also simultaneously stored in metastore table of the
@@ -257,13 +316,13 @@ public class CubeMetastoreClient {
if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(
storage) != null) {
for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
- for (String partCol : getTimePartsOfTable(storageTable)) {
+ for (String partCol : getTimePartColNamesOfTable(storageTable)) {
ensureEntry(storageTableName, updatePeriod, partCol);
}
}
}
// Then add all existing partitions for batch addition in respective timelines.
- List<String> timeParts = getTimePartsOfTable(storageTable);
+ List<String> timeParts = getTimePartColNamesOfTable(storageTable);
List<FieldSchema> partCols = storageTable.getPartCols();
for (Partition partition : getPartitionsByFilter(storageTableName, null)) {
UpdatePeriod period = deduceUpdatePeriod(partition);
@@ -290,7 +349,7 @@ public class CubeMetastoreClient {
String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
log.info("loading from table properties: " + storageTableName);
for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
- for (String partCol : getTimePartsOfTable(storageTableName)) {
+ for (String partCol : getTimePartColNamesOfTable(storageTableName)) {
ensureEntry(storageTableName, updatePeriod, partCol).init(getTable(storageTableName));
}
}
@@ -723,7 +782,7 @@ public class CubeMetastoreClient {
* @throws HiveException
*/
public void addPartition(StoragePartitionDesc partSpec, String storageName) throws HiveException, LensException {
- addPartitions(Arrays.asList(partSpec), storageName);
+ addPartitions(Collections.singletonList(partSpec), storageName);
}
/** batch addition */
@@ -731,9 +790,9 @@ public class CubeMetastoreClient {
throws HiveException, LensException {
for (Map.Entry<String, Map<UpdatePeriod, List<StoragePartitionDesc>>> group : groupPartitionDescs(
storagePartitionDescs).entrySet()) {
- String fact = group.getKey();
+ String factOrDimtable = group.getKey();
for (Map.Entry<UpdatePeriod, List<StoragePartitionDesc>> entry : group.getValue().entrySet()) {
- addPartitions(fact, storageName, entry.getKey(), entry.getValue());
+ addPartitions(factOrDimtable, storageName, entry.getKey(), entry.getValue());
}
}
}
@@ -1052,7 +1111,7 @@ public class CubeMetastoreClient {
public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException {
- String storageTableName = MetastoreUtil.getFactStorageTableName(factName, storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName);
return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec);
}
@@ -1086,6 +1145,15 @@ public class CubeMetastoreClient {
return getClient().getPartitions(getHiveTable(storageTableName));
}
+ public Partition getPartitionByFilter(String storageTableName, String filter) throws HiveException {
+ List<Partition> parts = getPartitionsByFilter(storageTableName, filter);
+ if (parts.size() != 1) {
+ throw new HiveException(
+ "filter " + filter + " did not result in unique partition. Got " + parts.size() + "partitions");
+ }
+ return parts.iterator().next();
+ }
+
public List<Partition> getPartitionsByFilter(String storageTableName, String filter) throws HiveException {
try {
return getClient().getPartitionsByFilter(getTable(storageTableName), filter);
@@ -1116,7 +1184,7 @@ public class CubeMetastoreClient {
boolean dimPartitionExists(String dimTblName, String storageName, Map<String, Date> partitionTimestamps)
throws HiveException {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTblName, storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName);
return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName),
partitionTimestamps);
}
@@ -1164,12 +1232,12 @@ public class CubeMetastoreClient {
return getTable(tableName);
}
- private List<String> getTimePartsOfTable(String storageTableName) throws HiveException {
- return getTimePartsOfTable(getTable(storageTableName));
+ public List<String> getTimePartColNamesOfTable(String storageTableName) throws HiveException {
+ return getTimePartColNamesOfTable(getTable(storageTableName));
}
/** extract from table properties */
- public List<String> getTimePartsOfTable(Table table) {
+ public List<String> getTimePartColNamesOfTable(Table table) {
List<String> ret = Arrays.asList(StringUtils.split(table.getParameters().get(MetastoreConstants.TIME_PART_COLUMNS),
","));
return ret == null ? new ArrayList<String>() : ret;
@@ -1380,7 +1448,8 @@ public class CubeMetastoreClient {
if (dimTable.getStorages() != null && !dimTable.getStorages().isEmpty()) {
for (String storageName : dimTable.getStorages()) {
if (dimTable.hasStorageSnapshots(storageName)) {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTable.getName(), storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(),
+ storageName);
if (dimLatestPartitionExists(storageTableName,
getDimension(dimTable.getDimName()).getTimedDimension())) {
latestLookupCache.add(storageTableName.trim().toLowerCase());
@@ -1976,7 +2045,7 @@ public class CubeMetastoreClient {
public void dropStorageFromFact(String factName, String storage) throws HiveException {
CubeFactTable cft = getFactTable(factName);
cft.dropStorage(storage);
- dropHiveTable(MetastoreUtil.getFactStorageTableName(factName, storage));
+ dropHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storage));
alterCubeTable(factName, getTable(factName), cft);
updateFactCache(factName);
}
@@ -1984,7 +2053,7 @@ public class CubeMetastoreClient {
// updateFact will be false when fact is fully dropped
private void dropStorageFromFact(String factName, String storage, boolean updateFact) throws HiveException {
CubeFactTable cft = getFactTable(factName);
- dropHiveTable(MetastoreUtil.getFactStorageTableName(factName, storage));
+ dropHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storage));
if (updateFact) {
cft.dropStorage(storage);
alterCubeTable(factName, getTable(factName), cft);
@@ -2006,7 +2075,7 @@ public class CubeMetastoreClient {
// updateDimTbl will be false when dropping dimTbl
private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws HiveException {
CubeDimensionTable cdt = getDimensionTable(dimTblName);
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTblName, storage);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storage);
dropHiveTable(storageTableName);
latestLookupCache.remove(storageTableName.trim().toLowerCase());
if (updateDimTbl) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
index 8107f4c..c4c3e9b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/HDFSStorage.java
@@ -21,6 +21,7 @@ package org.apache.lens.cube.metadata;
import java.util.List;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
public class HDFSStorage extends Storage {
@@ -52,6 +53,21 @@ public class HDFSStorage extends Storage {
}
@Override
+ public void preUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
+ public void commitUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
+ public void rollbackUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
public void preDropPartition(String storageTableName, List<String> partVals) throws HiveException {
// No op
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index 8e2c9ed..8fbfe63 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -34,14 +34,10 @@ public class MetastoreUtil {
}
- public static final String getFactStorageTableName(String factName, String storageName) {
+ public static final String getFactOrDimtableStorageTableName(String factName, String storageName) {
return getStorageTableName(factName, Storage.getPrefix(storageName));
}
- public static final String getDimStorageTableName(String dimName, String storageName) {
- return getStorageTableName(dimName, Storage.getPrefix(storageName));
- }
-
public static final String getStorageTableName(String cubeTableName, String storagePrefix) {
return storagePrefix + cubeTableName;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
index 1174d25..317c673 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/PartitionMetahook.java
@@ -21,11 +21,12 @@ package org.apache.lens.cube.metadata;
import java.util.List;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
public interface PartitionMetahook {
/**
- * Called before calling add partition
+ * Called before calling add partition(s)
*
* @param storagePartitionDesc
* @throws HiveException
@@ -33,7 +34,7 @@ public interface PartitionMetahook {
void preAddPartitions(List<StoragePartitionDesc> storagePartitionDesc) throws HiveException;
/**
- * Called after successfully adding the partition
+ * Called after successfully adding the partition(s)
*
* @param storagePartitionDesc
* @throws HiveException
@@ -49,6 +50,30 @@ public interface PartitionMetahook {
void rollbackAddPartitions(List<StoragePartitionDesc> storagePartitionDesc) throws HiveException;
/**
+ * Called before calling update partition(s)
+ *
+ * @param partitions
+ * @throws HiveException
+ */
+ void preUpdatePartition(List<Partition> partitions) throws HiveException;
+
+ /**
+ * Called after successfully updating partition(s)
+ *
+ * @param partitions
+ * @throws HiveException
+ */
+ void commitUpdatePartition(List<Partition> partitions) throws HiveException;
+
+ /**
+ * Called after failed updating partition(s)
+ *
+ * @param partitions
+ * @throws HiveException
+ */
+ void rollbackUpdatePartition(List<Partition> partitions) throws HiveException;
+
+ /**
* Called before calling drop partition
*
* @param storageTableName
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index c754042..873aa49 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -28,6 +28,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
import org.apache.hadoop.hive.ql.metadata.*;
@@ -348,16 +349,53 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
}
}
client.createPartitions(addParts);
- commitAddPartitions(storagePartitionDescs);
success = true;
} finally {
- if (!success) {
+ if (success) {
+ commitAddPartitions(storagePartitionDescs);
+ } else {
rollbackAddPartitions(storagePartitionDescs);
}
}
}
/**
+ * Update existing partition
+ * @param client hive client instance
+ * @param fact fact name
+ * @param partition partition to be updated
+ * @throws InvalidOperationException
+ * @throws HiveException
+ */
+ public void updatePartition(Hive client, String fact, Partition partition)
+ throws InvalidOperationException, HiveException {
+ client.alterPartition(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partition);
+ }
+
+ /**
+ * Update existing partitions
+ * @param client hive client instance
+ * @param fact fact name
+ * @param partitions partitions to be updated
+ * @throws InvalidOperationException
+ * @throws HiveException
+ */
+ public void updatePartitions(Hive client, String fact, List<Partition> partitions)
+ throws InvalidOperationException, HiveException {
+ boolean success = false;
+ try {
+ client.alterPartitions(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partitions);
+ success = true;
+ } finally {
+ if (success) {
+ commitUpdatePartition(partitions);
+ } else {
+ rollbackUpdatePartition(partitions);
+ }
+ }
+ }
+
+ /**
* Drop the partition in the underlying hive table and update latest partition link
*
* @param client The metastore client
@@ -413,10 +451,11 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
}
}
}
- commitDropPartition(storageTableName, partVals);
success = true;
} finally {
- if (!success) {
+ if (success) {
+ commitDropPartition(storageTableName, partVals);
+ } else {
rollbackDropPartition(storageTableName, partVals);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
index 7c966d8..11e6ef6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/TimePartition.java
@@ -24,8 +24,6 @@ import java.util.Date;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.lang3.time.DateUtils;
-
import lombok.Data;
import lombok.NonNull;
@@ -41,7 +39,7 @@ public class TimePartition implements Comparable<TimePartition>, Named {
this.updatePeriod = updatePeriod;
Calendar cal = Calendar.getInstance();
cal.setTime(date);
- this.date = truncate(date, updatePeriod);
+ this.date = updatePeriod.truncate(date);
this.dateString = updatePeriod.format().format(this.date);
}
@@ -109,22 +107,6 @@ public class TimePartition implements Comparable<TimePartition>, Named {
return this.date.after(when.date);
}
- private Date truncate(Date date, UpdatePeriod updatePeriod) {
- if (updatePeriod.equals(UpdatePeriod.WEEKLY)) {
- Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH);
- Calendar cal = Calendar.getInstance();
- cal.setTime(truncDate);
- cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
- return cal.getTime();
- } else if (updatePeriod.equals(UpdatePeriod.QUARTERLY)) {
- Date dt = DateUtils.truncate(date, updatePeriod.calendarField());
- dt.setMonth(dt.getMonth() - dt.getMonth() % 3);
- return dt;
- } else {
- return DateUtils.truncate(date, updatePeriod.calendarField());
- }
- }
-
protected static String getWrongUpdatePeriodMessage(UpdatePeriod up, String dateString) {
return String.format(UPDATE_PERIOD_WRONG_ERROR_MESSAGE, up, dateString);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
index 966dbdf..253323b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
@@ -23,11 +23,14 @@ import static java.util.Calendar.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.util.Calendar;
import java.util.Comparator;
import java.util.Date;
import org.apache.lens.cube.parse.DateUtil;
+import org.apache.commons.lang3.time.DateUtils;
+
public enum UpdatePeriod implements Named {
SECONDLY(SECOND, 1000, 1.4f, "yyyy-MM-dd-HH-mm-ss"),
MINUTELY(MINUTE, 60 * SECONDLY.weight(), 1.35f, "yyyy-MM-dd-HH-mm"),
@@ -214,6 +217,22 @@ public enum UpdatePeriod implements Named {
return normalizationFactor;
}
+ public Date truncate(Date date) {
+ if (this.equals(UpdatePeriod.WEEKLY)) {
+ Date truncDate = DateUtils.truncate(date, Calendar.DAY_OF_MONTH);
+ Calendar cal = Calendar.getInstance();
+ cal.setTime(truncDate);
+ cal.set(Calendar.DAY_OF_WEEK, cal.getFirstDayOfWeek());
+ return cal.getTime();
+ } else if (this.equals(UpdatePeriod.QUARTERLY)) {
+ Date dt = DateUtils.truncate(date, this.calendarField());
+ dt.setMonth(dt.getMonth() - dt.getMonth() % 3);
+ return dt;
+ } else {
+ return DateUtils.truncate(date, this.calendarField());
+ }
+ }
+
public static class UpdatePeriodComparator implements Comparator<UpdatePeriod> {
@Override
public int compare(UpdatePeriod o1, UpdatePeriod o2) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index ce6f434..373c80a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.cube.parse;
+import static org.apache.lens.cube.metadata.MetastoreUtil.getFactOrDimtableStorageTableName;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode.*;
import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipStorageCode.PART_COL_DOES_NOT_EXIST;
@@ -182,7 +183,7 @@ class StorageTableResolver implements ContextRewriter {
Map<String, SkipStorageCause> skipStorageCauses = new HashMap<String, SkipStorageCause>();
for (String storage : dimtable.getStorages()) {
if (isStorageSupported(storage)) {
- String tableName = MetastoreUtil.getDimStorageTableName(dimtable.getName(), storage).toLowerCase();
+ String tableName = getFactOrDimtableStorageTableName(dimtable.getName(), storage).toLowerCase();
if (validDimTables != null && !validDimTables.contains(tableName)) {
LOG.info("Not considering dim storage table:" + tableName + " as it is not a valid dim storage");
skipStorageCauses.put(tableName, new SkipStorageCause(SkipStorageCode.INVALID));
@@ -307,7 +308,7 @@ class StorageTableResolver implements ContextRewriter {
}
String getStorageTableName(CubeFactTable fact, String storage, List<String> validFactStorageTables) {
- String tableName = MetastoreUtil.getFactStorageTableName(fact.getName(), storage).toLowerCase();
+ String tableName = getFactOrDimtableStorageTableName(fact.getName(), storage).toLowerCase();
if (validFactStorageTables != null && !validFactStorageTables.contains(tableName)) {
LOG.info("Skipping storage table " + tableName + " as it is not valid");
return null;
@@ -370,7 +371,7 @@ class StorageTableResolver implements ContextRewriter {
String partCol = range.getPartitionColumn();
boolean partColNotSupported = rangeParts.isEmpty();
for(String storage: cfact.fact.getStorages()) {
- String storageTableName = MetastoreUtil.getFactStorageTableName(cfact.fact.getName(), storage).toLowerCase();
+ String storageTableName = getFactOrDimtableStorageTableName(cfact.fact.getName(), storage).toLowerCase();
partColNotSupported &= skipStorageCauses.containsKey(storageTableName)
&& skipStorageCauses.get(storageTableName).getCause().equals(PART_COL_DOES_NOT_EXIST)
&& skipStorageCauses.get(storageTableName).getNonExistantPartCols().contains(partCol);
[14/50] [abbrv] incubator-lens git commit: LENS-650 :Move pre-submit
hook out of User Config Loader into DriverQueryHook
Posted by sh...@apache.org.
LENS-650 :Move pre-submit hook out of User Config Loader into DriverQueryHook
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c7261d12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c7261d12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c7261d12
Branch: refs/heads/LENS-581
Commit: c7261d12b622d5990bd80edffe2a1654ba1a2a6d
Parents: 689c9a9
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Jul 20 12:21:37 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Jul 20 12:21:37 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 24 ++++-----
.../apache/lens/driver/hive/TestHiveDriver.java | 17 +++----
.../lens/driver/hive/TestRemoteHiveDriver.java | 14 +++---
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 32 ++++++------
.../driver/jdbc/JDBCDriverConfConstants.java | 1 +
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 17 +++----
.../lens/server/api/LensConfConstants.java | 2 +-
.../lens/server/api/driver/DriverQueryHook.java | 50 +++++++++++++++++++
.../lens/server/api/driver/LensDriver.java | 11 ++---
.../server/api/driver/NoOpDriverQueryHook.java | 35 +++++++++++++
.../lens/server/api/user/UserConfigLoader.java | 37 +++-----------
.../lens/server/api/driver/MockDriver.java | 6 +--
.../server/api/user/MockDriverQueryHook.java | 32 ++++++++++++
.../server/api/user/MockUserConfigLoader.java | 47 ------------------
.../server/query/QueryExecutionServiceImpl.java | 52 ++++++++++----------
.../server/user/DatabaseUserConfigLoader.java | 5 +-
.../lens/server/user/FixedUserConfigLoader.java | 25 +++++-----
.../user/PropertyBasedUserConfigLoader.java | 15 +++---
.../lens/server/user/FooBarConfigLoader.java | 11 +----
19 files changed, 222 insertions(+), 211 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index e87ed23..89d671f 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -44,7 +44,6 @@ import org.apache.lens.server.api.query.cost.QueryCostCalculator;
import org.apache.lens.server.api.query.priority.CostRangePriorityDecider;
import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
import org.apache.lens.server.api.query.priority.QueryPriorityDecider;
-import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -72,6 +71,8 @@ public class HiveDriver implements LensDriver {
/** The Constant HIVE_CONNECTION_CLASS. */
public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
+ public static final String HIVE_QUERY_HOOK_CLASS = "lens.driver.hive.query.hook.class";
+
/** The Constant HS2_CONNECTION_EXPIRY_DELAY. */
public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.hs2.connection.expiry.delay";
@@ -127,7 +128,7 @@ public class HiveDriver implements LensDriver {
QueryPriorityDecider queryPriorityDecider;
// package-local. Test case can change.
boolean whetherCalculatePriority;
- private UserConfigLoader userConfigLoader;
+ private DriverQueryHook queryHook;
private String sessionDbKey(String sessionHandle, String database) {
@@ -341,6 +342,13 @@ public class HiveDriver implements LensDriver {
queryPriorityDecider = new CostRangePriorityDecider(
new CostToPriorityRangeConf(driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES))
);
+ try {
+ queryHook = driverConf.getClass(
+ HIVE_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
+ ).newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
+ }
}
private QueryCost calculateQueryCost(AbstractQueryContext qctx) throws LensException {
@@ -350,6 +358,7 @@ public class HiveDriver implements LensDriver {
return new FactPartitionBasedQueryCost(Double.MAX_VALUE);
}
}
+
@Override
public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
log.info("Estimate: " + qctx.getDriverQuery(this));
@@ -520,9 +529,7 @@ public class HiveDriver implements LensDriver {
+ "User query: " + ctx.getUserQuery(), e);
}
}
- if (userConfigLoader != null) {
- userConfigLoader.preSubmit(ctx);
- }
+ queryHook.preLaunch(ctx);
OperationHandle op = getClient().executeStatementAsync(getSession(ctx), ctx.getSelectedDriverQuery(),
qdconf.getValByRegex(".*"));
ctx.setDriverOpHandle(op.toString());
@@ -758,11 +765,6 @@ public class HiveDriver implements LensDriver {
driverListeners.add(driverEventListener);
}
- @Override
- public void registerUserConfigLoader(UserConfigLoader userConfigLoader) {
- this.userConfigLoader = userConfigLoader;
- }
-
protected CLIServiceClient getClient() throws LensException {
if (isEmbedded) {
if (embeddedConnection == null) {
@@ -982,7 +984,7 @@ public class HiveDriver implements LensDriver {
listener.onCompletion(handle);
return;
}
- } catch(LensException e) {
+ } catch (LensException e) {
log.debug("query handle: {} Not yet launched on driver", handle);
}
Thread.sleep(pollInterval);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 67a4667..78b3320 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -18,9 +18,6 @@
*/
package org.apache.lens.driver.hive;
-import static org.apache.lens.server.api.user.MockUserConfigLoader.KEY;
-import static org.apache.lens.server.api.user.MockUserConfigLoader.VALUE;
-
import static org.testng.Assert.*;
import java.io.*;
@@ -40,7 +37,7 @@ import org.apache.lens.server.api.query.*;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.priority.CostRangePriorityDecider;
import org.apache.lens.server.api.query.priority.CostToPriorityRangeConf;
-import org.apache.lens.server.api.user.MockUserConfigLoader;
+import org.apache.lens.server.api.user.MockDriverQueryHook;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -56,6 +53,8 @@ import org.apache.hive.service.cli.ColumnDescriptor;
import org.testng.annotations.*;
+import com.google.common.collect.Lists;
+
/**
* The Class TestHiveDriver.
@@ -115,16 +114,12 @@ public class TestHiveDriver {
conf = new HiveConf();
conf.addResource("hivedriver-site.xml");
conf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
+ conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
conf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
driver.configure(conf);
- driver.registerUserConfigLoader(new MockUserConfigLoader(conf));
- drivers = new ArrayList<LensDriver>() {
- {
- add(driver);
- }
- };
+ drivers = Lists.<LensDriver>newArrayList(driver);
System.out.println("TestHiveDriver created");
}
@@ -477,7 +472,7 @@ public class TestHiveDriver {
*/
protected void validateExecuteAsync(QueryContext ctx, DriverQueryState finalState, boolean isPersistent,
boolean formatNulls) throws Exception {
- assertEquals(ctx.getSelectedDriverConf().get(KEY), VALUE);
+ assertEquals(ctx.getSelectedDriverConf().get(MockDriverQueryHook.KEY), MockDriverQueryHook.VALUE);
validateExecuteAsync(ctx, finalState, isPersistent, formatNulls, driver);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index c144fd8..98edc28 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -28,12 +28,13 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.driver.DriverQueryHook;
import org.apache.lens.server.api.driver.DriverQueryPlan;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.user.MockUserConfigLoader;
+import org.apache.lens.server.api.user.MockDriverQueryHook;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -46,6 +47,7 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
/**
@@ -134,13 +136,9 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
conf.addResource("hivedriver-site.xml");
driver = new HiveDriver();
conf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
+ conf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
driver.configure(conf);
- driver.registerUserConfigLoader(new MockUserConfigLoader(conf));
- drivers = new ArrayList<LensDriver>() {
- {
- add(driver);
- }
- };
+ drivers = Lists.<LensDriver>newArrayList(driver);
System.out.println("TestRemoteHiveDriver created");
}
@@ -198,7 +196,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
}
Thread.sleep(POLL_DELAY);
} catch (LensException e) {
- log.error("Got Exception " +e.getCause(), e);
+ log.error("Got Exception " + e.getCause(), e);
errCount.incrementAndGet();
break;
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index e048a78..a09cd1b 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -20,8 +20,6 @@ package org.apache.lens.driver.jdbc;
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.*;
-import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_TMP_FILE;
-
import java.io.IOException;
import java.io.ObjectInput;
import java.io.ObjectOutput;
@@ -33,7 +31,8 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lens.api.LensConf;
-import org.apache.lens.api.query.*;
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
@@ -41,11 +40,12 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
-import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.PreparedQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.rewrite.QueryRewriter;
-import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
@@ -92,7 +92,7 @@ public class JDBCDriver implements LensDriver {
private ConnectionProvider estimateConnectionProvider;
private LogSegregationContext logSegregationContext;
- private UserConfigLoader userConfigLoader;
+ private DriverQueryHook queryHook;
/**
* Data related to a query submitted to JDBCDriver.
@@ -419,6 +419,13 @@ public class JDBCDriver implements LensDriver {
this.conf.addResource("jdbcdriver-default.xml");
this.conf.addResource("jdbcdriver-site.xml");
init(conf);
+ try {
+ queryHook = this.conf.getClass(
+ JDBC_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
+ ).newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
+ }
configured = true;
LOG.info("JDBC Driver configured");
}
@@ -532,7 +539,7 @@ public class JDBCDriver implements LensDriver {
// check for insert clause
ASTNode dest = HQLParser.findNodeByPath(ast, HiveParser.TOK_INSERT);
if (dest != null
- && ((ASTNode) (dest.getChild(0).getChild(0).getChild(0))).getToken().getType() != TOK_TMP_FILE) {
+ && ((ASTNode) (dest.getChild(0).getChild(0).getChild(0))).getToken().getType() != HiveParser.TOK_TMP_FILE) {
throw new LensException("Not allowed statement:" + query);
}
}
@@ -889,9 +896,7 @@ public class JDBCDriver implements LensDriver {
String rewrittenQuery = rewriteQuery(context);
JdbcQueryContext jdbcCtx = new JdbcQueryContext(context, logSegregationContext);
jdbcCtx.setRewrittenQuery(rewrittenQuery);
- if (userConfigLoader != null) {
- userConfigLoader.preSubmit(context);
- }
+ queryHook.preLaunch(context);
try {
Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext));
jdbcCtx.setResultFuture(future);
@@ -1079,13 +1084,6 @@ public class JDBCDriver implements LensDriver {
}
-
- @Override
- public void registerUserConfigLoader(UserConfigLoader userConfigLoader) {
- this.userConfigLoader = userConfigLoader;
- }
-
-
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
index 5abd040..ebcdaca 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
@@ -33,6 +33,7 @@ public final class JDBCDriverConfConstants {
/** The Constant JDBC_QUERY_REWRITER_CLASS. */
public static final String JDBC_QUERY_REWRITER_CLASS = JDBC_DRIVER_PFX + "query.rewriter";
+ public static final String JDBC_QUERY_HOOK_CLASS = JDBC_DRIVER_PFX + "query.hook.class";
/** The Constant JDBC_DRIVER_CLASS. */
public static final String JDBC_DRIVER_CLASS = JDBC_DRIVER_PFX + "driver.class";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index a6c97ae..f534b69 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -18,9 +18,6 @@
*/
package org.apache.lens.driver.jdbc;
-import static org.apache.lens.server.api.user.MockUserConfigLoader.KEY;
-import static org.apache.lens.server.api.user.MockUserConfigLoader.VALUE;
-
import static org.testng.Assert.*;
import java.sql.*;
@@ -40,7 +37,7 @@ import org.apache.lens.server.api.query.ExplainQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
-import org.apache.lens.server.api.user.MockUserConfigLoader;
+import org.apache.lens.server.api.user.MockDriverQueryHook;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -52,6 +49,7 @@ import org.testng.Assert;
import org.testng.annotations.*;
import com.codahale.metrics.MetricRegistry;
+import com.google.common.collect.Lists;
import com.mchange.v2.c3p0.ComboPooledDataSource;
import lombok.extern.slf4j.Slf4j;
@@ -83,19 +81,16 @@ public class TestJdbcDriver {
baseConf.set(JDBCDriverConfConstants.JDBC_USER, "SA");
baseConf.set(JDBCDriverConfConstants.JDBC_PASSWORD, "");
baseConf.set(JDBCDriverConfConstants.JDBC_EXPLAIN_KEYWORD_PARAM, "explain plan for ");
+ baseConf.setClass(JDBCDriverConfConstants.JDBC_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
hConf = new HiveConf(baseConf, this.getClass());
driver = new JDBCDriver();
driver.configure(baseConf);
- driver.registerUserConfigLoader(new MockUserConfigLoader(hConf));
+
assertNotNull(driver);
assertTrue(driver.configured);
- drivers = new ArrayList<LensDriver>() {
- {
- add(driver);
- }
- };
+ drivers = Lists.<LensDriver>newArrayList(driver);
}
/**
@@ -650,7 +645,7 @@ public class TestJdbcDriver {
private void executeAsync(QueryContext ctx) throws LensException {
driver.executeAsync(ctx);
- assertEquals(ctx.getSelectedDriverConf().get(KEY), VALUE);
+ assertEquals(ctx.getSelectedDriverConf().get(MockDriverQueryHook.KEY), MockDriverQueryHook.VALUE);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index eab42d0..2b3a480 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -716,7 +716,7 @@ public final class LensConfConstants {
* Interval at which lens session expiry service runs
*/
public static final String SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS = SERVER_PFX
- + "session.expiry.serivce.interval.secs";
+ + "session.expiry.serivce.interval.secs";
public static final int DEFAULT_SESSION_EXPIRY_SERVICE_INTERVAL_IN_SECS = 3600;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
new file mode 100644
index 0000000..70d999a
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver;
+
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+/**
+ * Drivers can choose to initialize a DriverQueryHook object in their
+ * initialization and use that wherever they want.
+ *
+ * The only use case I see right now is to provide a hook just before query is
+ * launched on the driver.
+ *
+ * This interface is meant to unify drivers' needs of having hooks. Each driver
+ * can use the methods in their own way. Each driver can pose its own restrictions
+ * or guidelines on methods for its hooks.
+ * e.g. some driver may choose to not allow any hooks
+ * another driver may allow hooks but for restricted usage
+ * Some drivers may want their hooks to be initialized with some constructor params
+ * Currently, Hivedriver and Jdbcdriver only require their hook implementations to have a default constructor
+ *
+ * This interface is expected to evolve for some time as more needs of hooks are discovered
+ *
+ */
+public interface DriverQueryHook {
+ /**
+ * Should be Called before launch on the driver
+ * @param ctx
+ */
+ void preLaunch(AbstractQueryContext ctx);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index b56c67c..52207d3 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -24,9 +24,10 @@ import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
-import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.PreparedQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
-import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.conf.Configuration;
@@ -181,10 +182,4 @@ public interface LensDriver extends Externalizable {
* @param driverEventListener the driver event listener
*/
void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener);
-
- /**
- * Add the user config loader to driver for use
- * @param userConfigLoader
- */
- void registerUserConfigLoader(UserConfigLoader userConfigLoader);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
new file mode 100644
index 0000000..a5475e4
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver;
+
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class NoOpDriverQueryHook implements DriverQueryHook {
+ @Override
+ public void preLaunch(AbstractQueryContext ctx) {
+ log.info("Pre launch for {}, user query: {}, driver query: {}", ctx.getSubmittedUser(), ctx.getUserQuery(),
+ ctx.getSelectedDriverQuery());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java
index 93e11a2..61f3a5a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/user/UserConfigLoader.java
@@ -20,34 +20,13 @@ package org.apache.lens.server.api.user;
import java.util.Map;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.QueryContext;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import lombok.extern.slf4j.Slf4j;
-
/**
- * The Class UserConfigLoader. It's initialized once in the server lifetime. After that, t:
- * 1. Gets session configs for the user on each session open. This config applies to the particular session
- * and is forwarded for all actions. One Use case is to decide driver specific details e.g. priority/queue of
- * all queries of the user.
- * 2. Provides a pre-submit hook. Just before submission
+ * The Class UserConfigLoader. It's initialized once in the server lifetime. After that, it's job is to
+ * Get session configs for the user on each session open. This config applies to the particular session
+ * and is forwarded for all actions. One Use case is to decide driver specific details e.g. priority/queue of
+ * all queries of the user.
*/
-@Slf4j
-public abstract class UserConfigLoader {
-
- /** The hive conf. */
- protected final HiveConf hiveConf;
-
- /**
- * Instantiates a new user config loader.
- *
- * @param conf the conf
- */
- protected UserConfigLoader(HiveConf conf) {
- this.hiveConf = conf;
- }
+public interface UserConfigLoader {
/**
* Gets the user config.
@@ -56,9 +35,5 @@ public abstract class UserConfigLoader {
* @return the user config
* @throws UserConfigLoaderException the user config loader exception
*/
- public abstract Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException;
-
- public void preSubmit(QueryContext ctx) throws LensException {
- log.debug("Pre submit " + ctx + " on " + ctx.getSelectedDriver());
- }
+ Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index b52b08a..dffcfd3 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -36,7 +36,6 @@ import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
-import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hive.service.cli.ColumnDescriptor;
@@ -44,7 +43,7 @@ import org.apache.hive.service.cli.ColumnDescriptor;
/**
* The Class MockDriver.
*/
-public class MockDriver implements LensDriver {
+public class MockDriver implements LensDriver {
private static AtomicInteger mockDriverId = new AtomicInteger();
/**
@@ -186,9 +185,6 @@ public class MockDriver implements LensDriver {
public void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener) {
}
- @Override
- public void registerUserConfigLoader(UserConfigLoader userConfigLoader) {
- }
/*
* (non-Javadoc)
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
new file mode 100644
index 0000000..2505ef4
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.user;
+
+import org.apache.lens.server.api.driver.DriverQueryHook;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+public class MockDriverQueryHook implements DriverQueryHook {
+ public static final String KEY = "TEST_KEY";
+ public static final String VALUE = "TEST_VALUE";
+
+ @Override
+ public void preLaunch(AbstractQueryContext ctx) {
+ ctx.getSelectedDriverConf().set(KEY, VALUE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java
deleted file mode 100644
index f6ef0d4..0000000
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockUserConfigLoader.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.user;
-
-import java.util.Map;
-
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.QueryContext;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-
-import com.beust.jcommander.internal.Maps;
-
-public class MockUserConfigLoader extends UserConfigLoader {
- public static final String KEY = "TEST_KEY";
- public static final String VALUE = "TEST_VALUE";
-
- public MockUserConfigLoader(HiveConf conf) {
- super(conf);
- }
-
- @Override
- public Map<String, String> getUserConfig(String loggedInUser) throws UserConfigLoaderException {
- return Maps.newHashMap();
- }
-
- @Override
- public void preSubmit(QueryContext ctx) throws LensException {
- ctx.getSelectedDriverConf().set(KEY, VALUE);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 3511236..2d7a92a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -19,6 +19,7 @@
package org.apache.lens.server.query;
import static org.apache.lens.api.query.QueryStatus.Status.*;
+import static org.apache.lens.server.api.LensConfConstants.*;
import static org.apache.lens.server.session.LensSessionImpl.ResourceEntry;
import java.io.*;
@@ -44,7 +45,6 @@ import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.LensService;
import org.apache.lens.server.LensServices;
-import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.error.LensMultiCauseException;
@@ -60,7 +60,6 @@ import org.apache.lens.server.rewrite.RewriteUtil;
import org.apache.lens.server.rewrite.UserQueryToCubeQueryRewriter;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
-import org.apache.lens.server.user.UserConfigLoaderFactory;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.collections.CollectionUtils;
@@ -273,7 +272,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* Initialize query acceptors and listeners.
*/
private void initializeQueryAcceptors() throws LensException {
- String[] acceptorClasses = conf.getStrings(LensConfConstants.ACCEPTOR_CLASSES);
+ String[] acceptorClasses = conf.getStrings(ACCEPTOR_CLASSES);
if (acceptorClasses != null) {
for (String acceptorClass : acceptorClasses) {
try {
@@ -290,7 +289,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
private void initializeListeners() {
- if (conf.getBoolean(LensConfConstants.QUERY_STATE_LOGGER_ENABLED, true)) {
+ if (conf.getBoolean(QUERY_STATE_LOGGER_ENABLED, true)) {
getEventService().addListenerForType(new QueryStatusLogger(), StatusChange.class);
log.info("Registered query state logger");
}
@@ -309,17 +308,16 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
* @throws LensException the lens exception
*/
private void loadDriversAndSelector() throws LensException {
- String[] driverClasses = conf.getStrings(LensConfConstants.DRIVER_CLASSES);
+ Class<?>[] driverClasses = conf.getClasses(DRIVER_CLASSES);
if (driverClasses != null) {
- for (String driverClass : driverClasses) {
+ for (Class<?> driverClass : driverClasses) {
try {
- Class<?> clazz = Class.forName(driverClass);
- LensDriver driver = (LensDriver) clazz.newInstance();
+ LensDriver driver = (LensDriver) driverClass.newInstance();
driver.configure(LensServerConf.getConf());
- driver.registerUserConfigLoader(UserConfigLoaderFactory.getUserConfigLoader());
- driver.registerDriverEventListener(driverEventListener);
-
- drivers.put(driverClass, driver);
+ if (driver instanceof HiveDriver) {
+ driver.registerDriverEventListener(driverEventListener);
+ }
+ drivers.put(driverClass.getName(), driver);
log.info("Driver for " + driverClass + " is loaded");
} catch (Exception e) {
log.warn("Could not load the driver:" + driverClass, e);
@@ -330,15 +328,15 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
throw new LensException("No drivers specified");
}
try {
- Class<? extends DriverSelector> driverSelectorClass = conf.getClass(LensConfConstants.DRIVER_SELECTOR_CLASS,
+ Class<? extends DriverSelector> driverSelectorClass = conf.getClass(DRIVER_SELECTOR_CLASS,
MinQueryCostSelector.class,
DriverSelector.class);
log.info("Using driver selector class: " + driverSelectorClass.getCanonicalName());
driverSelector = driverSelectorClass.newInstance();
} catch (Exception e) {
throw new LensException("Couldn't instantiate driver selector class. Class name: "
- + conf.get(LensConfConstants.DRIVER_SELECTOR_CLASS) + ". Please supply a valid value for "
- + LensConfConstants.DRIVER_SELECTOR_CLASS);
+ + conf.get(DRIVER_SELECTOR_CLASS) + ". Please supply a valid value for "
+ + DRIVER_SELECTOR_CLASS);
}
}
@@ -928,8 +926,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LOG.error(e);
throw new IllegalStateException("Could not load drivers", e);
}
- maxFinishedQueries = conf.getInt(LensConfConstants.MAX_NUMBER_OF_FINISHED_QUERY,
- LensConfConstants.DEFAULT_FINISHED_QUERIES);
+ maxFinishedQueries = conf.getInt(MAX_NUMBER_OF_FINISHED_QUERY,
+ DEFAULT_FINISHED_QUERIES);
initalizeFinishedQueryStore(conf);
log.info("Query execution service initialized");
}
@@ -1043,12 +1041,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
private void startEstimatePool() {
- int minPoolSize = conf.getInt(LensConfConstants.ESTIMATE_POOL_MIN_THREADS,
- LensConfConstants.DEFAULT_ESTIMATE_POOL_MIN_THREADS);
- int maxPoolSize = conf.getInt(LensConfConstants.ESTIMATE_POOL_MAX_THREADS,
- LensConfConstants.DEFAULT_ESTIMATE_POOL_MAX_THREADS);
- int keepAlive = conf.getInt(LensConfConstants.ESTIMATE_POOL_KEEP_ALIVE_MILLIS,
- LensConfConstants.DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS);
+ int minPoolSize = conf.getInt(ESTIMATE_POOL_MIN_THREADS,
+ DEFAULT_ESTIMATE_POOL_MIN_THREADS);
+ int maxPoolSize = conf.getInt(ESTIMATE_POOL_MAX_THREADS,
+ DEFAULT_ESTIMATE_POOL_MAX_THREADS);
+ int keepAlive = conf.getInt(ESTIMATE_POOL_KEEP_ALIVE_MILLIS,
+ DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS);
final ThreadFactory defaultFactory = Executors.defaultThreadFactory();
final AtomicInteger thId = new AtomicInteger();
@@ -1111,8 +1109,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
// Wait for all rewrite and estimates to finish
try {
- long estimateLatchTimeout = ctx.getConf().getLong(LensConfConstants.ESTIMATE_TIMEOUT_MILLIS,
- LensConfConstants.DEFAULT_ESTIMATE_TIMEOUT_MILLIS);
+ long estimateLatchTimeout = ctx.getConf().getLong(ESTIMATE_TIMEOUT_MILLIS,
+ DEFAULT_ESTIMATE_TIMEOUT_MILLIS);
boolean completed = estimateCompletionLatch.await(estimateLatchTimeout, TimeUnit.MILLISECONDS);
// log operations yet to complete and check if we can proceed with at least one driver
@@ -2157,7 +2155,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private QueryContext createResourceQuery(String command, LensSessionHandle sessionHandle, LensDriver driver)
throws LensException {
LensConf qconf = new LensConf();
- qconf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false");
+ qconf.addProperty(QUERY_PERSISTENT_RESULT_INDRIVER, "false");
QueryContext addQuery = QueryContext.createContextWithSingleDriver(command,
getSession(sessionHandle).getLoggedInUser(), qconf, getLensConf(
sessionHandle, qconf), driver, sessionHandle.getPublicId().toString(), true);
@@ -2330,7 +2328,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.warn("Unable to get status for Result Directory", e);
throw new NotFoundException("Http result not available for query:" + queryHandle.toString());
}
- String resultFSReadUrl = ctx.getConf().get(LensConfConstants.RESULT_FS_READ_URL);
+ String resultFSReadUrl = ctx.getConf().get(RESULT_FS_READ_URL);
if (resultFSReadUrl != null) {
try {
URI resultReadPath = new URI(resultFSReadUrl + resultPath.toUri().getPath() + "?op=OPEN&user.name="
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
index ce0d112..e5e91b8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/DatabaseUserConfigLoader.java
@@ -39,7 +39,7 @@ import com.google.common.cache.CacheBuilder;
/**
* The Class DatabaseUserConfigLoader.
*/
-public class DatabaseUserConfigLoader extends UserConfigLoader {
+public class DatabaseUserConfigLoader implements UserConfigLoader {
/** The query sql. */
protected final String querySql;
@@ -49,6 +49,7 @@ public class DatabaseUserConfigLoader extends UserConfigLoader {
/** The cache. */
protected final Cache<String, Map<String, String>> cache;
+ private final HiveConf hiveConf;
/** The ds. */
protected BasicDataSource ds;
@@ -60,7 +61,7 @@ public class DatabaseUserConfigLoader extends UserConfigLoader {
* @throws UserConfigLoaderException the user config loader exception
*/
public DatabaseUserConfigLoader(HiveConf conf) throws UserConfigLoaderException {
- super(conf);
+ this.hiveConf = conf;
querySql = conf.get(LensConfConstants.USER_RESOLVER_DB_QUERY);
keys = conf.get(LensConfConstants.USER_RESOLVER_DB_KEYS).split("\\s*,\\s*", -1);
cache = CacheBuilder.newBuilder()
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
index 5813ac0..0c53b28 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/FixedUserConfigLoader.java
@@ -26,29 +26,28 @@ import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.hive.conf.HiveConf;
+import com.google.common.collect.Maps;
+
/**
* The Class FixedUserConfigLoader.
*/
-public class FixedUserConfigLoader extends UserConfigLoader {
+public class FixedUserConfigLoader implements UserConfigLoader {
+
+ private final String fixedValue;
- /**
- * Instantiates a new fixed user config loader.
- *
- * @param conf the conf
- */
public FixedUserConfigLoader(HiveConf conf) {
- super(conf);
+ fixedValue = conf.get(LensConfConstants.USER_RESOLVER_FIXED_VALUE);
}
/*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.user.UserConfigLoader#getUserConfig(java.lang.String)
- */
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.user.UserConfigLoader#getUserConfig(java.lang.String)
+ */
@Override
public Map<String, String> getUserConfig(String loggedInUser) {
- HashMap<String, String> userConfig = new HashMap<String, String>();
- userConfig.put(LensConfConstants.SESSION_CLUSTER_USER, hiveConf.get(LensConfConstants.USER_RESOLVER_FIXED_VALUE));
+ HashMap<String, String> userConfig = Maps.newHashMap();
+ userConfig.put(LensConfConstants.SESSION_CLUSTER_USER, fixedValue);
return userConfig;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
index 009e34a..1c025dd 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/PropertyBasedUserConfigLoader.java
@@ -18,10 +18,7 @@
*/
package org.apache.lens.server.user;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
+import java.io.*;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
@@ -32,13 +29,15 @@ import org.apache.lens.server.api.user.UserConfigLoaderException;
import org.apache.hadoop.hive.conf.HiveConf;
+import com.google.common.collect.Maps;
+
/**
* The Class PropertyBasedUserConfigLoader.
*/
-public class PropertyBasedUserConfigLoader extends UserConfigLoader {
+public class PropertyBasedUserConfigLoader implements UserConfigLoader {
/** The user map. */
- private HashMap<String, Map<String, String>> userMap;
+ private HashMap<String, Map<String, String>> userMap = Maps.newHashMap();
/**
* Instantiates a new property based user config loader.
@@ -47,10 +46,8 @@ public class PropertyBasedUserConfigLoader extends UserConfigLoader {
* @throws UserConfigLoaderException the user config loader exception
*/
public PropertyBasedUserConfigLoader(HiveConf conf) throws UserConfigLoaderException {
- super(conf);
- userMap = new HashMap<String, Map<String, String>>();
Properties properties = new Properties();
- String filename = hiveConf.get(LensConfConstants.USER_RESOLVER_PROPERTYBASED_FILENAME, null);
+ String filename = conf.get(LensConfConstants.USER_RESOLVER_PROPERTYBASED_FILENAME, null);
if (filename == null) {
throw new UserConfigLoaderException("property file path not provided for property based resolver."
+ "Please set property " + LensConfConstants.USER_RESOLVER_PROPERTYBASED_FILENAME);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c7261d12/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java b/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
index 0a319ef..d0c5816 100644
--- a/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
+++ b/lens-server/src/test/java/org/apache/lens/server/user/FooBarConfigLoader.java
@@ -21,8 +21,6 @@ package org.apache.lens.server.user;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -30,7 +28,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
/**
* The Class FooBarConfigLoader.
*/
-public class FooBarConfigLoader extends UserConfigLoader {
+public class FooBarConfigLoader implements UserConfigLoader {
/** The const hash map. */
public static final HashMap<String, String> CONST_HASH_MAP = new HashMap<String, String>() {
@@ -45,7 +43,6 @@ public class FooBarConfigLoader extends UserConfigLoader {
* @param conf the conf
*/
public FooBarConfigLoader(HiveConf conf) {
- super(conf);
}
/*
@@ -57,10 +54,4 @@ public class FooBarConfigLoader extends UserConfigLoader {
public Map<String, String> getUserConfig(String loggedInUser) {
return CONST_HASH_MAP;
}
-
- @Override
- public void preSubmit(QueryContext ctx) throws LensException {
- super.preSubmit(ctx);
- ctx.getSelectedDriverConf().set("random.conf.key", "random.conf.value");
- }
}
[05/50] [abbrv] incubator-lens git commit: LENS-630: Using Duration
and Fact Weight Based Query Cost Calculator for Hive Driver
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
deleted file mode 100644
index 09a8ace..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/QueryPriorityDecider.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.priority;
-
-import org.apache.lens.api.Priority;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-
-import org.apache.log4j.Logger;
-
-public interface QueryPriorityDecider {
- /**
- * The Constant LOG.
- */
- Logger LOG = Logger.getLogger(QueryPriorityDecider.class);
-
- /**
- * @param queryContext
- * @return calculated Priority based on the explained plans for each driver
- * @throws LensException when can't decide priority.
- */
- Priority decidePriority(AbstractQueryContext queryContext) throws LensException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
deleted file mode 100644
index 75295ab..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/priority/RangeConf.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.priority;
-
-import java.util.TreeMap;
-
-/**
- * Class for storing range configurations. An Example would be grading system.
- * The value F,30,D,40,C,60,B,80,A corresponds to a system where
- * - inf < marks <= 30 : F
- * 30 < marks <= 40 : D
- * 40 < marks <= 60 : C
- * 60 < marks <= 80 : B
- * 80 < marks <= + Inf : A
- * <p></p>
- * rangeConfInstance.get(marks) would give you the grade depending on the range.
- * <p></p>
- * The utility is for easily storing range configs in config xml files.
- * <p></p>
- * Implementation is done by storing the least value(floor) and keeping a treemap on rest of values
- *
- * @param <K> Key type. Integer in the grade range example
- * @param <V> Value Type. String(or a Grade class) in the grade range example
- */
-
-public abstract class RangeConf<K extends Comparable<K>, V> {
-
- /**
- * The data structure is one map and one floor value
- *
- */
-
- /**
- * Treemap of lower boundary value and next grade
- */
- TreeMap<K, V> map = new TreeMap<K, V>();
-
- /**
- * Lowest boundary value.
- */
- V floor;
-
- /**
- * Constructor. parses the string to form the data structure.
- *
- * @param confValue
- */
- RangeConf(String confValue) {
- if (confValue == null || confValue.isEmpty()) {
- confValue = getDefaultConf();
- }
- String[] split = confValue.split("\\s*,\\s*");
- assert split.length % 2 == 1;
- floor = parseValue(split[0]);
- for (int i = 1; i < split.length; i += 2) {
- map.put(parseKey(split[i]), parseValue(split[i + 1]));
- }
- }
-
- /**
- * parse key type from its string representation
- */
- protected abstract K parseKey(String s);
-
- /**
- * parse value type from its string representation
- */
- protected abstract V parseValue(String s);
-
- /**
- * When null/blank conf string passed, this would be the value from which data structure will be formed
- */
- protected abstract String getDefaultConf();
-
- /**
- * Get method.
- *
- * @param key
- * @return the value depending on which range the key lies in
- */
- public V get(K key) {
- return map.floorEntry(key) == null ? floor : map.floorEntry(key).getValue();
- }
-
- /**
- * toString representation
- *
- * @return string representation
- */
- @Override
- public String toString() {
- return "lower value: " + floor + ", map: " + map;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index c398215..0cccaef 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -19,15 +19,11 @@
package org.apache.lens.server.api.query;
import java.io.Serializable;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.lens.api.LensConf;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.DriverQueryPlan;
import org.apache.lens.server.api.driver.LensDriver;
@@ -35,6 +31,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.query.DriverSelectorQueryContext.DriverQueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.commons.logging.Log;
@@ -192,6 +189,14 @@ public abstract class AbstractQueryContext implements Serializable {
return estimateRunnables;
}
+ public Map<String, Double> getTableWeights(LensDriver driver) {
+ return getDriverContext().getDriverRewriterPlan(driver).getTableWeights();
+ }
+
+ public DriverQueryPlan getDriverRewriterPlan(LensDriver driver) {
+ return getDriverContext().getDriverRewriterPlan(driver);
+ }
+
/**
* Runnable to wrap estimate computation for a driver. Failure cause and success status
* are stored as field members
@@ -210,7 +215,7 @@ public abstract class AbstractQueryContext implements Serializable {
private LensException cause;
public DriverEstimateRunnable(AbstractQueryContext queryContext,
- LensDriver driver) {
+ LensDriver driver) {
this.queryContext = queryContext;
this.driver = driver;
}
@@ -242,10 +247,10 @@ public abstract class AbstractQueryContext implements Serializable {
String expMsg = LensUtil.getCauseMessage(e);
driverQueryContext.setDriverQueryCostEstimateError(e);
failureCause = new StringBuilder("Driver :")
- .append(driver.getClass().getName())
- .append(" Cause :")
- .append(expMsg)
- .toString();
+ .append(driver.getClass().getName())
+ .append(" Cause :")
+ .append(expMsg)
+ .toString();
LOG.error("Setting driver cost failed for driver " + driver + " Cause: " + failureCause, e);
}
}
@@ -278,6 +283,7 @@ public abstract class AbstractQueryContext implements Serializable {
public String getFinalDriverQuery(LensDriver driver) {
return driverContext.getFinalDriverQuery(driver);
}
+
/**
* Get driver conf
*
@@ -293,11 +299,12 @@ public abstract class AbstractQueryContext implements Serializable {
* Get query cost for the driver
*
* @param driver
- * @return QueryCost
+ * @return QueryCostTO
*/
public QueryCost getDriverQueryCost(LensDriver driver) {
return driverContext.getDriverQueryCost(driver);
}
+
/**
* Wrapper method for convenience on driver context
*
@@ -363,7 +370,7 @@ public abstract class AbstractQueryContext implements Serializable {
/**
* Get selected driver's cost
*
- * @return QueryCost
+ * @return QueryCostTO
* @throws LensException
*/
public QueryCost getSelectedDriverQueryCost() throws LensException {
@@ -433,6 +440,10 @@ public abstract class AbstractQueryContext implements Serializable {
isDriverQueryExplicitlySet = true;
}
+ public void setDriverCost(LensDriver driver, QueryCost cost) {
+ driverContext.setDriverCost(driver, cost);
+ }
+
/**
* Get handle of the query for logging purposes
* @return
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 0945222..3e7de17 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -20,11 +20,11 @@ package org.apache.lens.server.api.query;
import java.util.*;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.DriverQueryPlan;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.commons.lang3.StringUtils;
@@ -56,7 +56,7 @@ public class DriverSelectorQueryContext {
@Getter
@Setter
protected Map<LensDriver, DriverQueryContext> driverQueryContextMap = new HashMap<LensDriver,
- DriverQueryContext>();
+ DriverQueryContext>();
public DriverSelectorQueryContext(final String userQuery, final Configuration queryConf,
final Collection<LensDriver> drivers) {
@@ -282,6 +282,10 @@ public class DriverSelectorQueryContext {
driverQueryContextMap.get(driver).setDriverSpecificConf(conf);
}
+ public void setDriverCost(LensDriver driver, QueryCost cost) {
+ driverQueryContextMap.get(driver).setDriverCost(cost);
+ }
+
public void setSelectedDriverQuery(String driverQuery) {
if (driverQueryContextMap != null && driverQueryContextMap.get(getSelectedDriver()) != null) {
driverQueryContextMap.get(getSelectedDriver()).setQuery(driverQuery);
@@ -331,7 +335,7 @@ public class DriverSelectorQueryContext {
public String getFinalDriverQuery(LensDriver driver) {
return driverQueryContextMap.get(driver) != null
- ? driverQueryContextMap.get(driver).getFinalDriverQuery() : null;
+ ? driverQueryContextMap.get(driver).getFinalDriverQuery() : null;
}
public QueryCost getDriverQueryCost(LensDriver driver) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
index 3c0f787..5d7163f 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
@@ -26,6 +26,7 @@ import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.query.*;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.cost.QueryCost;
/**
* The Interface QueryExecutionService.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
new file mode 100644
index 0000000..7fa5cdb
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.query.cost;
+
+import org.apache.lens.api.query.QueryCostType;
+
+import com.google.common.base.Preconditions;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+
+@ToString
+@EqualsAndHashCode
+public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBasedQueryCost> {
+ private final double partitionCost;
+
+ public FactPartitionBasedQueryCost(final double partitionCost) {
+ Preconditions.checkArgument(partitionCost >= 0, "Cost can't be negative");
+ this.partitionCost = partitionCost;
+ }
+
+ @Override
+ public FactPartitionBasedQueryCost add(final FactPartitionBasedQueryCost other) {
+ return new FactPartitionBasedQueryCost(partitionCost + other.partitionCost);
+ }
+
+ @Override
+ public QueryCostType getQueryCostType() {
+ return partitionCost == 0 ? QueryCostType.LOW : QueryCostType.HIGH;
+ }
+
+ @Override
+ public long getEstimatedExecTimeMillis() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("Estimated time is not implemented");
+ }
+
+ @Override
+ public double getEstimatedResourceUsage() {
+ return partitionCost;
+ }
+
+ @Override
+ public int compareTo(final FactPartitionBasedQueryCost o) {
+ return new Double(partitionCost).compareTo(o.partitionCost);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
new file mode 100644
index 0000000..be72321
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.query.cost;
+
+
+import org.apache.lens.api.Addable;
+import org.apache.lens.api.query.QueryCostType;
+
+/**
+ * QueryCost Interface.
+ * @param <T>
+ */
+public interface QueryCost<T extends QueryCost<T>> extends Comparable<T>, Addable<T> {
+ QueryCostType getQueryCostType();
+
+ long getEstimatedExecTimeMillis() throws UnsupportedOperationException;
+
+ double getEstimatedResourceUsage() throws UnsupportedOperationException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java
new file mode 100644
index 0000000..8c71c00
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostCalculator.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.cost;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+public interface QueryCostCalculator {
+ /**
+ * @param queryContext
+ * @return calculated cost based on anything available in abstract query context
+ * @throws LensException when can't calculate cost
+ */
+ QueryCost calculateCost(AbstractQueryContext queryContext, LensDriver driver) throws LensException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTOBuilder.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTOBuilder.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTOBuilder.java
new file mode 100644
index 0000000..81f1fd1
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCostTOBuilder.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.query.cost;
+
+import org.apache.lens.api.result.QueryCostTO;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class QueryCostTOBuilder {
+ @NonNull private final QueryCost cost;
+
+ public QueryCostTO build() {
+ Long estimatedExecTimeMillis = null;
+ Double estimatedResourceUsage = null;
+ try {
+ estimatedExecTimeMillis = cost.getEstimatedExecTimeMillis();
+ } catch (UnsupportedOperationException e) {
+ log.debug("estimated exec time not supported for {}", cost.getClass().getCanonicalName());
+ }
+ try {
+ estimatedResourceUsage = cost.getEstimatedResourceUsage();
+ } catch (UnsupportedOperationException e) {
+ log.debug("estimated resource usage not supported for {}", cost.getClass().getCanonicalName());
+ }
+ return new QueryCostTO(estimatedExecTimeMillis, estimatedResourceUsage, cost.getQueryCostType());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostRangePriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostRangePriorityDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostRangePriorityDecider.java
new file mode 100644
index 0000000..eef09cf
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostRangePriorityDecider.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.priority;
+
+import org.apache.lens.api.Priority;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import lombok.NonNull;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public class CostRangePriorityDecider implements QueryPriorityDecider {
+
+ @NonNull
+ private final CostToPriorityRangeConf costToPriorityRangeMap;
+
+ @Override
+ public Priority decidePriority(@NonNull final QueryCost cost) throws LensException {
+ Priority p = costToPriorityRangeMap.get(cost.getEstimatedResourceUsage());
+ log.info("cost was: {}, decided priority: {}", cost, p);
+ return p;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostToPriorityRangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostToPriorityRangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostToPriorityRangeConf.java
new file mode 100644
index 0000000..72228e7
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/CostToPriorityRangeConf.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.priority;
+
+
+import org.apache.lens.api.Priority;
+
+/**
+ * Cost to priority range conf class.
+ *
+ * @see RangeConf
+ */
+
+public class CostToPriorityRangeConf extends RangeConf<Double, Priority> {
+ /**
+ * Super constructor
+ *
+ * @param confValue
+ * @see RangeConf#RangeConf(String)
+ */
+ public CostToPriorityRangeConf(String confValue) {
+ super(confValue);
+ }
+
+ /**
+ * Parse key method
+ *
+ * @param s
+ * @return parsed float from string s
+ * @see RangeConf#parseKey(String)
+ */
+ @Override
+ protected Double parseKey(String s) {
+ return Double.parseDouble(s);
+ }
+
+ /**
+ * Parse value method
+ *
+ * @param s
+ * @return parsed Priority from String s
+ * @see RangeConf#parseValue(String)
+ */
+ @Override
+ protected Priority parseValue(String s) {
+ return Priority.valueOf(s);
+ }
+
+ /**
+ * Default value is "NORMAL". Which implies {@link #get(Comparable)} will always return Priotity.NORMAL
+ *
+ * @return "NORMAL"
+ */
+ @Override
+ protected String getDefaultConf() {
+ return Priority.NORMAL.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/QueryPriorityDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/QueryPriorityDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/QueryPriorityDecider.java
new file mode 100644
index 0000000..872006c
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/QueryPriorityDecider.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.priority;
+
+import org.apache.lens.api.Priority;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+public interface QueryPriorityDecider {
+
+ /**
+ * @param cost
+ * @return calculated Priority based on the pre calculated query cost
+ * @throws LensException when can't decide priority.
+ */
+ Priority decidePriority(QueryCost cost) throws LensException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java
new file mode 100644
index 0000000..43cc53c
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/priority/RangeConf.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query.priority;
+
+import java.util.TreeMap;
+
+/**
+ * Class for storing range configurations. An Example would be grading system.
+ * The value F,30,D,40,C,60,B,80,A corresponds to a system where
+ * - inf < marks <= 30 : F
+ * 30 < marks <= 40 : D
+ * 40 < marks <= 60 : C
+ * 60 < marks <= 80 : B
+ * 80 < marks <= + Inf : A
+ * <p></p>
+ * rangeConfInstance.get(marks) would give you the grade depending on the range.
+ * <p></p>
+ * The utility is for easily storing range configs in config xml files.
+ * <p></p>
+ * Implementation is done by storing the least value(floor) and keeping a treemap on rest of values
+ *
+ * @param <K> Key type. Integer in the grade range example
+ * @param <V> Value Type. String(or a Grade class) in the grade range example
+ */
+
+public abstract class RangeConf<K extends Comparable<K>, V> {
+
+ /**
+ * The data structure is one map and one floor value
+ *
+ */
+
+ /**
+ * Treemap of lower boundary value and next grade
+ */
+ TreeMap<K, V> map = new TreeMap<K, V>();
+
+ /**
+ * Lowest boundary value.
+ */
+ V floor;
+
+ /**
+ * Constructor. parses the string to form the data structure.
+ *
+ * @param confValue
+ */
+ RangeConf(String confValue) {
+ if (confValue == null || confValue.isEmpty()) {
+ confValue = getDefaultConf();
+ }
+ String[] split = confValue.split("\\s*,\\s*");
+ assert split.length % 2 == 1;
+ floor = parseValue(split[0]);
+ for (int i = 1; i < split.length; i += 2) {
+ map.put(parseKey(split[i]), parseValue(split[i + 1]));
+ }
+ }
+
+ /**
+ * parse key type from its string representation
+ */
+ protected abstract K parseKey(String s);
+
+ /**
+ * parse value type from its string representation
+ */
+ protected abstract V parseValue(String s);
+
+ /**
+ * When null/blank conf string passed, this would be the value from which data structure will be formed
+ */
+ protected abstract String getDefaultConf();
+
+ /**
+ * Get method.
+ *
+ * @param key
+ * @return the value depending on which range the key lies in
+ */
+ public V get(K key) {
+ return map.floorEntry(key) == null ? floor : map.floorEntry(key).getValue();
+ }
+
+ /**
+ * toString representation
+ *
+ * @return string representation
+ */
+ @Override
+ public String toString() {
+ return "lower value: " + floor + ", map: " + map;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 4b80918..b52b08a 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -25,7 +25,6 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.api.query.ResultRow;
@@ -35,6 +34,8 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.hadoop.conf.Configuration;
@@ -43,7 +44,7 @@ import org.apache.hive.service.cli.ColumnDescriptor;
/**
* The Class MockDriver.
*/
-public class MockDriver implements LensDriver {
+public class MockDriver implements LensDriver {
private static AtomicInteger mockDriverId = new AtomicInteger();
/**
@@ -121,7 +122,7 @@ public class MockDriver implements LensDriver {
@Override
public QueryCost getCost() {
- return new QueryCost(0L, 0.0);
+ return new FactPartitionBasedQueryCost(0);
}
}
@@ -375,7 +376,7 @@ public class MockDriver implements LensDriver {
@Override
public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
- return new QueryCost(0L, 0.0);
+ return new FactPartitionBasedQueryCost(0);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
index 060a867..c981a4f 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockFailDriver.java
@@ -18,9 +18,9 @@
*/
package org.apache.lens.server.api.driver;
-import org.apache.lens.api.query.QueryCost;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
public class MockFailDriver extends MockDriver {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
new file mode 100644
index 0000000..0ab0b0e
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.query.cost;
+
+import static org.testng.Assert.*;
+
+import org.apache.lens.api.query.QueryCostType;
+
+import org.testng.annotations.Test;
+
+
+public class TestFactPartitionBasedQueryCost {
+ FactPartitionBasedQueryCost cost0 = new FactPartitionBasedQueryCost(0.0);
+ FactPartitionBasedQueryCost cost1 = new FactPartitionBasedQueryCost(0.2);
+ FactPartitionBasedQueryCost cost11 = new FactPartitionBasedQueryCost(0.2);
+ FactPartitionBasedQueryCost cost2 = new FactPartitionBasedQueryCost(0.3);
+
+ @Test(expectedExceptions = {IllegalArgumentException.class})
+ public void testInvalid() {
+ new FactPartitionBasedQueryCost(-0.5);
+ }
+
+ @Test
+ public void testAdd() throws Exception {
+ assertEquals(cost1.add(cost2), new FactPartitionBasedQueryCost(0.5));
+ }
+
+ @Test
+ public void testGetQueryCostType() throws Exception {
+ assertEquals(cost1.getQueryCostType(), QueryCostType.HIGH);
+ assertEquals(cost2.getQueryCostType(), QueryCostType.HIGH);
+ assertEquals(cost0.getQueryCostType(), QueryCostType.LOW);
+ }
+
+ @Test(expectedExceptions = {UnsupportedOperationException.class})
+ public void testGetEstimatedExecTimeMillis() throws Exception {
+ cost1.getEstimatedExecTimeMillis();
+ }
+
+ @Test
+ public void testGetEstimatedResourceUsage() throws Exception {
+ assertEquals(cost1.getEstimatedResourceUsage(), 0.2);
+ }
+
+ @Test
+ public void testCompareTo() throws Exception {
+ assertEquals(cost1.compareTo(cost2), -1);
+ assertEquals(cost2.compareTo(cost1), 1);
+ assertEquals(cost1.compareTo(cost11), 0);
+ }
+
+ @Test
+ public void testEquals() throws Exception {
+ assertTrue(cost1.equals(cost11));
+ assertTrue(cost11.equals(cost1));
+ assertFalse(cost1.equals(cost2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 429b262..8a32202 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.server.query;
+import static org.apache.lens.api.query.QueryStatus.Status.*;
import static org.apache.lens.server.session.LensSessionImpl.ResourceEntry;
import java.io.*;
@@ -53,6 +54,7 @@ import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.session.LensSessionImpl;
@@ -78,10 +80,8 @@ import org.codehaus.jackson.map.*;
import org.codehaus.jackson.map.module.SimpleModule;
import com.google.common.collect.ImmutableList;
-
import lombok.Getter;
import lombok.NonNull;
-
import lombok.extern.slf4j.Slf4j;
/**
@@ -315,9 +315,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
LensDriver driver = (LensDriver) clazz.newInstance();
driver.configure(LensServerConf.getConf());
driver.registerUserConfigLoader(UserConfigLoaderFactory.getUserConfigLoader());
- if (driver instanceof HiveDriver) {
- driver.registerDriverEventListener(driverEventListener);
- }
+ driver.registerDriverEventListener(driverEventListener);
drivers.put(driverClass, driver);
log.info("Driver for " + driverClass + " is loaded");
@@ -507,7 +505,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
logSegregationContext.set(ctx.getQueryHandleString());
synchronized (ctx) {
- if (ctx.getStatus().getStatus().equals(Status.QUEUED)) {
+ if (ctx.getStatus().getStatus().equals(QUEUED)) {
log.info("Launching query:" + ctx.getUserQuery());
try {
// acquire session before any query operation.
@@ -623,14 +621,14 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
throws LensException {
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0f, QueryStatus.Status.FAILED, statusMsg, false, null, reason, lensErrorTO));
+ ctx.setStatus(new QueryStatus(0.0f, FAILED, statusMsg, false, null, reason, lensErrorTO));
updateFinishedQuery(ctx, before);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
}
private void setLaunchedStatus(QueryContext ctx) throws LensException {
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(ctx.getStatus().getProgress(), QueryStatus.Status.LAUNCHED, "launched on the driver",
+ ctx.setStatus(new QueryStatus(ctx.getStatus().getProgress(), LAUNCHED, "launched on the driver",
false, null, null, null));
launchedQueries.add(ctx);
ctx.setLaunchTime(System.currentTimeMillis());
@@ -647,7 +645,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
*/
private void setCancelledStatus(QueryContext ctx, String statusMsg) throws LensException {
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0f, QueryStatus.Status.CANCELED, statusMsg, false, null, null, null));
+ ctx.setStatus(new QueryStatus(0.0f, CANCELED, statusMsg, false, null, null, null));
updateFinishedQuery(ctx, before);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
}
@@ -661,7 +659,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private void updateFinishedQuery(QueryContext ctx, QueryStatus before) {
// before would be null in case of server restart
if (before != null) {
- if (before.getStatus().equals(Status.QUEUED)) {
+ if (before.getStatus().equals(QUEUED)) {
queuedQueries.remove(ctx);
} else {
launchedQueries.remove(ctx);
@@ -673,7 +671,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
void setSuccessState(QueryContext ctx) throws LensException {
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(1.0f, QueryStatus.Status.SUCCESSFUL, "Query is successful!", ctx
+ ctx.setStatus(new QueryStatus(1.0f, SUCCESSFUL, "Query is successful!", ctx
.isResultAvailableInDriver(), null, null, null));
updateFinishedQuery(ctx, before);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
@@ -690,7 +688,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (ctx != null) {
synchronized (ctx) {
QueryStatus before = ctx.getStatus();
- if (!ctx.getStatus().getStatus().equals(QueryStatus.Status.QUEUED) && !ctx.getDriverStatus().isFinished()
+ if (!ctx.getStatus().getStatus().equals(QUEUED) && !ctx.getDriverStatus().isFinished()
&& !ctx.getStatus().finished()) {
log.info("Updating status for " + ctx.getQueryHandle());
try {
@@ -706,7 +704,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
// query is successfully executed by driver and
// if query result need not be persisted or there is no result available in driver, move the query to
// succeeded state immediately, otherwise result formatter will format the result and move it to succeeded
- if (ctx.getStatus().getStatus().equals(QueryStatus.Status.EXECUTED) && (!ctx.isPersistent()
+ if (ctx.getStatus().getStatus().equals(EXECUTED) && (!ctx.isPersistent()
|| !ctx.isResultAvailableInDriver())) {
setSuccessState(ctx);
} else {
@@ -816,7 +814,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
try {
FinishedLensQuery finishedQuery = new FinishedLensQuery(finished.getCtx());
- if (finished.ctx.getStatus().getStatus() == Status.SUCCESSFUL) {
+ if (finished.ctx.getStatus().getStatus() == SUCCESSFUL) {
if (finished.ctx.getStatus().isResultSetAvailable()) {
LensResultSet set = getResultset(finished.getCtx().getQueryHandle());
if (set != null && PersistentResultSet.class.isAssignableFrom(set.getClass())) {
@@ -853,7 +851,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
resultSets.remove(finished.getCtx().getQueryHandle());
}
fireStatusChangeEvent(finished.getCtx(),
- new QueryStatus(1f, Status.CLOSED, "Query purged", false, null, null, null), finished.getCtx().getStatus());
+ new QueryStatus(1f, CLOSED, "Query purged", false, null, null, null), finished.getCtx().getStatus());
log.info("Query purged: " + finished.getCtx().getQueryHandle());
} catch (LensException e) {
@@ -1569,7 +1567,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
private QueryHandle executeAsyncInternal(LensSessionHandle sessionHandle, QueryContext ctx) throws LensException {
ctx.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0, QueryStatus.Status.QUEUED, "Query is queued", false, null, null, null));
+ ctx.setStatus(new QueryStatus(0.0, QUEUED, "Query is queued", false, null, null, null));
queuedQueries.add(ctx);
allQueries.put(ctx.getQueryHandle(), ctx);
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
@@ -1590,7 +1588,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("UpdateQueryConf:" + sessionHandle.toString() + " query: " + queryHandle);
acquire(sessionHandle);
QueryContext ctx = getQueryContext(sessionHandle, queryHandle);
- if (ctx != null && ctx.getStatus().getStatus() == QueryStatus.Status.QUEUED) {
+ if (ctx != null && ctx.getStatus().getStatus() == QUEUED) {
ctx.updateConf(newconf.getProperties());
// TODO COnf changed event tobe raised
return true;
@@ -1742,7 +1740,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
QueryHandle handle = executeAsyncInternal(sessionHandle, ctx);
QueryHandleWithResultSet result = new QueryHandleWithResultSet(handle);
// getQueryContext calls updateStatus, which fires query events if there's a change in status
- while (getQueryContext(sessionHandle, handle).getStatus().getStatus().equals(QueryStatus.Status.QUEUED)) {
+ while (getQueryContext(sessionHandle, handle).getStatus().getStatus().equals(QUEUED)) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
@@ -1753,17 +1751,24 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
if (getQueryContext(sessionHandle, handle).getSelectedDriver() == null) {
return result;
}
- getQueryContext(sessionHandle, handle).getSelectedDriver()
- .registerForCompletionNotification(handle, timeoutMillis, listener);
- try {
- synchronized (listener) {
- listener.wait(timeoutMillis);
+ synchronized (ctx) {
+ if (!ctx.getStatus().finished()) {
+ getQueryContext(sessionHandle, handle).getSelectedDriver()
+ .registerForCompletionNotification(handle, timeoutMillis, listener);
+ try {
+ synchronized (listener) {
+ listener.wait(timeoutMillis);
+ }
+ } catch (InterruptedException e) {
+ log.info("Waiting thread interrupted");
+ }
}
- } catch (InterruptedException e) {
- log.info("Waiting thread interrupted");
}
+
if (getQueryContext(sessionHandle, handle).getStatus().finished()) {
- result.setResult(getResultset(handle).toQueryResult());
+ if (getQueryContext(sessionHandle, handle).getStatus().isResultSetAvailable()) {
+ result.setResult(getResultset(handle).toQueryResult());
+ }
}
return result;
}
@@ -1901,8 +1906,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
return false;
}
synchronized (ctx) {
- if (ctx.getStatus().getStatus().equals(QueryStatus.Status.LAUNCHED)
- || ctx.getStatus().getStatus().equals(QueryStatus.Status.RUNNING)) {
+ if (ctx.getStatus().getStatus().equals(LAUNCHED)
+ || ctx.getStatus().getStatus().equals(RUNNING)) {
boolean ret = ctx.getSelectedDriver().cancelQuery(queryHandle);
if (!ret) {
return false;
@@ -1959,7 +1964,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
// Unless user wants to get queries in 'non finished' state, get finished queries from DB as well
- if (status == null || status == Status.CANCELED || status == Status.SUCCESSFUL || status == Status.FAILED) {
+ if (status == null || status == CANCELED || status == SUCCESSFUL || status == FAILED) {
if ("all".equalsIgnoreCase(userName)) {
userName = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index ec97b8d..19ed94a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -38,6 +38,7 @@ import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.annotations.MultiPurposeResource;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.api.query.cost.QueryCostTOBuilder;
import org.apache.lens.server.error.UnSupportedQuerySubmitOpException;
import org.apache.lens.server.model.LogSegregationContext;
@@ -197,7 +198,7 @@ public class QueryServiceResource {
* {@link org.apache.lens.api.query.SubmitOp#EXECUTE} operation.
* {@link QueryPlan} in case of {@link org.apache.lens.api.query.SubmitOp#EXPLAIN} operation.
* {@link QueryHandleWithResultSet} in case {@link org.apache.lens.api.query.SubmitOp#EXECUTE_WITH_TIMEOUT}
- * operation. {@link QueryCost} in case of {@link org.apache.lens.api.query.SubmitOp#ESTIMATE} operation.
+ * operation. {@link QueryCostTO} in case of {@link org.apache.lens.api.query.SubmitOp#ESTIMATE} operation.
*/
@POST
@Path("queries")
@@ -219,7 +220,7 @@ public class QueryServiceResource {
QuerySubmitResult result;
switch (sop) {
case ESTIMATE:
- result = queryServer.estimate(requestId, sessionid, query, conf);
+ result = new QueryCostTOBuilder(queryServer.estimate(requestId, sessionid, query, conf)).build();
break;
case EXECUTE:
result = queryServer.executeAsync(sessionid, query, conf, queryName);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
index 2d20789..80e29ba 100644
--- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
+++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
@@ -58,7 +58,7 @@ public class RestAPITestUtil {
Optional.of(userName), Optional.of(passwd), Optional.of(conf));
return target.path("session").request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- LensSessionHandle.class);
+ LensSessionHandle.class);
}
public static Response estimate(final WebTarget target, final Optional<LensSessionHandle> sessionId,
@@ -66,6 +66,10 @@ public class RestAPITestUtil {
return runQuery(target, sessionId, query, Optional.of("estimate"));
}
+ public static Response explain(final WebTarget target, final Optional<LensSessionHandle> sessionId,
+ final Optional<String> query) {
+ return runQuery(target, sessionId, query, Optional.of("explain"));
+ }
public static Response runQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId,
final Optional<String> query, final Optional<String> operation) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/2e5748a8/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 17305e2..76d3dee 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -39,9 +39,7 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.error.LensCommonErrorCode;
import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
-import org.apache.lens.api.result.LensAPIResult;
-import org.apache.lens.api.result.LensErrorTO;
-import org.apache.lens.api.result.LensJAXBContextResolver;
+import org.apache.lens.api.result.*;
import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.driver.hive.TestHiveDriver;
import org.apache.lens.server.LensJerseyTest;
@@ -1319,12 +1317,12 @@ public class TestQueryService extends LensJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(),
MediaType.APPLICATION_XML_TYPE));
- final QueryCost result = target.request()
+ final QueryCostTO result = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryCost>>() {}).getData();
+ new GenericType<LensAPIResult<QueryCostTO>>() {}).getData();
Assert.assertNotNull(result);
- Assert.assertEquals(result.getEstimatedExecTimeMillis(), 1L);
- Assert.assertEquals(result.getEstimatedResourceUsage(), 1.0);
+ Assert.assertEquals(result.getEstimatedExecTimeMillis(), null);
+ Assert.assertEquals(result.getEstimatedResourceUsage(), Double.MAX_VALUE);
}
@@ -1474,11 +1472,11 @@ public class TestQueryService extends LensJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
- final QueryCost queryCost = target.request()
+ final QueryCostTO queryCostTO = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryCost>>() {
+ new GenericType<LensAPIResult<QueryCostTO>>() {
}).getData();
- Assert.assertNotNull(queryCost);
+ Assert.assertNotNull(queryCostTO);
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
[43/50] [abbrv] incubator-lens git commit: LENS-699 : Check for
service to be up instead of using sleep in docker bootstrap
Posted by sh...@apache.org.
LENS-699 : Check for service to be up instead of using sleep in docker bootstrap
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c5c4f763
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c5c4f763
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c5c4f763
Branch: refs/heads/LENS-581
Commit: c5c4f7635f7bb53f58f75d16ca0f2d41c16a5c9c
Parents: 701d28b
Author: Sharad Agarwal <sh...@apache.org>
Authored: Tue Jul 28 13:08:53 2015 +0530
Committer: Sharad Agarwal <sh...@flipkarts-mbp.fkipl.flipkart.com>
Committed: Tue Jul 28 13:08:53 2015 +0530
----------------------------------------------------------------------
lens-docker/lens-test/lens-bootstrap.sh | 27 +++++++++++++++++++++++----
1 file changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c5c4f763/lens-docker/lens-test/lens-bootstrap.sh
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index bdc5e2c..98e448a 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -24,6 +24,11 @@ echo "LENS_CLIENT_CONF " $LENS_CLIENT_CONF
echo "LENS_ML " $LENS_ML
echo "SPARK_HOME " $SPARK_HOME
+#do the clean
+rm $LENS_HOME/*.log
+rm $LENS_HOME/logs/*
+rm -rf $LENS_HOME/metastore_db
+
#set ml classpath into LENS_EXT_CLASSPATH
LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh`
export LENS_EXT_CLASSPATH
@@ -40,15 +45,29 @@ echo "HIVE_AUX_JARS_PATH " $HIVE_AUX_JARS_PATH
#start hive bootstrap script
/etc/hive-bootstrap.sh
-echo "Waiting for 20 secs for servers to start ..."
-sleep 20
+checkservice() {
+ servicename=$1
+ port=$2
+ up=`netstat -plnt |grep :$port.*LISTEN | wc -l`
+ while [ $up != "1" ]
+ do
+ echo "$servicename not yet up at $port. waiting for 5 sec"
+ sleep 5
+ up=`netstat -plnt |grep :$port.*LISTEN | wc -l`
+ done
+ echo "$servicename up at $port"
+}
+
+checkservice "Namenode" 9000
+checkservice "RM" 8088
+checkservice "HiveMetastore" 9083
+checkservice "HiveServer" 10000
#start lens server
echo "Starting Lens server..."
$LENS_HOME/bin/lens-ctl start --conf $LENS_SERVER_CONF
-echo "Waiting for 60 secs for Lens Server to start ..."
-sleep 60
+checkservice "LensServer" 9999
#Setting up client
$LENS_CLIENT/bin/run-examples.sh sample-metastore --conf $LENS_CLIENT_CONF
[20/50] [abbrv] incubator-lens git commit: LENS-667 : Use SLF4j
logger in driver modules
Posted by sh...@apache.org.
LENS-667 : Use SLF4j logger in driver modules
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d8e071c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d8e071c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d8e071c1
Branch: refs/heads/LENS-581
Commit: d8e071c1eb253a0077f595ac448e32f6c359c4bb
Parents: 9492add
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Tue Jul 21 13:16:45 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Tue Jul 21 13:16:45 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/hive/HiveDriver.java | 80 ++++++++++----------
.../apache/lens/driver/hive/HiveQueryPlan.java | 2 +-
.../driver/hive/RemoteThriftConnection.java | 19 +++--
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 34 ++++-----
.../jdbc/DataSourceConnectionProvider.java | 13 ++--
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 51 ++++++-------
.../apache/lens/driver/jdbc/JDBCResultSet.java | 11 ++-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 6 +-
8 files changed, 105 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 89d671f..7c3e3e0 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -179,7 +179,7 @@ public class HiveDriver implements LensDriver {
if (thConn != null) {
try {
- log.info("Closed connection:" + expired.getConnId());
+ log.info("Closed connection: {}", expired.getConnId());
thConn.close();
} catch (IOException e) {
log.error("Error closing connection", e);
@@ -361,7 +361,7 @@ public class HiveDriver implements LensDriver {
@Override
public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
- log.info("Estimate: " + qctx.getDriverQuery(this));
+ log.info("Estimate: {}", qctx.getDriverQuery(this));
if (qctx.getDriverQuery(this) == null) {
throw new NullPointerException("Null driver query for " + qctx.getUserQuery());
}
@@ -396,7 +396,7 @@ public class HiveDriver implements LensDriver {
// explain called again and again
return (HiveQueryPlan) explainCtx.getDriverContext().getDriverQueryPlan(this);
}
- log.info("Explain: " + explainCtx.getDriverQuery(this));
+ log.info("Explain: {}", explainCtx.getDriverQuery(this));
Configuration explainConf = new Configuration(explainCtx.getDriverConf(this));
explainConf.setClassLoader(explainCtx.getConf().getClassLoader());
explainConf.setBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false);
@@ -475,7 +475,7 @@ public class HiveDriver implements LensDriver {
qdconf.set("mapred.job.name", ctx.getQueryHandle().toString());
OperationHandle op = getClient().executeStatement(getSession(ctx), ctx.getSelectedDriverQuery(),
qdconf.getValByRegex(".*"));
- log.info("The hive operation handle: " + op);
+ log.info("The hive operation handle: {}", op);
ctx.setDriverOpHandle(op.toString());
hiveHandles.put(ctx.getQueryHandle(), op);
updateStatus(ctx);
@@ -512,7 +512,7 @@ public class HiveDriver implements LensDriver {
Configuration qdconf = ctx.getDriverConf(this);
qdconf.set("mapred.job.name", ctx.getQueryHandle().toString());
//Query is already explained.
- log.info("whetherCalculatePriority: " + whetherCalculatePriority);
+ log.info("whetherCalculatePriority: {}", whetherCalculatePriority);
if (whetherCalculatePriority) {
try {
// Inside try since non-data fetching queries can also be executed by async method.
@@ -521,19 +521,19 @@ public class HiveDriver implements LensDriver {
}
String priority = queryPriorityDecider.decidePriority(ctx.getDriverQueryCost(this)).toString();
qdconf.set("mapred.job.priority", priority);
- log.info("set priority to " + priority);
+ log.info("set priority to {}", priority);
} catch (Exception e) {
// not failing query launch when setting priority fails
// priority will be set to usually NORMAL - the default in underlying system.
- log.error("could not set priority for lens session id:" + ctx.getLensSessionIdentifier()
- + "User query: " + ctx.getUserQuery(), e);
+ log.error("could not set priority for lens session id:{} User query: {}", ctx.getLensSessionIdentifier(),
+ ctx.getUserQuery(), e);
}
}
queryHook.preLaunch(ctx);
OperationHandle op = getClient().executeStatementAsync(getSession(ctx), ctx.getSelectedDriverQuery(),
qdconf.getValByRegex(".*"));
ctx.setDriverOpHandle(op.toString());
- log.info("QueryHandle: " + ctx.getQueryHandle() + " HiveHandle:" + op);
+ log.info("QueryHandle: {} HiveHandle:{}", ctx.getQueryHandle(), op);
hiveHandles.put(ctx.getQueryHandle(), op);
} catch (IOException e) {
throw new LensException("Error adding persistent path", e);
@@ -550,7 +550,7 @@ public class HiveDriver implements LensDriver {
*/
@Override
public void updateStatus(QueryContext context) throws LensException {
- log.debug("GetStatus: " + context.getQueryHandle());
+ log.debug("GetStatus: {}", context.getQueryHandle());
if (context.getDriverStatus().isFinished()) {
return;
}
@@ -558,9 +558,9 @@ public class HiveDriver implements LensDriver {
ByteArrayInputStream in = null;
try {
// Get operation status from hive server
- log.debug("GetStatus hiveHandle: " + hiveHandle);
+ log.debug("GetStatus hiveHandle: {}", hiveHandle);
OperationStatus opStatus = getClient().getOperationStatus(hiveHandle);
- log.debug("GetStatus on hiveHandle: " + hiveHandle + " returned state:" + opStatus.getState().name());
+ log.debug("GetStatus on hiveHandle: {} returned state:", hiveHandle, opStatus.getState().name());
switch (opStatus.getState()) {
case CANCELED:
@@ -672,7 +672,7 @@ public class HiveDriver implements LensDriver {
*/
@Override
public LensResultSet fetchResultSet(QueryContext ctx) throws LensException {
- log.info("FetchResultSet: " + ctx.getQueryHandle());
+ log.info("FetchResultSet: {}", ctx.getQueryHandle());
// This should be applicable only for a async query
return createResultSet(ctx, false);
}
@@ -697,10 +697,10 @@ public class HiveDriver implements LensDriver {
if (handle == null) {
return;
}
- log.info("CloseQuery: " + handle);
+ log.info("CloseQuery: {}", handle);
OperationHandle opHandle = hiveHandles.remove(handle);
if (opHandle != null) {
- log.info("CloseQuery: " + opHandle);
+ log.info("CloseQuery hiveHandle: {}", opHandle);
try {
getClient().closeOperation(opHandle);
} catch (HiveSQLException e) {
@@ -717,10 +717,10 @@ public class HiveDriver implements LensDriver {
*/
@Override
public boolean cancelQuery(QueryHandle handle) throws LensException {
- log.info("CancelQuery: " + handle);
+ log.info("CancelQuery: {}", handle);
OperationHandle hiveHandle = getHiveHandle(handle);
try {
- log.info("CancelQuery hiveHandle: " + hiveHandle);
+ log.info("CancelQuery hiveHandle: {}", hiveHandle);
getClient().cancelOperation(hiveHandle);
return true;
} catch (HiveSQLException e) {
@@ -745,8 +745,8 @@ public class HiveDriver implements LensDriver {
getClient().closeSession(lensToHiveSession.get(lensSessionDbKey));
} catch (Exception e) {
checkInvalidSession(e);
- log.warn("Error closing session for lens session: " + lensSessionDbKey + ", hive session: "
- + lensToHiveSession.get(lensSessionDbKey), e);
+ log.warn("Error closing session for lens session: {}, hive session: ", lensSessionDbKey,
+ lensToHiveSession.get(lensSessionDbKey), e);
}
}
lensToHiveSession.clear();
@@ -774,7 +774,7 @@ public class HiveDriver implements LensDriver {
} catch (Exception e) {
throw new LensException(e);
}
- log.info("New thrift connection " + connectionClass);
+ log.info("New thrift connection {}", connectionClass);
}
return embeddedConnection.getClient();
} else {
@@ -792,8 +792,8 @@ public class HiveDriver implements LensDriver {
connection = new ExpirableConnection(tconn, connectionExpiryTimeout);
thriftConnExpiryQueue.offer(connection);
threadConnections.put(connectionKey, connection);
- log.info("New thrift connection " + connectionClass + " for thread:" + Thread.currentThread().getId()
- + " for user:" + user + " connection ID=" + connection.getConnId());
+ log.info("New thrift connection {} for thread: {} for user: {} connection ID={}", connectionClass,
+ Thread.currentThread().getId(), user, connection.getConnId());
} catch (Exception e) {
throw new LensException(e);
}
@@ -817,7 +817,7 @@ public class HiveDriver implements LensDriver {
*/
private LensResultSet createResultSet(QueryContext context, boolean closeAfterFetch) throws LensException {
OperationHandle op = getHiveHandle(context.getQueryHandle());
- log.info("Creating result set for hiveHandle:" + op);
+ log.info("Creating result set for hiveHandle:{}", op);
try {
if (context.isDriverPersistent()) {
return new HivePersistentResultSet(new Path(context.getHdfsoutPath()), op, getClient());
@@ -864,7 +864,7 @@ public class HiveDriver implements LensDriver {
context.unSetDriverPersistent();
hiveQuery = context.getSelectedDriverQuery();
}
- log.info("Hive driver query:" + hiveQuery);
+ log.info("Hive driver query:{}", hiveQuery);
context.setSelectedDriverQuery(hiveQuery);
}
@@ -893,14 +893,14 @@ public class HiveDriver implements LensDriver {
try {
hiveSession = getClient().openSession(ctx.getClusterUser(), "");
lensToHiveSession.put(sessionDbKey, hiveSession);
- log.info("New hive session for user: " + ctx.getClusterUser() + ", lens session: " + sessionDbKey
- + " hive session handle: " + hiveSession.getHandleIdentifier());
+ log.info("New hive session for user: {} , lens session: {} hive session handle: {}", ctx.getClusterUser(),
+ sessionDbKey, hiveSession.getHandleIdentifier());
for (LensEventListener<DriverEvent> eventListener : driverListeners) {
try {
eventListener.onEvent(new DriverSessionStarted(System.currentTimeMillis(), this, lensSession, hiveSession
.getSessionId().toString()));
} catch (Exception exc) {
- log.error("Error sending driver start event to listener " + eventListener, exc);
+ log.error("Error sending driver start event to listener {}", eventListener, exc);
}
}
} catch (Exception e) {
@@ -1048,9 +1048,9 @@ public class HiveDriver implements LensDriver {
QueryHandle qhandle = (QueryHandle) in.readObject();
OperationHandle opHandle = new OperationHandle((TOperationHandle) in.readObject());
hiveHandles.put(qhandle, opHandle);
- log.debug("Hive driver recovered " + qhandle + ":" + opHandle);
+ log.debug("Hive driver recovered {}:{}", qhandle, opHandle);
}
- log.info("HiveDriver recovered " + hiveHandles.size() + " queries");
+ log.info("HiveDriver recovered {} queries", hiveHandles.size());
int numSessions = in.readInt();
for (int i = 0; i < numSessions; i++) {
String lensId = in.readUTF();
@@ -1058,7 +1058,7 @@ public class HiveDriver implements LensDriver {
TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6);
lensToHiveSession.put(lensId, sHandle);
}
- log.info("HiveDriver recovered " + lensToHiveSession.size() + " sessions");
+ log.info("HiveDriver recovered {} sessions", lensToHiveSession.size());
}
}
@@ -1075,15 +1075,15 @@ public class HiveDriver implements LensDriver {
for (Map.Entry<QueryHandle, OperationHandle> entry : hiveHandles.entrySet()) {
out.writeObject(entry.getKey());
out.writeObject(entry.getValue().toTOperationHandle());
- log.debug("Hive driver persisted " + entry.getKey() + ":" + entry.getValue());
+ log.debug("Hive driver persisted {}:{}", entry.getKey(), entry.getValue());
}
- log.info("HiveDriver persisted " + hiveHandles.size() + " queries");
+ log.info("HiveDriver persisted {} queries ", hiveHandles.size());
out.writeInt(lensToHiveSession.size());
for (Map.Entry<String, SessionHandle> entry : lensToHiveSession.entrySet()) {
out.writeUTF(entry.getKey());
out.writeObject(entry.getValue().toTSessionHandle());
}
- log.info("HiveDriver persisted " + lensToHiveSession.size() + " sessions");
+ log.info("HiveDriver persisted {} sessions", lensToHiveSession.size());
}
}
@@ -1145,7 +1145,7 @@ public class HiveDriver implements LensDriver {
for (SessionHandle session : hiveSessionsToCheck) {
if (isSessionInvalid(exc, session)) {
// We have to expire previous session
- log.info("Hive server session " + session + " for lens session " + lensSession + " has become invalid");
+ log.info("Hive server session {} for lens session {} has become invalid", session, lensSession);
sessionLock.lock();
try {
// We should close all connections and clear the session map since
@@ -1169,12 +1169,12 @@ public class HiveDriver implements LensDriver {
protected void checkInvalidOperation(QueryHandle queryHandle, HiveSQLException exc) {
final OperationHandle operation = hiveHandles.get(queryHandle);
if (operation == null) {
- log.info("No hive operation available for " + queryHandle);
+ log.info("No hive operation available for {}", queryHandle);
return;
}
if (exc.getMessage() != null && exc.getMessage().contains("Invalid OperationHandle:")
&& exc.getMessage().contains(operation.toString())) {
- log.info("Hive operation " + operation + " for query " + queryHandle + " has become invalid");
+ log.info("Hive operation {} for query {} has become invalid", operation, queryHandle);
hiveHandles.remove(queryHandle);
return;
}
@@ -1216,11 +1216,11 @@ public class HiveDriver implements LensDriver {
if (hiveSession != null) {
try {
getClient().closeSession(hiveSession);
- log.info("Closed Hive session " + hiveSession.getHandleIdentifier() + " for lens session "
- + sessionDbKey);
+ log.info("Closed Hive session {} for lens session {}", hiveSession.getHandleIdentifier(),
+ sessionDbKey);
} catch (Exception e) {
- log.error("Error closing hive session " + hiveSession.getHandleIdentifier()
- + " for lens session " + sessionDbKey, e);
+ log.error("Error closing hive session {} for lens session {}", hiveSession.getHandleIdentifier(),
+ sessionDbKey, e);
}
resourcesAddedForSession.remove(hiveSession);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
index f38b1da..f53e2b6 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveQueryPlan.java
@@ -166,7 +166,7 @@ public class HiveQueryPlan extends DriverQueryPlan {
Table tbl = metastore.getTable(table, false);
if (tbl == null) {
// table not found, possible case if query is create table
- log.info("Table " + table + " not found while extracting plan details");
+ log.info("Table {} not found while extracting plan details", table);
continue;
}
tablesQueried.add(table);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
index 0104670..741294c 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/RemoteThriftConnection.java
@@ -20,21 +20,19 @@ package org.apache.lens.driver.hive;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.cli.CLIServiceClient;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.thrift.RetryingThriftCLIServiceClient;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Connect to a remote Hive Server 2 service to run driver queries.
*/
+@Slf4j
public class RemoteThriftConnection implements ThriftConnection {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(RemoteThriftConnection.class);
-
/** The connected. */
private boolean connected;
@@ -64,12 +62,13 @@ public class RemoteThriftConnection implements ThriftConnection {
public CLIServiceClient getClient() throws LensException {
if (!connected) {
try {
- LOG.info("HiveDriver connecting to HiveServer @ "
- + conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST) + ":"
- + conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
+ log.info("HiveDriver connecting to HiveServer @ {}:{}",
+ conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST),
+ conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
hs2Client = RetryingThriftCLIServiceClient.newRetryingCLIServiceClient(conf);
- LOG.info("HiveDriver connected to HiveServer @ " + conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST)
- + ":" + conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
+ log.info("HiveDriver connected to HiveServer @ {}:{}",
+ conf.getVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST),
+ conf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT));
} catch (HiveSQLException e) {
throw new LensException(e);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 18f84cd..0556d0c 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -31,8 +31,6 @@ import org.apache.lens.server.api.query.rewrite.QueryRewriter;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.mutable.MutableInt;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -44,9 +42,12 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.antlr.runtime.CommonToken;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class ColumnarSQLRewriter.
*/
+@Slf4j
public class ColumnarSQLRewriter implements QueryRewriter {
/** The clause name. */
@@ -115,9 +116,6 @@ public class ColumnarSQLRewriter implements QueryRewriter {
/** The map aliases. */
private final Map<String, String> mapAliases = new HashMap<String, String>();
- /** The Constant LOG. */
- private static final Log LOG = LogFactory.getLog(ColumnarSQLRewriter.class);
-
/** The where tree. */
private String whereTree;
@@ -197,7 +195,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
}
if (!qb.getSubqAliases().isEmpty()) {
- LOG.warn("Subqueries in from clause is not supported by " + this + " Query : " + this.query);
+ log.warn("Subqueries in from clause is not supported by {} Query : {}", this, this.query);
throw new SemanticException("Subqueries in from clause is not supported by " + this + " Query : " + this.query);
}
@@ -302,7 +300,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
} else if (joinToken.equals("TOK_UNIQUEJOIN")) {
joinType = "unique join";
} else {
- LOG.info("Non supported join type : " + joinToken);
+ log.info("Non supported join type : {}", joinToken);
}
if (node.getChildCount() > 2) {
@@ -382,7 +380,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
public void getFilterInJoinCond(ASTNode node) {
if (node == null) {
- LOG.debug("Join AST is null ");
+ log.debug("Join AST is null ");
return;
}
@@ -422,7 +420,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
public void factFilterPushDown(ASTNode node) {
if (node == null) {
- LOG.debug("Join AST is null ");
+ log.debug("Join AST is null ");
return;
}
@@ -460,7 +458,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
*/
public void getFactKeysFromNode(ASTNode node) {
if (node == null) {
- LOG.debug("AST is null ");
+ log.debug("AST is null ");
return;
}
if (node.getToken().getType() == HiveParser.DOT
@@ -507,7 +505,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
*/
public void buildSubqueries(ASTNode node) {
if (node == null) {
- LOG.debug("Join AST is null ");
+ log.debug("Join AST is null ");
return;
}
@@ -880,7 +878,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
if (whereTree == null || joinTree == null || allSubQueries.length() == 0
|| aggColumn.isEmpty() || isExpressionsUsed(selectAST)) {
- LOG.info("@@@Query not eligible for inner subquery rewrite");
+ log.info("@@@Query not eligible for inner subquery rewrite");
// construct query without fact sub query
constructQuery(selectTree, whereTree, groupByTree, havingTree, orderByTree, limit);
return;
@@ -1079,7 +1077,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
String finalRewrittenQuery = "";
String[] queries = query.toLowerCase().split("union all");
for (int i = 0; i < queries.length; i++) {
- LOG.info("Union Query Part " + i + " : " + queries[i]);
+ log.info("Union Query Part {} : {}", i, queries[i]);
ast = HQLParser.parseHQL(queries[i], metastoreConf);
buildQuery(conf, metastoreConf);
mergedQuery = rewrittenQuery.append(" union all ");
@@ -1087,14 +1085,14 @@ public class ColumnarSQLRewriter implements QueryRewriter {
reset();
}
queryReplacedUdf = replaceUDFForDB(finalRewrittenQuery);
- LOG.info("Input Query : " + query);
- LOG.info("Rewritten Query : " + queryReplacedUdf);
+ log.info("Input Query : {}", query);
+ log.info("Rewritten Query : {}", queryReplacedUdf);
} else {
ast = HQLParser.parseHQL(query, metastoreConf);
buildQuery(conf, metastoreConf);
queryReplacedUdf = replaceUDFForDB(rewrittenQuery.toString());
- LOG.info("Input Query : " + query);
- LOG.info("Rewritten Query : " + queryReplacedUdf);
+ log.info("Input Query : {}", query);
+ log.info("Rewritten Query : {}", queryReplacedUdf);
}
} catch (SemanticException e) {
throw new LensException(e);
@@ -1158,7 +1156,7 @@ public class ColumnarSQLRewriter implements QueryRewriter {
}
}
} catch (HiveException e) {
- LOG.warn("No corresponding table in metastore:" + e.getMessage());
+ log.warn("No corresponding table in metastore:", e);
}
} else {
for (int i = 0; i < tree.getChildCount(); i++) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java
index 7b454fa..b14835a 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/DataSourceConnectionProvider.java
@@ -28,18 +28,17 @@ import java.util.Map;
import org.apache.commons.lang.builder.EqualsBuilder;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.Logger;
import com.mchange.v2.c3p0.ComboPooledDataSource;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class DataSourceConnectionProvider.
*/
+@Slf4j
public class DataSourceConnectionProvider implements ConnectionProvider {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(DataSourceConnectionProvider.class);
-
/** The data source map. */
private Map<DriverConfig, ComboPooledDataSource> dataSourceMap;
@@ -187,7 +186,7 @@ public class DataSourceConnectionProvider implements ConnectionProvider {
cpds.setCheckoutTimeout(conf.getInt(JDBCDriverConfConstants.JDBC_GET_CONNECTION_TIMEOUT,
JDBCDriverConfConstants.JDBC_GET_CONNECTION_TIMEOUT_DEFAULT));
dataSourceMap.put(config, cpds);
- LOG.info("Created new datasource for config: " + config);
+ log.info("Created new datasource for config: {}", config);
}
return dataSourceMap.get(config).getConnection();
}
@@ -201,10 +200,10 @@ public class DataSourceConnectionProvider implements ConnectionProvider {
public void close() throws IOException {
for (Map.Entry<DriverConfig, ComboPooledDataSource> entry : dataSourceMap.entrySet()) {
entry.getValue().close();
- LOG.info("Closed datasource: " + entry.getKey());
+ log.info("Closed datasource: {}", entry.getKey());
}
dataSourceMap.clear();
- LOG.info("Closed datasource connection provider");
+ log.info("Closed datasource connection provider");
}
protected final ComboPooledDataSource getDataSource(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 9445436..e3a8799 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -54,20 +54,19 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
-import org.apache.log4j.Logger;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
/**
* This driver is responsible for running queries against databases which can be queried using the JDBC API.
*/
+@Slf4j
public class JDBCDriver implements LensDriver {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(JDBCDriver.class);
-
/** The Constant THID. */
public static final AtomicInteger THID = new AtomicInteger();
@@ -233,7 +232,7 @@ public class JDBCDriver implements LensDriver {
try {
stmt.close();
} catch (SQLException e) {
- LOG.error("Error closing SQL statement", e);
+ log.error("Error closing SQL statement", e);
}
}
} finally {
@@ -241,7 +240,7 @@ public class JDBCDriver implements LensDriver {
try {
conn.close();
} catch (SQLException e) {
- LOG.error("Error closing SQL Connection", e);
+ log.error("Error closing SQL Connection", e);
}
}
}
@@ -306,7 +305,7 @@ public class JDBCDriver implements LensDriver {
conn = getConnection();
result.conn = conn;
} catch (LensException e) {
- LOG.error("Error obtaining connection: " + e.getMessage(), e);
+ log.error("Error obtaining connection: ", e);
result.error = e;
}
@@ -321,11 +320,11 @@ public class JDBCDriver implements LensDriver {
queryContext.notifyComplete();
} catch (SQLException sqlEx) {
if (queryContext.isClosed()) {
- LOG.info("Ignored exception on already closed query: " + queryContext.getLensContext().getQueryHandle()
- + " - " + sqlEx);
+ log.info("Ignored exception on already closed query : {} - {}",
+ queryContext.getLensContext().getQueryHandle(), sqlEx.getMessage());
} else {
- LOG.error("Error executing SQL query: " + queryContext.getLensContext().getQueryHandle() + " reason: "
- + sqlEx.getMessage(), sqlEx);
+ log.error("Error executing SQL query: {} reason: {}", queryContext.getLensContext().getQueryHandle(),
+ sqlEx.getMessage(), sqlEx);
result.error = sqlEx;
// Close connection in case of failed queries. For successful queries, connection is closed
// When result set is closed or driver.closeQuery is called
@@ -355,7 +354,7 @@ public class JDBCDriver implements LensDriver {
JDBCDriverConfConstants.DEFAULT_JDBC_ENABLE_RESULTSET_STREAMING_RETRIEVAL);
if (enabledRowRetrieval) {
- LOG.info("JDBC streaming retrieval is enabled for " + queryContext.getLensContext().getQueryHandle());
+ log.info("JDBC streaming retrieval is enabled for {}", queryContext.getLensContext().getQueryHandle());
if (queryContext.isPrepared()) {
stmt = conn.prepareStatement(queryContext.getRewrittenQuery(),
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
@@ -427,7 +426,7 @@ public class JDBCDriver implements LensDriver {
throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
}
configured = true;
- LOG.info("JDBC Driver configured");
+ log.info("JDBC Driver configured");
}
/**
@@ -453,7 +452,7 @@ public class JDBCDriver implements LensDriver {
connectionProvider = cpClass.newInstance();
estimateConnectionProvider = cpClass.newInstance();
} catch (Exception e) {
- LOG.error("Error initializing connection provider: " + e.getMessage(), e);
+ log.error("Error initializing connection provider: ", e);
throw new LensException(e);
}
this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
@@ -492,9 +491,9 @@ public class JDBCDriver implements LensDriver {
DummyQueryRewriter.class, QueryRewriter.class);
try {
rewriter = queryRewriterClass.newInstance();
- LOG.info("Initialized :" + queryRewriterClass);
+ log.info("Initialized :{}", queryRewriterClass);
} catch (Exception e) {
- LOG.error("Unable to create rewriter object", e);
+ log.error("Unable to create rewriter object", e);
throw new LensException(e);
}
rewriter.init(conf);
@@ -614,7 +613,7 @@ public class JDBCDriver implements LensDriver {
explainQuery = rewrittenQuery.replaceAll("select ", "select "
+ explainKeyword + " ");
}
- LOG.info("Explain Query : " + explainQuery);
+ log.info("Explain Query : {}", explainQuery);
QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null,
new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier(), false);
QueryResult result = null;
@@ -784,12 +783,12 @@ public class JDBCDriver implements LensDriver {
try {
conn.close();
} catch (SQLException e) {
- LOG.error("Error closing connection: " + rewrittenQuery, e);
+ log.error("Error closing connection: {}", rewrittenQuery, e);
}
}
jdbcPrepareGauge.markSuccess();
}
- LOG.info("Prepared: " + rewrittenQuery);
+ log.info("Prepared: {}", rewrittenQuery);
return stmt;
}
@@ -858,7 +857,7 @@ public class JDBCDriver implements LensDriver {
checkConfigured();
String rewrittenQuery = rewriteQuery(context);
- LOG.info("Execute " + context.getQueryHandle());
+ log.info("Execute {}", context.getQueryHandle());
QueryResult result = executeInternal(context, rewrittenQuery);
return result.getLensResultSet(true);
@@ -901,11 +900,11 @@ public class JDBCDriver implements LensDriver {
Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext));
jdbcCtx.setResultFuture(future);
} catch (RejectedExecutionException e) {
- LOG.error("Query execution rejected: " + context.getQueryHandle() + " reason:" + e.getMessage(), e);
+ log.error("Query execution rejected: {} reason:{}", context.getQueryHandle(), e.getMessage(), e);
throw new LensException("Query execution rejected: " + context.getQueryHandle() + " reason:" + e.getMessage(), e);
}
queryContextMap.put(context.getQueryHandle(), jdbcCtx);
- LOG.info("ExecuteAsync: " + context.getQueryHandle());
+ log.info("ExecuteAsync: {}", context.getQueryHandle());
}
/**
@@ -1018,7 +1017,7 @@ public class JDBCDriver implements LensDriver {
context.setEndTime(System.currentTimeMillis());
}
context.closeResult();
- LOG.info("Cancelled query: " + handle);
+ log.info("Cancelled query: {}", handle);
}
return cancelResult;
}
@@ -1039,7 +1038,7 @@ public class JDBCDriver implements LensDriver {
} finally {
queryContextMap.remove(handle);
}
- LOG.info("Closed query " + handle.getHandleId());
+ log.info("Closed query {}", handle.getHandleId());
}
/**
@@ -1055,7 +1054,7 @@ public class JDBCDriver implements LensDriver {
try {
closeQuery(query);
} catch (LensException e) {
- LOG.warn("Error closing query : " + query.getHandleId(), e);
+ log.warn("Error closing query : {}", query.getHandleId(), e);
}
}
for (QueryPrepareHandle query : new ArrayList<QueryPrepareHandle>(preparedQueries.keySet())) {
@@ -1066,7 +1065,7 @@ public class JDBCDriver implements LensDriver {
throw new LensException();
}
} catch (LensException e) {
- LOG.warn("Error closing prapared query : " + query, e);
+ log.warn("Error closing prapared query : {}", query, e);
}
}
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
index 2687673..f80cdd8 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
@@ -37,7 +37,6 @@ import org.apache.hive.service.cli.ColumnDescriptor;
import org.apache.hive.service.cli.Type;
import org.apache.hive.service.cli.TypeDescriptor;
import org.apache.hive.service.cli.TypeQualifiers;
-import org.apache.log4j.Logger;
import org.codehaus.jackson.annotate.JsonIgnore;
@@ -46,14 +45,14 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class JDBCResultSet.
*/
+@Slf4j
public class JDBCResultSet extends InMemoryResultSet {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(JDBCResultSet.class);
-
/** The result meta. */
ResultSetMetaData resultMeta;
@@ -101,7 +100,7 @@ public class JDBCResultSet extends InMemoryResultSet {
*/
@Override
public int size() throws LensException {
- LOG.warn("Size of result set is not supported");
+ log.warn("Size of result set is not supported");
return -1;
}
@@ -120,7 +119,7 @@ public class JDBCResultSet extends InMemoryResultSet {
jdbcResultSetMetadata.getFieldSchemas().add(col);
}
} catch (Exception e) {
- LOG.error("Error getting JDBC type information: " + e.getMessage(), e);
+ log.error("Error getting JDBC type information: {}", e.getMessage(), e);
jdbcResultSetMetadata.setFieldSchemas(null);
}
lensResultMeta = jdbcResultSetMetadata;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d8e071c1/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index f534b69..9041709 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -261,7 +261,7 @@ public class TestJdbcDriver {
driver.getEstimateConnectionConf().getInt(JDBCDriverConfConstants.JDBC_POOL_MAX_SIZE, 50);
for (int i = 0; i < maxEstimateConnections + 10; i++) {
try {
- log.info("Iteration#" + (i + 1));
+ log.info("Iteration#{}", (i + 1));
String query = i > maxEstimateConnections ? "SELECT * FROM estimate_test" : "CREATE TABLE FOO(ID INT)";
ExplainQueryContext context = createExplainContext(query, baseConf);
cost = driver.estimate(context);
@@ -807,8 +807,8 @@ public class TestJdbcDriver {
DataSourceConnectionProvider.DriverConfig queryCfg =
queryCp.getDriverConfigfromConf(driver.getConf());
- log.info("@@@ ESTIMATE_CFG " + estimateCfg);
- log.info("@@@ QUERY CFG " + queryCfg);
+ log.info("@@@ ESTIMATE_CFG {}", estimateCfg);
+ log.info("@@@ QUERY CFG {}", queryCfg);
// Get connection from each so that pools get initialized
try {
[32/50] [abbrv] incubator-lens git commit: LENS-668 : Use Slf4j in
lens-server module
Posted by sh...@apache.org.
LENS-668 : Use Slf4j in lens-server module
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/44ec2afd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/44ec2afd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/44ec2afd
Branch: refs/heads/LENS-581
Commit: 44ec2afd9c7f2f060b564a9e5d4e51cbad2d9e51
Parents: c879f99
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Thu Jul 23 13:23:26 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 23 13:23:27 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/ml/TestMLRunner.java | 7 +-
.../lens/server/AuthenticationFilter.java | 9 +-
.../apache/lens/server/EventServiceImpl.java | 22 ++--
.../org/apache/lens/server/LensApplication.java | 18 ++--
.../lens/server/LensApplicationListener.java | 12 +--
.../java/org/apache/lens/server/LensServer.java | 24 ++---
.../org/apache/lens/server/LensService.java | 20 ++--
.../org/apache/lens/server/LensServices.java | 48 ++++-----
.../server/metastore/MetastoreResource.java | 108 +++++++++----------
.../lens/server/query/QueryEndNotifier.java | 15 ++-
.../server/query/QueryExecutionServiceImpl.java | 11 +-
.../lens/server/query/ResultFormatter.java | 29 +++--
.../apache/lens/server/rewrite/RewriteUtil.java | 31 +++---
.../server/session/DatabaseResourceService.java | 45 ++++----
.../lens/server/session/LensSessionImpl.java | 17 ++-
.../lens/server/session/SessionResource.java | 11 +-
.../store/log/StatisticsLogFileScannerTask.java | 19 ++--
.../lens/server/ui/MetastoreUIResource.java | 46 ++++----
.../lens/server/ui/QueryServiceUIResource.java | 12 +--
.../lens/server/ui/SessionUIResource.java | 11 +-
.../lens/server/ui/StaticFileResource.java | 5 -
.../org/apache/lens/server/LensJerseyTest.java | 19 ++--
.../org/apache/lens/server/TestResource.java | 6 --
.../apache/lens/server/TestServerRestart.java | 58 +++++-----
.../lens/server/TestStartupOnMetastoreDown.java | 9 +-
.../lens/server/TestStaticFileResource.java | 5 -
.../server/metastore/TestMetastoreService.java | 2 -
.../metrics/TestResourceMethodMetrics.java | 16 +--
.../lens/server/query/TestEventService.java | 44 ++++----
.../apache/lens/server/query/TestLensDAO.java | 7 +-
.../server/query/TestQueryEndEmailNotifier.java | 4 -
.../lens/server/query/TestResultFormatting.java | 9 +-
.../server/session/TestSessionClassLoaders.java | 28 ++---
.../lens/server/user/TestUserConfigLoader.java | 3 -
34 files changed, 329 insertions(+), 401 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
index 67e3379..ef3d53e 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
@@ -46,8 +46,11 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
@Test
+@Slf4j
public class TestMLRunner extends LensJerseyTest {
private static final String TEST_DB = TestMLRunner.class.getSimpleName();
@@ -98,7 +101,7 @@ public class TestMLRunner extends LensJerseyTest {
@Test
public void trainAndEval() throws Exception {
- LOG.info("Starting train & eval");
+ log.info("Starting train & eval");
String algoName = "spark_naive_bayes";
String database = "default";
String trainTable = "naivebayes_training_table";
@@ -122,7 +125,7 @@ public class TestMLRunner extends LensJerseyTest {
@Test
public void trainAndEvalFromDir() throws Exception {
- LOG.info("Starting train & eval from Dir");
+ log.info("Starting train & eval from Dir");
MLRunner runner = new MLRunner();
runner.init(mlClient, "data/naive_bayes");
MLTask task = runner.train();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
index e39c7a2..e982520 100644
--- a/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/AuthenticationFilter.java
@@ -24,17 +24,14 @@ import javax.ws.rs.container.ContainerRequestContext;
import javax.ws.rs.container.ContainerRequestFilter;
import javax.ws.rs.core.SecurityContext;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class AuthenticationFilter.
*/
+@Slf4j
public class AuthenticationFilter implements ContainerRequestFilter {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(AuthenticationFilter.class);
-
/*
* (non-Javadoc)
*
@@ -45,6 +42,6 @@ public class AuthenticationFilter implements ContainerRequestFilter {
final SecurityContext securityContext = requestContext.getSecurityContext();
String user = securityContext.getUserPrincipal() != null ? securityContext.getUserPrincipal().getName() : null;
- LOG.info("Request from user: " + user + ", path=" + requestContext.getUriInfo().getPath());
+ log.info("Request from user: {} , path={}", user, requestContext.getUriInfo().getPath());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
index 5855ac6..2c6f912 100644
--- a/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
@@ -28,19 +28,17 @@ import org.apache.lens.server.api.events.LensEvent;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.events.LensEventService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.AbstractService;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Implementation of LensEventService
*/
+@Slf4j
public class EventServiceImpl extends AbstractService implements LensEventService {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(EventServiceImpl.class);
-
/** The event listeners. */
private final Map<Class<? extends LensEvent>, List<LensEventListener>> eventListeners
= new HashMap<Class<? extends LensEvent>, List<LensEventListener>>();
@@ -82,7 +80,7 @@ public class EventServiceImpl extends AbstractService implements LensEventServic
synchronized (eventListeners) {
for (List<LensEventListener> listeners : eventListeners.values()) {
if (listeners.remove(listener)) {
- LOG.info("Removed listener " + listener);
+ log.info("Removed listener {}", listener);
}
}
}
@@ -101,7 +99,7 @@ public class EventServiceImpl extends AbstractService implements LensEventServic
try {
listener.onEvent(evt);
} catch (Exception exc) {
- LOG.error("Error in handling event" + evt.getEventId() + " for listener " + listener, exc);
+ log.error("Error in handling event {} for listener {}", evt.getEventId(), listener, exc);
}
}
}
@@ -199,11 +197,11 @@ public class EventServiceImpl extends AbstractService implements LensEventServic
pendingMsg.append(((EventHandler) handler).event.getEventId()).append(",");
}
}
- LOG.info("Event listener service stopped while " + pending.size() + " events still pending");
- LOG.info(pendingMsg.toString());
+ log.info("Event listener service stopped while {} events still pending", pending.size());
+ log.info(pendingMsg.toString());
}
}
- LOG.info("Event service stopped");
+ log.info("Event service stopped");
super.stop();
}
@@ -227,7 +225,7 @@ public class EventServiceImpl extends AbstractService implements LensEventServic
}
listeners.add(listener);
}
- LOG.info("Added listener " + listener + " for type:" + eventType.getName());
+ log.info("Added listener {} for type:{}", listener, eventType.getName());
}
/*
@@ -242,7 +240,7 @@ public class EventServiceImpl extends AbstractService implements LensEventServic
List<LensEventListener> listeners = eventListeners.get(eventType);
if (listeners != null) {
if (listeners.remove(listener)) {
- LOG.info("Removed listener " + listener);
+ log.info("Removed listener {}", listener);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
index a3079cc..a446ab6 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensApplication.java
@@ -26,19 +26,17 @@ import javax.ws.rs.core.Application;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class LensApplication.
*/
@ApplicationPath("/")
+@Slf4j
public class LensApplication extends Application {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensApplication.class);
-
/** The conf. */
public static final Configuration CONF = LensServerConf.getConf();
@@ -56,25 +54,25 @@ public class LensApplication extends Application {
for (String rName : resourceNames) {
Class wsResourceClass = CONF.getClass(LensConfConstants.getWSResourceImplConfKey(rName), null);
classes.add(wsResourceClass);
- LOG.info("Added resource " + wsResourceClass);
+ log.info("Added resource {}", wsResourceClass);
}
for (String fName : featureNames) {
Class wsFeatureClass = CONF.getClass(LensConfConstants.getWSFeatureImplConfKey(fName), null);
classes.add(wsFeatureClass);
- LOG.info("Added feature " + wsFeatureClass);
+ log.info("Added feature {}", wsFeatureClass);
}
for (String lName : listenerNames) {
Class wsListenerClass = CONF.getClass(LensConfConstants.getWSListenerImplConfKey(lName), null);
classes.add(wsListenerClass);
- LOG.info("Added listener " + wsListenerClass);
+ log.info("Added listener {}", wsListenerClass);
}
for (String filterName : filterNames) {
Class wsFilterClass = CONF.getClass(LensConfConstants.getWSFilterImplConfKey(filterName), null);
classes.add(wsFilterClass);
- LOG.info("Added filter " + wsFilterClass);
+ log.info("Added filter {}", wsFilterClass);
}
- LOG.debug("LensRequestContextInitFilter added...");
+ log.debug("LensRequestContextInitFilter added...");
classes.add(LensRequestContextInitFilter.class);
return classes;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java b/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
index 2c8ce46..2b3f7e3 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensApplicationListener.java
@@ -20,13 +20,13 @@ package org.apache.lens.server;
import org.apache.lens.server.api.metrics.MetricsService;
-import org.apache.log4j.Logger;
-
import org.glassfish.jersey.server.monitoring.ApplicationEvent;
import org.glassfish.jersey.server.monitoring.ApplicationEventListener;
import org.glassfish.jersey.server.monitoring.RequestEvent;
import org.glassfish.jersey.server.monitoring.RequestEventListener;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The listener interface for receiving lensApplication events. The class that is interested in processing a
* lensApplication event implements this interface, and the object created with that class is registered with a
@@ -34,11 +34,9 @@ import org.glassfish.jersey.server.monitoring.RequestEventListener;
* When the lensApplication event occurs, that object's appropriate method is invoked.
*
*/
+@Slf4j
public class LensApplicationListener implements ApplicationEventListener {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(LensApplicationListener.class);
-
/** The req listener. */
/*
@@ -71,10 +69,10 @@ public class LensApplicationListener implements ApplicationEventListener {
public void onEvent(ApplicationEvent event) {
switch (event.getType()) {
case INITIALIZATION_FINISHED:
- LOG.info("Application " + event.getResourceConfig().getApplicationName() + " was initialized.");
+ log.info("Application {} was initialized.", event.getResourceConfig().getApplicationName());
break;
case DESTROY_FINISHED:
- LOG.info("Application " + event.getResourceConfig().getApplicationName() + " was destroyed");
+ log.info("Application {} was destroyed", event.getResourceConfig().getApplicationName());
break;
default:
break;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/LensServer.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
index d4c89c0..0da37ad 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
@@ -33,8 +33,6 @@ import org.apache.lens.server.metrics.MetricsServiceImpl;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
import org.apache.lens.server.ui.UIApp;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.glassfish.grizzly.http.server.HttpServer;
@@ -46,16 +44,16 @@ import org.glassfish.jersey.server.ResourceConfig;
import org.slf4j.bridge.SLF4JBridgeHandler;
import com.codahale.metrics.servlets.AdminServlet;
+
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class LensServer.
*/
+@Slf4j
public class LensServer {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensServer.class);
-
private static final String SEP_LINE =
"\n###############################################################\n";
@@ -169,11 +167,11 @@ public class LensServer {
try {
wait(2000);
} catch (InterruptedException e) {
- LOG.warn("Received an interrupt in the main loop", e);
+ log.warn("Received an interrupt in the main loop", e);
}
}
}
- LOG.info("Exiting main run loop...");
+ log.info("Exiting main run loop...");
}
/**
@@ -197,11 +195,11 @@ public class LensServer {
thisServer.start();
thisServer.join();
} catch (Exception exc) {
- LOG.fatal("Error while creating Lens server", exc);
+ log.error("Error while creating Lens server", exc);
try {
LensServices.get().stop();
} catch (Exception e) {
- LOG.error("Error stopping services", e);
+ log.error("Error stopping services", e);
}
}
}
@@ -230,7 +228,7 @@ public class LensServer {
buffer.append("*** Unable to get build info ***");
}
buffer.append(SEP_LINE);
- LOG.info(buffer.toString());
+ log.info(buffer.toString());
}
/**
@@ -241,7 +239,7 @@ public class LensServer {
buffer.append(SEP_LINE);
buffer.append(" Lens Server (SHUTDOWN)");
buffer.append(SEP_LINE);
- LOG.info(buffer.toString());
+ log.info(buffer.toString());
}
/**
@@ -253,7 +251,7 @@ public class LensServer {
@Override
public void run() {
Thread.currentThread().setName("Shutdown");
- LOG.info("Server has been requested to be stopped.");
+ log.info("Server has been requested to be stopped.");
thisServer.canRun = false;
thisServer.stop();
}
@@ -265,7 +263,7 @@ public class LensServer {
Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
@Override
public void uncaughtException(Thread t, Throwable e) {
- LOG.fatal("Uncaught exception in Thread " + t, e);
+ log.error("Uncaught exception in Thread " + t, e);
}
});
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/LensService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensService.java b/lens-server/src/main/java/org/apache/lens/server/LensService.java
index ae9af6a..0e564a9 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensService.java
@@ -42,8 +42,6 @@ import org.apache.lens.server.user.UserConfigLoaderFactory;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -59,14 +57,14 @@ import org.apache.hive.service.cli.SessionHandle;
import org.apache.hive.service.cli.session.SessionManager;
import org.apache.hive.service.cli.thrift.TSessionHandle;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class LensService.
*/
+@Slf4j
public abstract class LensService extends CompositeService implements Externalizable {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensService.class);
-
/** The cli service. */
private final CLIService cliService;
@@ -129,11 +127,11 @@ public abstract class LensService extends CompositeService implements Externaliz
sessionConf.putAll(configuration);
}
Map<String, String> userConfig = UserConfigLoaderFactory.getUserConfig(username);
- LOG.info("Got user config: " + userConfig);
+ log.info("Got user config: {}", userConfig);
UtilityMethods.mergeMaps(sessionConf, userConfig, false);
sessionConf.put(LensConfConstants.SESSION_LOGGEDIN_USER, username);
if (sessionConf.get(LensConfConstants.SESSION_CLUSTER_USER) == null) {
- LOG.info("Didn't get cluster user from user config loader. Setting same as logged in user: " + username);
+ log.info("Didn't get cluster user from user config loader. Setting same as logged in user: {}", username);
sessionConf.put(LensConfConstants.SESSION_CLUSTER_USER, username);
}
String clusterUser = sessionConf.get(LensConfConstants.SESSION_CLUSTER_USER);
@@ -217,7 +215,7 @@ public abstract class LensService extends CompositeService implements Externaliz
cliService.getHiveConf());
provider.Authenticate(userName, password);
} catch (Exception e) {
- LOG.error("Auth error: " + e);
+ log.error("Auth error: ", e);
throw new NotAuthorizedException(e);
}
}
@@ -256,7 +254,7 @@ public abstract class LensService extends CompositeService implements Externaliz
try {
return ((LensSessionImpl) getSessionManager().getSession(getHiveSessionHandle(sessionHandle)));
} catch (HiveSQLException exc) {
- LOG.warn("Session " + sessionHandle.getPublicId() + " not found", exc);
+ log.warn("Session {} not found", sessionHandle.getPublicId(), exc);
// throw resource gone exception (410)
throw new ClientErrorException("Session " + sessionHandle.getPublicId() + " is invalid " + sessionHandle, 410);
}
@@ -269,7 +267,7 @@ public abstract class LensService extends CompositeService implements Externaliz
*/
public void acquire(LensSessionHandle sessionHandle) {
if (sessionHandle != null) {
- LOG.debug("Acquiring lens session:" + sessionHandle.getPublicId());
+ log.debug("Acquiring lens session:{}", sessionHandle.getPublicId());
getSession(sessionHandle).acquire();
}
}
@@ -297,7 +295,7 @@ public abstract class LensService extends CompositeService implements Externaliz
public void release(LensSessionHandle sessionHandle) {
if (sessionHandle != null) {
getSession(sessionHandle).release();
- LOG.debug("Released lens session:" + sessionHandle.getPublicId());
+ log.debug("Released lens session:{}", sessionHandle.getPublicId());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index fe58e96..1bcc5944 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -40,8 +40,6 @@ import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.user.UserConfigLoaderFactory;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -53,16 +51,14 @@ import org.apache.hive.service.cli.CLIService;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
-
+import lombok.extern.slf4j.Slf4j;
/**
* Manage lifecycle of all Lens services
*/
+@Slf4j
public class LensServices extends CompositeService implements ServiceProvider {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensServices.class);
-
/** The Constant LENS_SERVICES_NAME. */
public static final String LENS_SERVICES_NAME = "lens_services";
@@ -207,7 +203,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
String serviceClassName = conf.get(getServiceImplConfKey(sName));
if (StringUtils.isBlank(serviceClassName)) {
- LOG.warn("Invalid class for service " + sName + " class=" + serviceClassName);
+ log.warn("Invalid class for service {} class={}", sName, serviceClassName);
continue;
}
@@ -215,7 +211,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
if (LensService.class.isAssignableFrom(cls)) {
Class<? extends LensService> serviceClass = (Class<? extends LensService>) cls;
- LOG.info("Adding " + sName + " service with " + serviceClass);
+ log.info("Adding {} service with {}", sName, serviceClass);
Constructor<?> constructor = serviceClass.getConstructor(CLIService.class);
LensService service = (LensService) constructor.newInstance(new Object[]{cliService});
addService(service);
@@ -226,10 +222,10 @@ public class LensServices extends CompositeService implements ServiceProvider {
Service svc = serviceClass.newInstance();
addService(svc);
} else {
- LOG.warn("Unsupported service class " + serviceClassName + " for service " + sName);
+ log.warn("Unsupported service class {} for service {}", serviceClassName, sName);
}
} catch (Exception e) {
- LOG.warn("Could not add service:" + sName, e);
+ log.warn("Could not add service:{}", sName, e);
throw new RuntimeException("Could not add service:" + sName, e);
}
}
@@ -252,16 +248,16 @@ public class LensServices extends CompositeService implements ServiceProvider {
int outStreamBufferSize = conf.getInt(STATE_PERSIST_OUT_STREAM_BUFF_SIZE,
DEFAULT_STATE_PERSIST_OUT_STREAM_BUFF_SIZE);
configuration.setInt(FS_IO_FILE_BUFFER_SIZE, outStreamBufferSize);
- LOG.info("STATE_PERSIST_OUT_STREAM_BUFF_SIZE IN BYTES:" + outStreamBufferSize);
+ log.info("STATE_PERSIST_OUT_STREAM_BUFF_SIZE IN BYTES:{}", outStreamBufferSize);
persistenceFS = FileSystem.newInstance(persistDir.toUri(), configuration);
setupPersistedState();
} catch (Exception e) {
- LOG.error("Could not recover from persisted state", e);
+ log.error("Could not recover from persisted state", e);
throw new RuntimeException("Could not recover from persisted state", e);
}
snapShotInterval = conf.getLong(SERVER_SNAPSHOT_INTERVAL,
DEFAULT_SERVER_SNAPSHOT_INTERVAL);
- LOG.info("Initialized services: " + services.keySet().toString());
+ log.info("Initialized services: {}", services.keySet().toString());
timer = new Timer("lens-server-snapshotter", true);
}
}
@@ -282,10 +278,10 @@ public class LensServices extends CompositeService implements ServiceProvider {
final String runId = UUID.randomUUID().toString();
logSegregationContext.setLogSegregationId(runId);
persistLensServiceState();
- LOG.info("SnapShot of Lens Services created");
+ log.info("SnapShot of Lens Services created");
} catch (IOException e) {
incrCounter(SERVER_STATE_PERSISTENCE_ERRORS);
- LOG.warn("Unable to persist lens server state", e);
+ log.warn("Unable to persist lens server state", e);
}
}
}, snapShotInterval, snapShotInterval);
@@ -307,11 +303,11 @@ public class LensServices extends CompositeService implements ServiceProvider {
try {
in = new ObjectInputStream(persistenceFS.open(getServicePersistPath(service)));
} catch (FileNotFoundException fe) {
- LOG.warn("No persist path available for service:" + service.getName());
+ log.warn("No persist path available for service:{}", service.getName());
continue;
}
service.readExternal(in);
- LOG.info("Recovered service " + service.getName() + " from persisted state");
+ log.info("Recovered service {} from persisted state", service.getName());
} finally {
if (in != null) {
in.close();
@@ -331,11 +327,11 @@ public class LensServices extends CompositeService implements ServiceProvider {
synchronized (statePersistenceLock) {
if (conf.getBoolean(SERVER_RESTART_ENABLED, DEFAULT_SERVER_RESTART_ENABLED)) {
if (persistDir != null) {
- LOG.info("Persisting server state in " + persistDir);
+ log.info("Persisting server state in {}", persistDir);
long now = System.currentTimeMillis();
for (LensService service : lensServices) {
- LOG.info("Persisting state of service:" + service.getName());
+ log.info("Persisting state of service: {}", service.getName());
Path serviceWritePath = new Path(persistDir, service.getName() + ".out" + "." + now);
ObjectOutputStream out = null;
try {
@@ -350,18 +346,18 @@ public class LensServices extends CompositeService implements ServiceProvider {
if (persistenceFS.exists(servicePath)) {
// delete the destination first, because rename is no-op in HDFS, if destination exists
if (!persistenceFS.delete(servicePath, true)) {
- LOG.error("Failed to delete [" + servicePath + "]");
+ log.error("Failed to delete [{}]", servicePath);
}
}
if (!persistenceFS.rename(serviceWritePath, servicePath)) {
incrCounter(SERVER_STATE_PERSISTENCE_ERRORS);
- LOG.error("Failed to persist " + service.getName() + " to [" + servicePath + "]");
+ log.error("Failed to persist {} to [{}]", service.getName(), servicePath);
} else {
- LOG.info("Persisted service " + service.getName() + " to [" + servicePath + "]");
+ log.info("Persisted service {} to [{}]", service.getName(), servicePath);
}
}
} else {
- LOG.info("Server restart is not enabled. Not persisting the server state");
+ log.info("Server restart is not enabled. Not persisting the server state");
}
}
}
@@ -384,7 +380,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
*/
public synchronized void stop() {
if (getServiceState() != STATE.STOPPED) {
- LOG.info("Stopping lens server");
+ log.info("Stopping lens server");
stopping = true;
for (LensService service : lensServices) {
service.prepareStopping();
@@ -399,10 +395,10 @@ public class LensServices extends CompositeService implements ServiceProvider {
persistLensServiceState();
persistenceFS.close();
- LOG.info("Persistence File system object close complete");
+ log.info("Persistence File system object close complete");
} catch (IOException e) {
incrCounter(SERVER_STATE_PERSISTENCE_ERRORS);
- LOG.error("Could not persist server state", e);
+ log.error("Could not persist server state", e);
throw new IllegalStateException(e);
} finally {
super.stop();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index cb9516b..d6ef04d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -33,13 +33,13 @@ import org.apache.lens.server.api.metastore.CubeMetastoreService;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.Logger;
import org.glassfish.jersey.media.multipart.FormDataParam;
import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
+
/**
* metastore resource api
* <p> </p>
@@ -47,8 +47,8 @@ import com.google.common.collect.Lists;
*/
@Path("metastore")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+@Slf4j
public class MetastoreResource {
- public static final Logger LOG = LogManager.getLogger(MetastoreResource.class);
public static final APIResult SUCCESS = new APIResult(Status.SUCCEEDED, "");
public static final ObjectFactory X_CUBE_OBJECT_FACTORY = new ObjectFactory();
@@ -111,7 +111,7 @@ public class MetastoreResource {
@Path("databases/current")
public String getDatabase(@QueryParam("sessionid") LensSessionHandle sessionid) {
checkSessionId(sessionid);
- LOG.info("Get database");
+ log.info("Get database");
try {
return getSvc().getCurrentDatabase(sessionid);
} catch (LensException e) {
@@ -132,11 +132,11 @@ public class MetastoreResource {
@Consumes({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON})
public APIResult setDatabase(@QueryParam("sessionid") LensSessionHandle sessionid, String dbName) {
checkSessionId(sessionid);
- LOG.info("Set database:" + dbName);
+ log.info("Set database:{}", dbName);
try {
getSvc().setCurrentDatabase(sessionid, dbName);
} catch (LensException e) {
- LOG.error("Error changing current database", e);
+ log.error("Error changing current database", e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -157,11 +157,11 @@ public class MetastoreResource {
@PathParam("dbName") String dbName,
@QueryParam("cascade") boolean cascade) {
checkSessionId(sessionid);
- LOG.info("Drop database " + dbName + " cascade?" + cascade);
+ log.info("Drop database {} cascade?{}", dbName, cascade);
try {
getSvc().dropDatabase(sessionid, dbName, cascade);
} catch (LensException e) {
- LOG.error("Error dropping " + dbName, e);
+ log.error("Error dropping {}", dbName, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -182,12 +182,12 @@ public class MetastoreResource {
@QueryParam("ignoreIfExisting") @DefaultValue("true") boolean ignoreIfExisting,
String dbName) {
checkSessionId(sessionid);
- LOG.info("Create database " + dbName + " Ignore Existing? " + ignoreIfExisting);
+ log.info("Create database {} Ignore Existing? {}", dbName, ignoreIfExisting);
try {
getSvc().createDatabase(sessionid, dbName, ignoreIfExisting);
} catch (LensException e) {
- LOG.error("Error creating database " + dbName, e);
+ log.error("Error creating database {}", dbName, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -242,7 +242,7 @@ public class MetastoreResource {
return X_CUBE_OBJECT_FACTORY.createXNativeTable(getSvc().getNativeTable(sessionid, tableName));
} catch (LensException e) {
checkTableNotFound(e, tableName);
- LOG.error("Error getting native table", e);
+ log.error("Error getting native table", e);
throw new WebApplicationException(e);
}
}
@@ -273,7 +273,7 @@ public class MetastoreResource {
+ " values are 'all' or 'base' or 'derived' or 'queryable'");
}
} catch (LensException e) {
- LOG.error("Error getting cube names", e);
+ log.error("Error getting cube names", e);
throw new WebApplicationException(e);
}
}
@@ -299,7 +299,7 @@ public class MetastoreResource {
numDeleted++;
}
} catch (LensException e) {
- LOG.error("Error deleting cubes:", e);
+ log.error("Error deleting cubes:", e);
failed = true;
}
if (cubeNames != null && numDeleted == cubeNames.size()) {
@@ -336,7 +336,7 @@ public class MetastoreResource {
// parent should exist
checkTableNotFound(e, ((XDerivedCube) cube).getParent());
}
- LOG.error("Error creating cube " + cube.getName(), e);
+ log.error("Error creating cube {}", cube.getName(), e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -378,7 +378,7 @@ public class MetastoreResource {
checkTableNotFound(e, ((XDerivedCube) cube).getParent());
}
checkTableNotFound(e, cube.getName());
- LOG.error("Error updating cube " + cube.getName(), e);
+ log.error("Error updating cube {}", cube.getName(), e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -400,7 +400,7 @@ public class MetastoreResource {
return X_CUBE_OBJECT_FACTORY.createXCube(getSvc().getCube(sessionid, cubeName));
} catch (LensException e) {
checkTableNotFound(e, cubeName);
- LOG.error("Error getting cube", e);
+ log.error("Error getting cube", e);
throw new WebApplicationException(e);
}
}
@@ -422,7 +422,7 @@ public class MetastoreResource {
getSvc().dropCube(sessionid, cubeName);
} catch (LensException e) {
checkTableNotFound(e, cubeName);
- LOG.error("Error droping cube " + cubeName, e);
+ log.error("Error droping cube {}", cubeName, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -442,7 +442,7 @@ public class MetastoreResource {
try {
return new StringList(getSvc().getAllStorageNames(sessionid));
} catch (LensException e) {
- LOG.error("Error getting storages", e);
+ log.error("Error getting storages", e);
throw new WebApplicationException(e);
}
}
@@ -462,7 +462,7 @@ public class MetastoreResource {
try {
getSvc().createStorage(sessionid, storage);
} catch (LensException e) {
- LOG.error("Error creating storage " + storage.getName(), e);
+ log.error("Error creating storage {}", storage.getName(), e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -489,7 +489,7 @@ public class MetastoreResource {
numDeleted++;
}
} catch (LensException e) {
- LOG.error("Error deleting storages:", e);
+ log.error("Error deleting storages:", e);
failed = true;
}
if (storageNames != null && numDeleted == storageNames.size()) {
@@ -525,7 +525,7 @@ public class MetastoreResource {
getSvc().alterStorage(sessionid, storageName, storage);
} catch (LensException e) {
checkTableNotFound(e, storageName);
- LOG.error("Error updating storage" + storageName, e);
+ log.error("Error updating storage {}", storageName, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -568,7 +568,7 @@ public class MetastoreResource {
getSvc().dropStorage(sessionid, storageName);
} catch (LensException e) {
checkTableNotFound(e, storageName);
- LOG.error("Error dropping storage" + storageName, e);
+ log.error("Error dropping storage {}", storageName, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -588,7 +588,7 @@ public class MetastoreResource {
try {
return new StringList(getSvc().getAllDimensionNames(sessionid));
} catch (LensException e) {
- LOG.error("Error getting dimensions", e);
+ log.error("Error getting dimensions", e);
throw new WebApplicationException(e);
}
}
@@ -608,7 +608,7 @@ public class MetastoreResource {
try {
getSvc().createDimension(sessionid, dimension);
} catch (LensException e) {
- LOG.error("Error creating dimension " + dimension.getName(), e);
+ log.error("Error creating dimension {}", dimension.getName(), e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -635,7 +635,7 @@ public class MetastoreResource {
numDeleted++;
}
} catch (LensException e) {
- LOG.error("Error deleting dimensions:", e);
+ log.error("Error deleting dimensions:", e);
failed = true;
}
if (dimNames != null && numDeleted == dimNames.size()) {
@@ -671,7 +671,7 @@ public class MetastoreResource {
getSvc().updateDimension(sessionid, dimName, dimension);
} catch (LensException e) {
checkTableNotFound(e, dimName);
- LOG.error("Error updating dimension" + dimName, e);
+ log.error("Error updating dimension {}", dimName, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -714,7 +714,7 @@ public class MetastoreResource {
getSvc().dropDimension(sessionid, dimName);
} catch (LensException e) {
checkTableNotFound(e, dimName);
- LOG.error("Error dropping dimName" + dimName, e);
+ log.error("Error dropping dimension {}", dimName, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -799,7 +799,7 @@ public class MetastoreResource {
numDeleted++;
}
} catch (LensException e) {
- LOG.error("Error deleting cubes:", e);
+ log.error("Error deleting cubes:", e);
failed = true;
}
if (factNames != null && numDeleted == factNames.size()) {
@@ -854,10 +854,10 @@ public class MetastoreResource {
throws LensException {
checkSessionId(sessionid);
try {
- LOG.info("Create fact table");
+ log.info("Create fact table");
getSvc().createFactTable(sessionid, fact);
} catch (LensException exc) {
- LOG.error("Exception creating fact:", exc);
+ log.error("Exception creating fact:", exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -882,7 +882,7 @@ public class MetastoreResource {
getSvc().updateFactTable(sessionid, fact);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error updating fact" + factName, exc);
+ log.error("Error updating fact {}", factName, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -908,7 +908,7 @@ public class MetastoreResource {
getSvc().dropFactTable(sessionid, factName, cascade);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error dropping fact" + factName, exc);
+ log.error("Error dropping fact {}", factName, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -952,7 +952,7 @@ public class MetastoreResource {
getSvc().dropAllStoragesOfFact(sessionid, factName);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error dropping storages of fact" + factName, exc);
+ log.error("Error dropping storages of fact {}", factName, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -977,7 +977,7 @@ public class MetastoreResource {
getSvc().addStorageToFact(sessionid, factName, storageTable);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error adding storage to fact" + factName, exc);
+ log.error("Error adding storage to fact {}", factName, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1003,7 +1003,7 @@ public class MetastoreResource {
getSvc().dropStorageOfFact(sessionid, factName, storage);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error dropping storage of fact" + factName, exc);
+ log.error("Error dropping storage of fact {}", factName, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1074,7 +1074,7 @@ public class MetastoreResource {
try {
getSvc().dropPartitionFromStorageByFilter(sessionid, factName, storage, filter);
} catch (LensException exc) {
- LOG.warn("Got exception while dropping partition.", exc);
+ log.warn("Got exception while dropping partition.", exc);
checkTableNotFound(exc, factName);
return new APIResult(Status.PARTIAL, exc.getMessage());
}
@@ -1103,7 +1103,7 @@ public class MetastoreResource {
getSvc().addPartitionToFactStorage(sessionid, factName, storage, partition);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error adding partition to storage of fact" + factName + ":" + storage, exc);
+ log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1131,7 +1131,7 @@ public class MetastoreResource {
getSvc().updatePartition(sessionid, factName, storage, partition);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error adding partition to storage of fact" + factName + ":" + storage, exc);
+ log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1159,7 +1159,7 @@ public class MetastoreResource {
getSvc().addPartitionsToFactStorage(sessionid, factName, storage, partitions);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error adding partition to storage of fact" + factName + ":" + storage, exc);
+ log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1187,7 +1187,7 @@ public class MetastoreResource {
getSvc().updatePartitions(sessionid, factName, storage, partitions);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error adding partition to storage of fact" + factName + ":" + storage, exc);
+ log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1216,7 +1216,7 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, factName);
- LOG.error("Error dropping partition to storage of fact" + factName + ":" + storage, e);
+ log.error("Error dropping partition to storage of fact {}:{}", factName, storage, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -1250,7 +1250,7 @@ public class MetastoreResource {
try {
getSvc().createDimensionTable(sessionid, dimensionTable);
} catch (LensException exc) {
- LOG.error("Error creating cube dimension table " + dimensionTable.getTableName(), exc);
+ log.error("Error creating cube dimension table {}", dimensionTable.getTableName(), exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1274,7 +1274,7 @@ public class MetastoreResource {
getSvc().updateDimensionTable(sessionid, dimensionTable);
} catch (LensException exc) {
checkTableNotFound(exc, dimTableName);
- LOG.error("Error updating cube dimension table " + dimTableName, exc);
+ log.error("Error updating cube dimension table {}", dimTableName, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1299,7 +1299,7 @@ public class MetastoreResource {
getSvc().dropDimensionTable(sessionid, dimension, cascade);
} catch (LensException e) {
checkTableNotFound(e, dimension);
- LOG.error("Error dropping cube dimension table " + dimension, e);
+ log.error("Error dropping cube dimension table {}", dimension, e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -1362,7 +1362,7 @@ public class MetastoreResource {
getSvc().addDimTableStorage(sessionid, dimTableName, storageTbl);
} catch (LensException e) {
checkTableNotFound(e, dimTableName);
- LOG.error("Error creating dimension table storage " + dimTableName + ":" + storageTbl.getStorageName(), e);
+ log.error("Error creating dimension table storage {}:{}", dimTableName, storageTbl.getStorageName(), e);
return new APIResult(Status.FAILED, e.getMessage());
}
return SUCCESS;
@@ -1402,7 +1402,7 @@ public class MetastoreResource {
getSvc().dropAllStoragesOfDimTable(sessionid, dimTableName);
} catch (LensException exc) {
checkTableNotFound(exc, dimTableName);
- LOG.error("Error dropping storages of dimension table " + dimTableName, exc);
+ log.error("Error dropping storages of dimension table {}", dimTableName, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1427,7 +1427,7 @@ public class MetastoreResource {
getSvc().dropStorageOfDimTable(sessionid, dimTableName, storage);
} catch (LensException exc) {
checkTableNotFound(exc, dimTableName);
- LOG.error("Error dropping storage of dimension table " + dimTableName + ":" + storage, exc);
+ log.error("Error dropping storage of dimension table {}:{}", dimTableName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1478,7 +1478,7 @@ public class MetastoreResource {
try {
getSvc().dropPartitionFromStorageByFilter(sessionid, dimTableName, storage, filter);
} catch (LensException exc) {
- LOG.error("Error dropping partition on storage of dimension table " + dimTableName + ":" + storage, exc);
+ log.error("Error dropping partition on storage of dimension table {}:{}", dimTableName, storage, exc);
return new APIResult(Status.PARTIAL, exc.getMessage());
}
return SUCCESS;
@@ -1505,7 +1505,7 @@ public class MetastoreResource {
getSvc().dropPartitionFromStorageByValues(sessionid, dimTableName, storage,
values);
} catch (LensException exc) {
- LOG.error("Error dropping partitions on storage of dimension table " + dimTableName + ":" + storage, exc);
+ log.error("Error dropping partitions on storage of dimension table {}:{}", dimTableName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1532,7 +1532,7 @@ public class MetastoreResource {
try {
getSvc().addPartitionToDimStorage(sessionid, dimTableName, storage, partition);
} catch (LensException exc) {
- LOG.error("Error adding partition to storage of dimension table " + dimTableName + ":" + storage, exc);
+ log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1559,7 +1559,7 @@ public class MetastoreResource {
try {
getSvc().updatePartition(sessionid, dimTableName, storage, partition);
} catch (LensException exc) {
- LOG.error("Error adding partition to storage of dimension table " + dimTableName + ":" + storage, exc);
+ log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1586,7 +1586,7 @@ public class MetastoreResource {
try {
getSvc().addPartitionsToDimStorage(sessionid, dimTableName, storage, partitions);
} catch (LensException exc) {
- LOG.error("Error adding partition to storage of dimension table " + dimTableName + ":" + storage, exc);
+ log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1613,7 +1613,7 @@ public class MetastoreResource {
try {
getSvc().updatePartitions(sessionid, dimTableName, storage, partitions);
} catch (LensException exc) {
- LOG.error("Error adding partition to storage of dimension table " + dimTableName + ":" + storage, exc);
+ log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
return new APIResult(Status.FAILED, exc.getMessage());
}
return SUCCESS;
@@ -1705,7 +1705,7 @@ public class MetastoreResource {
updatePeriod, timeDimension));
} catch (LensException exc) {
checkTableNotFound(exc, factName);
- LOG.error("Error finding partition timelines for fact: " + factName);
+ log.error("Error finding partition timelines for fact: {}", factName, exc);
throw exc;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
index 757205a..e004318 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
@@ -40,24 +40,21 @@ import org.apache.lens.server.api.query.QueryEnded;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import lombok.Data;
import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class QueryEndNotifier.
*/
+@Slf4j
public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
/** The query service. */
private final QueryExecutionServiceImpl queryService;
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(QueryEndNotifier.class);
-
/** The Constant EMAIL_ERROR_COUNTER. */
public static final String EMAIL_ERROR_COUNTER = "email-send-errors";
@@ -113,8 +110,8 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
}
QueryContext queryContext = queryService.getQueryContext(event.getQueryHandle());
if (queryContext == null) {
- LOG.warn("Could not find the context for " + event.getQueryHandle() + " for event:" + event.getCurrentValue()
- + ". No email generated");
+ log.warn("Could not find the context for {} for event:{}. No email generated", event.getQueryHandle(),
+ event.getCurrentValue());
return;
}
this.logSegregationContext.setLogSegragationAndQueryId(queryContext.getQueryHandleString());
@@ -138,7 +135,7 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
String cc = queryContext.getConf().get(LensConfConstants.QUERY_RESULT_EMAIL_CC,
LensConfConstants.QUERY_RESULT_DEFAULT_EMAIL_CC);
- LOG.info("Sending completion email for query handle: " + event.getQueryHandle());
+ log.info("Sending completion email for query handle: {}", event.getQueryHandle());
sendMail(host, port, new Email(from, to, cc, mailSubject, mailMessage), mailSmtpTimeout, mailSmtpConnectionTimeout);
}
@@ -225,7 +222,7 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
} catch (Exception e) {
MetricsService metricsService = (MetricsService) LensServices.get().getService(MetricsService.NAME);
metricsService.incrCounter(QueryEndNotifier.class, EMAIL_ERROR_COUNTER);
- LOG.error("Error sending query end email", e);
+ log.error("Error sending query end email", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 9e5fcd5..5e0f24a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -72,8 +72,6 @@ import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -86,6 +84,8 @@ import org.apache.hive.service.cli.TypeDescriptor;
import org.codehaus.jackson.*;
import org.codehaus.jackson.map.*;
import org.codehaus.jackson.map.module.SimpleModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -412,13 +412,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
/**
* The Class QueryStatusLogger.
*/
- @Slf4j
public static class QueryStatusLogger implements LensEventListener<StatusChange> {
/**
* The Constant STATUS_LOG.
*/
- public static final Log STATUS_LOG = LogFactory.getLog(QueryStatusLogger.class);
+ public static final Logger STATUS_LOG = LoggerFactory.getLogger(QueryStatusLogger.class);
/*
* (non-Javadoc)
@@ -1040,7 +1039,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
loadDriversAndSelector();
} catch (LensException e) {
- LOG.error(e);
+ log.error("Error while loading drivers", e);
throw new IllegalStateException("Could not load drivers", e);
}
maxFinishedQueries = conf.getInt(MAX_NUMBER_OF_FINISHED_QUERY,
@@ -1180,7 +1179,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
};
- LOG.debug("starting estimate pool");
+ log.debug("starting estimate pool");
ThreadPoolExecutor estimatePool = new ThreadPoolExecutor(minPoolSize, maxPoolSize, keepAlive, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(), threadFactory);
estimatePool.allowCoreThreadTimeOut(true);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index 877e296..2295248 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -30,22 +30,19 @@ import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
import org.apache.lens.server.model.LogSegregationContext;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.ReflectionUtils;
import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class ResultFormatter.
*/
+@Slf4j
public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(ResultFormatter.class);
-
/** The query service. */
QueryExecutionServiceImpl queryService;
@@ -82,11 +79,11 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
this.logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString());
try {
if (!ctx.isPersistent()) {
- LOG.info("No result formatting required for query " + queryHandle);
+ log.info("No result formatting required for query " + queryHandle);
return;
}
if (ctx.isResultAvailableInDriver()) {
- LOG.info("Result formatter for " + queryHandle);
+ log.info("Result formatter for {}", queryHandle);
LensResultSet resultSet = queryService.getDriverResultset(queryHandle);
boolean isPersistedInDriver = resultSet instanceof PersistentResultSet;
if (isPersistedInDriver) {
@@ -96,10 +93,10 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
long size = fs.getContentSummary(persistedDirectory).getLength();
long threshold = ctx.getConf().getLong(LensConfConstants.RESULT_FORMAT_SIZE_THRESHOLD,
LensConfConstants.DEFAULT_RESULT_FORMAT_SIZE_THRESHOLD);
- LOG.info(" size :" + size + " threshold:" + threshold);
+ log.info(" size :{} threshold:{}", size, threshold);
if (size > threshold) {
- LOG.warn("Persisted result size more than the threshold, size:" + size + " and threshold:" + threshold
- + "; Skipping formatter");
+ log.warn("Persisted result size more than the threshold, size:{} and threshold:{}; Skipping formatter",
+ size, threshold);
queryService.setSuccessState(ctx);
return;
}
@@ -114,12 +111,12 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
formatter.writeHeader();
}
if (isPersistedInDriver) {
- LOG.info("Result formatter for " + queryHandle + " in persistent result");
+ log.info("Result formatter for {} in persistent result", queryHandle);
Path persistedDirectory = new Path(ctx.getHdfsoutPath());
// write all files from persistent directory
((PersistedOutputFormatter) formatter).addRowsFromPersistedPath(persistedDirectory);
} else {
- LOG.info("Result formatter for " + queryHandle + " in inmemory result");
+ log.info("Result formatter for {} in inmemory result", queryHandle);
InMemoryResultSet inmemory = (InMemoryResultSet) resultSet;
while (inmemory.hasNext()) {
((InMemoryOutputFormatter) formatter).writeRow(inmemory.next());
@@ -134,16 +131,16 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
formatter.close();
}
queryService.setSuccessState(ctx);
- LOG.info("Result formatter has completed. Final path:" + formatter.getFinalOutputPath());
+ log.info("Result formatter has completed. Final path:{}", formatter.getFinalOutputPath());
}
} catch (Exception e) {
MetricsService metricsService = (MetricsService) LensServices.get().getService(MetricsService.NAME);
metricsService.incrCounter(ResultFormatter.class, "formatting-errors");
- LOG.warn("Exception while formatting result for " + queryHandle, e);
+ log.warn("Exception while formatting result for {}", queryHandle, e);
try {
queryService.setFailedStatus(ctx, "Result formatting failed!", e.getMessage(), null);
} catch (LensException e1) {
- LOG.error("Exception while setting failure for " + queryHandle, e1);
+ log.error("Exception while setting failure for {}", queryHandle, e1);
}
}
}
@@ -178,7 +175,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
} catch (ClassNotFoundException e) {
throw new LensException(e);
}
- LOG.info("Created result formatter:" + formatter.getClass().getCanonicalName());
+ log.info("Created result formatter:{}", formatter.getClass().getCanonicalName());
ctx.setQueryOutputFormatter(formatter);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
index 447d638..52edffc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
@@ -41,18 +41,18 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.log4j.Logger;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class RewriteUtil.
*/
+@Slf4j
public final class RewriteUtil {
private RewriteUtil() {
}
- public static final Logger LOG = Logger.getLogger(RewriteUtil.class);
/** The cube pattern. */
static Pattern cubePattern = Pattern.compile(".*CUBE(.*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE
@@ -91,7 +91,7 @@ public final class RewriteUtil {
throws SemanticException, LensException {
ASTNode ast = HQLParser.parseHQL(query, conf);
- LOG.debug("User query AST:" + ast.dump());
+ log.debug("User query AST:{}", ast.dump());
List<CubeQueryInfo> cubeQueries = new ArrayList<CubeQueryInfo>();
findCubePositions(ast, cubeQueries, query);
for (CubeQueryInfo cqi : cubeQueries) {
@@ -113,12 +113,11 @@ public final class RewriteUtil {
int childCount = ast.getChildCount();
if (ast.getToken() != null) {
if (ast.getChild(0) != null) {
- LOG.debug("First child:" + ast.getChild(0) + " Type:"
- + ((ASTNode) ast.getChild(0)).getToken().getType());
+ log.debug("First child: {} Type:{}", ast.getChild(0), ((ASTNode) ast.getChild(0)).getToken().getType());
}
if (ast.getToken().getType() == HiveParser.TOK_QUERY
&& ((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) {
- LOG.debug("Inside cube clause");
+ log.debug("Inside cube clause");
CubeQueryInfo cqi = new CubeQueryInfo();
cqi.cubeAST = ast;
if (ast.getParent() != null) {
@@ -138,7 +137,7 @@ public final class RewriteUtil {
cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine() - 1, "UNION ALL");
} else {
// Not expected to reach here
- LOG.warn("Unknown query pattern found with AST:" + ast.dump());
+ log.warn("Unknown query pattern found with AST:{}", ast.dump());
throw new SemanticException("Unknown query pattern");
}
} else {
@@ -146,12 +145,12 @@ public final class RewriteUtil {
// one for next AST
// and one for the close parenthesis if there are no more unionall
// or one for the string 'UNION ALL' if there are more union all
- LOG.debug("Child of union all");
+ log.debug("Child of union all");
cqi.endPos = getEndPos(originalQuery, parent.getParent().getChild(1).getCharPositionInLine(), ")",
"UNION ALL");
}
}
- LOG.debug("Adding cqi " + cqi + " query:" + originalQuery.substring(cqi.startPos, cqi.endPos));
+ log.debug("Adding cqi {} query:{}", cqi, originalQuery.substring(cqi.startPos, cqi.endPos));
cubeQueries.add(cqi);
} else {
for (int childPos = 0; childPos < childCount; ++childPos) {
@@ -159,7 +158,7 @@ public final class RewriteUtil {
}
}
} else {
- LOG.warn("Null AST!");
+ log.warn("Null AST!");
}
}
@@ -310,9 +309,7 @@ public final class RewriteUtil {
// user query. We are looping through all sub queries here.
int qIndex = 1;
for (RewriteUtil.CubeQueryInfo cqi : cubeQueries) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Rewriting cube query:" + cqi.query);
- }
+ log.debug("Rewriting cube query: {}", cqi.query);
if (start != cqi.startPos) {
builder.append(replacedQuery.substring(start, cqi.startPos));
@@ -328,9 +325,7 @@ public final class RewriteUtil {
toHQLGauge.markSuccess();
qIndex++;
- if (LOG.isDebugEnabled()) {
- LOG.debug("Rewritten query:" + hqlQuery);
- }
+ log.debug("Rewritten query:", hqlQuery);
builder.append(hqlQuery);
start = cqi.endPos;
@@ -343,7 +338,7 @@ public final class RewriteUtil {
ctx.getDriverContext().setDriverRewriterPlan(driver, getRewriterPlan(this));
succeeded = true;
ctx.setDriverQuery(driver, rewrittenQuery);
- LOG.info("Final rewritten query for driver:" + driver + " is: " + rewrittenQuery);
+ log.info("Final rewritten query for driver: {} is: {}", driver, rewrittenQuery);
} catch (final LensException e) {
@@ -368,7 +363,7 @@ public final class RewriteUtil {
private void captureExceptionInformation(final Exception e) {
- LOG.warn("Driver : " + driver + " Skipped for the query rewriting due to ", e);
+ log.warn("Driver : {} Skipped for the query rewriting due to ", driver, e);
ctx.setDriverRewriteError(driver, e);
failureCause = new StringBuilder(" Driver :")
.append(driver.getClass().getName())
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
index 2a5cce5..79ebe28 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java
@@ -33,19 +33,19 @@ import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.AbstractService;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Service to maintain DB specific static jars. This service is managed by HiveSessionService.
*/
+@Slf4j
public class DatabaseResourceService extends AbstractService {
- public static final Log LOG = LogFactory.getLog(DatabaseResourceService.class);
public static final String NAME = "database-resources";
private Map<String, ClassLoader> classLoaderCache;
private Map<String, List<LensSessionImpl.ResourceEntry>> dbResEntryMap;
@@ -96,12 +96,12 @@ public class DatabaseResourceService extends AbstractService {
super.start();
try {
- LOG.info("Starting loading DB specific resources");
+ log.info("Starting loading DB specific resources");
loadDbResourceEntries();
loadResources();
} catch (LensException e) {
incrCounter(LOAD_RESOURCES_ERRORS);
- LOG.warn("Failed to load DB resource mapping, resources must be added explicitly to session.");
+ log.warn("Failed to load DB resource mapping, resources must be added explicitly to session.");
}
}
@@ -120,13 +120,13 @@ public class DatabaseResourceService extends AbstractService {
try {
String resTopDir =
getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR);
- LOG.info("Database specific resources at " + resTopDir);
+ log.info("Database specific resources at {}", resTopDir);
Path resTopDirPath = new Path(resTopDir);
serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf());
if (!serverFs.exists(resTopDirPath)) {
incrCounter(LOAD_RESOURCES_ERRORS);
- LOG.warn("Database resource location does not exist - " + resTopDir + ". Database jars will not be available");
+ log.warn("Database resource location does not exist - {}. Database jars will not be available", resTopDir);
return;
}
@@ -138,23 +138,20 @@ public class DatabaseResourceService extends AbstractService {
// Get all resources for that db
findResourcesInDir(serverFs, dbName, dbDirPath);
} else {
- LOG.warn("DB resource DIR is not a directory: " + dbDirPath);
+ log.warn("DB resource DIR is not a directory: {}", dbDirPath);
}
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("Found resources " + dbResEntryMap);
- }
-
+ log.debug("Found resources {}", dbResEntryMap);
} catch (IOException io) {
- LOG.error("Error getting list of dbs to load resources from", io);
+ log.error("Error getting list of dbs to load resources from", io);
throw new LensException(io);
} finally {
if (serverFs != null) {
try {
serverFs.close();
} catch (IOException e) {
- LOG.error("Error closing file system instance", e);
+ log.error("Error closing file system instance", e);
}
}
}
@@ -170,14 +167,14 @@ public class DatabaseResourceService extends AbstractService {
jarOrderInputStream = serverFs.open(jarOrderFile);
jars = IOUtils.readLines(jarOrderInputStream, Charset.forName("UTF-8"));
} catch (IOException ioexc) {
- LOG.error("Unable to load jar order file for " + dbDirPath, ioexc);
+ log.error("Unable to load jar order file for {}", dbDirPath, ioexc);
} finally {
IOUtils.closeQuietly(jarOrderInputStream);
}
}
if (jars != null && !jars.isEmpty()) {
- LOG.info(database + " picking jar in jar_order: " + jars);
+ log.info("{} picking jar in jar_order: {}", database, jars);
for (String jar : jars) {
if (StringUtils.isBlank(jar)) {
// skipping empty lines. usually the last line could be empty
@@ -185,13 +182,13 @@ public class DatabaseResourceService extends AbstractService {
}
Path jarFilePath = new Path(dbDirPath, jar);
if (!jar.endsWith(".jar") || !serverFs.exists(jarFilePath)) {
- LOG.info("Resource skipped " + jarFilePath + " for db " + database);
+ log.info("Resource skipped {} for db {}", jarFilePath, database);
continue;
}
addResourceEntry(new LensSessionImpl.ResourceEntry("jar", jarFilePath.toUri().toString()), database);
}
} else {
- LOG.info(database + " picking jars in file list order");
+ log.info("{} picking jars in file list order", database);
for (FileStatus dbResFile : serverFs.listStatus(dbDirPath)) {
// Skip subdirectories
if (serverFs.isDirectory(dbResFile.getPath())) {
@@ -204,14 +201,14 @@ public class DatabaseResourceService extends AbstractService {
if (dbResName.endsWith(".jar")) {
addResourceEntry(new LensSessionImpl.ResourceEntry("jar", dbResUri), database);
} else {
- LOG.info("Resource skipped " + dbResFile.getPath() + " for db " + database);
+ log.info("Resource skipped {} for db {}", dbResFile.getPath(), database);
}
}
}
}
private void addResourceEntry(LensSessionImpl.ResourceEntry entry, String dbName) {
- LOG.info("Adding resource entry " + entry.getLocation() + " for " + dbName);
+ log.info("Adding resource entry {} for {}", entry.getLocation(), dbName);
synchronized (dbResEntryMap) {
List<LensSessionImpl.ResourceEntry> dbEntryList = dbResEntryMap.get(dbName);
if (dbEntryList == null) {
@@ -230,10 +227,10 @@ public class DatabaseResourceService extends AbstractService {
try {
createClassLoader(db);
loadDBJars(db, dbResEntryMap.get(db), true);
- LOG.info("Loaded resources for db " + db + " resources: " + dbResEntryMap.get(db));
+ log.info("Loaded resources for db {} resources: {}", db, dbResEntryMap.get(db));
} catch (LensException exc) {
incrCounter(LOAD_RESOURCES_ERRORS);
- LOG.warn("Failed to load resources for db " + db, exc);
+ log.warn("Failed to load resources for db {}", db, exc);
classLoaderCache.remove(db);
}
}
@@ -280,7 +277,7 @@ public class DatabaseResourceService extends AbstractService {
newUrls.add(new URL(res.getLocation()));
} catch (MalformedURLException e) {
incrCounter(LOAD_RESOURCES_ERRORS);
- LOG.error("Invalid URL " + res.getLocation() + " adding to db " + database, e);
+ log.error("Invalid URL {} adding to db {}", res.getLocation(), database, e);
}
}
@@ -292,7 +289,7 @@ public class DatabaseResourceService extends AbstractService {
return newClassLoader;
} else {
- LOG.warn("Only URL class loader supported");
+ log.warn("Only URL class loader supported");
return Thread.currentThread().getContextClassLoader();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
index 5344fee..d0677cc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
@@ -34,8 +34,6 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
import org.apache.lens.server.util.UtilityMethods;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -49,15 +47,14 @@ import lombok.AccessLevel;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class LensSessionImpl.
*/
+@Slf4j
public class LensSessionImpl extends HiveSessionImpl {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensSessionImpl.class);
-
/** The persist info. */
private LensSessionPersistInfo persistInfo = new LensSessionPersistInfo();
@@ -185,7 +182,7 @@ public class LensSessionImpl extends HiveSessionImpl {
JavaUtils.closeClassLoader(entry.getValue());
}
} catch (Exception e) {
- LOG.error("Error closing session classloader for session: " + getSessionHandle().getSessionId(), e);
+ log.error("Error closing session classloader for session: {}", getSessionHandle().getSessionId(), e);
}
}
sessionDbClassLoaders.clear();
@@ -323,9 +320,7 @@ public class LensSessionImpl extends HiveSessionImpl {
try {
ClassLoader classLoader = getDbResService().getClassLoader(database);
if (classLoader == null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("DB resource service gave null class loader for " + database);
- }
+ log.debug("DB resource service gave null class loader for {}", database);
} else {
if (areResourcesAdded()) {
// We need to update DB specific classloader with added resources
@@ -336,8 +331,8 @@ public class LensSessionImpl extends HiveSessionImpl {
return classLoader == null ? getSessionState().getConf().getClassLoader() : classLoader;
} catch (LensException e) {
- LOG.error("Error getting classloader for database " + database + " for session "
- + getSessionHandle().getSessionId() + " defaulting to session state class loader", e);
+ log.error("Error getting classloader for database {} for session {} "
+ + " defaulting to session state class loader", database, getSessionHandle().getSessionId(), e);
return getSessionState().getConf().getClassLoader();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
index a5033fe..0635e95 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
@@ -36,22 +36,19 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
import org.apache.lens.server.util.ScannedPaths;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import org.glassfish.jersey.media.multipart.FormDataParam;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Session resource api
* <p></p>
* This provides api for all things in session.
*/
@Path("/session")
+@Slf4j
public class SessionResource {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(SessionResource.class);
-
/** The session service. */
private SessionService sessionService;
@@ -212,7 +209,7 @@ public class SessionResource {
service.deleteResource(sessionid, type, matchedPath);
numDeleted++;
} catch (LensException e) {
- LOG.error("Failed to delete resource in service:" + service, e);
+ log.error("Failed to delete resource in service:{}", service, e);
if (numDeleted != 0) {
return new APIResult(Status.PARTIAL, "Delete resource is partial, failed for service:" + service.getName());
} else {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
index 305c031..4144416 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
@@ -35,19 +35,16 @@ import org.apache.lens.server.model.LogSegregationContext;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Logger;
-import org.slf4j.LoggerFactory;
-
import lombok.NonNull;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
/**
* Timer class for monitoring log file rollup.
*/
+@Slf4j
public class StatisticsLogFileScannerTask extends TimerTask {
- /** The Constant LOG. */
- private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(StatisticsLogFileScannerTask.class);
-
/** The Constant LOG_SCANNER_ERRORS. */
public static final String LOG_SCANNER_ERRORS = "log-scanner-errors";
@@ -88,13 +85,13 @@ public class StatisticsLogFileScannerTask extends TimerTask {
try {
service.notifyEvent(event);
} catch (LensException e) {
- LOG.warn("Unable to Notify partition event" + event.getEventName() + " with map " + event.getPartMap());
+ log.warn("Unable to Notify partition event {} with map {}", event.getEventName(), event.getPartMap());
}
}
} catch (Exception exc) {
MetricsService svc = (MetricsService) LensServices.get().getService(MetricsService.NAME);
svc.incrCounter(StatisticsLogFileScannerTask.class, LOG_SCANNER_ERRORS);
- LOG.error("Unknown error in log file scanner ", exc);
+ log.error("Unknown error in log file scanner ", exc);
}
}
@@ -141,12 +138,12 @@ public class StatisticsLogFileScannerTask extends TimerTask {
return;
}
String appenderName = event.substring(event.lastIndexOf(".") + 1, event.length());
- Logger log = Logger.getLogger(event);
- if (log.getAppender(appenderName) == null) {
- LOG.error("Unable to find " + "statistics log appender for " + event + " with appender name " + appenderName);
+ Logger logger = Logger.getLogger(event);
+ if (logger.getAppender(appenderName) == null) {
+ log.error("Unable to find statistics log appender for {} with appender name {}", event, appenderName);
return;
}
- String location = ((FileAppender) log.getAppender(appenderName)).getFile();
+ String location = ((FileAppender) logger.getAppender(appenderName)).getFile();
scanSet.put(appenderName, location);
classSet.put(appenderName, event);
}
[45/50] [abbrv] incubator-lens git commit: Revert "Lens-581 :
Rafactored all API to new api (Vikas Singh via sharad)"
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLServices.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLServices.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLServices.java
deleted file mode 100644
index dabd398..0000000
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLServices.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml;
-
-import java.io.File;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Application;
-import javax.ws.rs.core.UriBuilder;
-
-import org.apache.lens.client.LensClient;
-import org.apache.lens.client.LensClientConfig;
-import org.apache.lens.client.LensMLClient;
-import org.apache.lens.ml.algo.spark.dt.DecisionTreeAlgo;
-import org.apache.lens.ml.algo.spark.kmeans.KMeansAlgo;
-import org.apache.lens.ml.algo.spark.lr.LogisticRegressionAlgo;
-import org.apache.lens.ml.algo.spark.nb.NaiveBayesAlgo;
-import org.apache.lens.ml.algo.spark.svm.SVMAlgo;
-import org.apache.lens.ml.api.*;
-import org.apache.lens.ml.server.MLApp;
-import org.apache.lens.ml.server.MLServiceResource;
-import org.apache.lens.server.LensJerseyTest;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.query.QueryServiceResource;
-import org.apache.lens.server.session.SessionResource;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.ql.metadata.Hive;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.metadata.Table;
-
-import org.glassfish.jersey.client.ClientConfig;
-import org.glassfish.jersey.media.multipart.MultiPartFeature;
-import org.testng.Assert;
-import org.testng.annotations.AfterTest;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.BeforeTest;
-import org.testng.annotations.Test;
-
-
-@Test
-public class TestMLServices extends LensJerseyTest {
- private static final Log LOG = LogFactory.getLog(TestMLServices.class);
- private static final String TEST_DB = "default";
-
- private WebTarget mlTarget;
- private LensMLClient mlClient;
-
- @Override
- protected int getTestPort() {
- return 10002;
- }
-
- @Override
- protected Application configure() {
- return new MLApp(SessionResource.class, QueryServiceResource.class);
- }
-
- @Override
- protected void configureClient(ClientConfig config) {
- config.register(MultiPartFeature.class);
- }
-
- @Override
- protected URI getBaseUri() {
- return UriBuilder.fromUri("http://localhost/").port(getTestPort()).path("/lensapi").build();
- }
-
- @BeforeTest
- public void setUp() throws Exception {
- LOG.info("Testing started!");
- super.setUp();
- Hive hive = Hive.get(new HiveConf());
- Database db = new Database();
- db.setName(TEST_DB);
- //hive.createDatabase(db, true);
- LensClientConfig lensClientConfig = new LensClientConfig();
- lensClientConfig.setLensDatabase(TEST_DB);
- lensClientConfig.set(LensConfConstants.SERVER_BASE_URL,
- "http://localhost:" + 10002 + "/lensapi");
- LensClient client = new LensClient(lensClientConfig);
- mlClient = new LensMLClient(client);
- }
-
- @AfterTest
- public void tearDown() throws Exception {
- super.tearDown();
- Hive hive = Hive.get(new HiveConf());
-
- try {
- hive.dropDatabase(TEST_DB);
- } catch (Exception exc) {
- // Ignore drop db exception
- LOG.error(exc.getMessage());
- }
- mlClient.close();
- }
-
- @BeforeMethod
- public void setMLTarget() {
- mlTarget = target().path("ml");
- }
-
- @Test
- public void testMLResourceUp() throws Exception {
- String mlUpMsg = mlTarget.request().get(String.class);
- Assert.assertEquals(mlUpMsg, MLServiceResource.ML_UP_MESSAGE);
- }
-
- @Test
- public void testGetAlgos() throws Exception {
-
- LOG.info("Testing registered algorithm.");
-
- List<Algo> algos = mlClient.getAlgos();
-
- Assert.assertNotNull(algos);
-
- ArrayList<String> algoNames = new ArrayList();
- for (Algo algo : algos) {
- algoNames.add(algo.getName());
- }
-
- Assert.assertTrue(
- algoNames.contains(new NaiveBayesAlgo().getName()),
- new NaiveBayesAlgo().getName());
-
- Assert.assertTrue(algoNames.contains(new SVMAlgo().getName()),
- new SVMAlgo().getName());
-
- Assert.assertTrue(
- algoNames.contains(new LogisticRegressionAlgo().getName()),
- new LogisticRegressionAlgo().getName());
-
- Assert.assertTrue(
- algoNames.contains(new DecisionTreeAlgo().getName()),
- new DecisionTreeAlgo().getName());
-
- Assert.assertTrue(
- algoNames.contains(new KMeansAlgo().getName()),
- new KMeansAlgo().getName());
- }
-
- @Test
- public void trainAndEval() throws Exception {
- LOG.info("Starting train and eval");
- HiveConf conf = new HiveConf();
-
- String sampleDataFilePath = "data/lr/lr_train.data";
-
- File sampleDataFile = new File(sampleDataFilePath);
- URI sampleDataFileURI = sampleDataFile.toURI();
-
- ArrayList<Feature> features = new ArrayList();
- Feature feature1 = new Feature("feature1", "description", Feature.Type.Categorical, "feature1");
- Feature feature2 = new Feature("feature2", "description", Feature.Type.Categorical, "feature2");
- Feature feature3 = new Feature("feature3", "description", Feature.Type.Categorical, "feature3");
- Feature label = new Feature("label", "description", Feature.Type.Categorical, "label");
-
- features.add(feature1);
- features.add(feature2);
- features.add(feature3);
-
- String tableName = "lr_table_6";
-
- String labelColumn = "label";
- String[] featureNames = {"feature1", "feature2", "feature3"};
- Map<String, String> tableParams = new HashMap<String, String>();
- try {
- ExampleUtils.createTable(conf, TEST_DB, tableName,
- sampleDataFileURI.toString(), labelColumn, tableParams, featureNames);
- } catch (HiveException exc) {
- LOG.error(exc.getLocalizedMessage());
- }
-
- mlClient.createDataSet("lr_table_6", "lr_table_6", TEST_DB);
-
-
- String modelId =
- mlClient.createModel("lr_model", "spark_logistic_regression", new HashMap<String, String>(), features,
- label, mlClient.getSessionHandle());
- LOG.info("model created with Id: " + modelId);
-
- Model model = mlClient.getModel(modelId);
- Assert.assertTrue(model != null, "Null model returned after creation");
-
- String modelInstanceId = mlClient.trainModel(modelId, "lr_table_6", mlClient.getSessionHandle());
- LOG.info("Model Instance created with Id:" + modelInstanceId);
-
- ModelInstance modelInstance;
-
- do {
- modelInstance = mlClient.getModelInstance(modelInstanceId);
- Thread.sleep(2000);
- } while (!(modelInstance.getStatus() == Status.COMPLETED || modelInstance.getStatus() == Status.FAILED));
-
- Assert.assertTrue(modelInstance.getStatus() == Status.COMPLETED, "Training model failed");
-
- Map<String, String> featureMap = new HashMap();
- featureMap.put("feature1", "1");
- featureMap.put("feature2", "0");
- featureMap.put("feature3", "4");
-
- String predictedValue = mlClient.predict(modelInstanceId, featureMap);
- LOG.info("Predicting :" + predictedValue);
-
- Assert.assertTrue(predictedValue.equals("1.0"), "Predicted value incorrect");
-
-
- String predictionId = mlClient.predict(modelInstanceId, "lr_table_6", mlClient.getSessionHandle());
- Prediction prediction;
- do {
- prediction = mlClient.getPrediction(predictionId);
- Thread.sleep(2000);
- } while (!(prediction.getStatus() == Status.COMPLETED || prediction.getStatus() == Status.FAILED));
-
- Assert.assertTrue(prediction != null, "Prediction failed");
-
- Hive metastoreClient = Hive.get(conf);
- Table outputHiveTableForPrediction = metastoreClient.getTable(prediction.getOutputDataSet());
- List<Partition> predictionPartitions = metastoreClient.getPartitions(outputHiveTableForPrediction);
-
- Assert.assertNotNull(predictionPartitions);
-
- String evaluationId = mlClient.evaluate(modelInstanceId, "lr_table_6", mlClient.getSessionHandle());
- LOG.info("Evaluation started with id:" + evaluationId);
-
- Evaluation evaluation;
- do {
- evaluation = mlClient.getEvaluation(evaluationId);
- Thread.sleep(2000);
- } while (!(evaluation.getStatus() == Status.COMPLETED || evaluation.getStatus() == Status.FAILED));
-
- Assert.assertTrue(evaluation.getStatus() == Status.COMPLETED, "Evaluating model failed");
-
- Table outputHiveTableForEvaluation = metastoreClient.getTable(prediction.getOutputDataSet());
- List<Partition> evaluationPartitions = metastoreClient.getPartitions(outputHiveTableForEvaluation);
-
- Assert.assertNotNull(evaluationPartitions);
-
- //Testing cancellation
-
- evaluationId = mlClient.evaluate(modelInstanceId, "lr_table_6", mlClient.getSessionHandle());
- LOG.info("Evaluation started with id:" + evaluationId);
- boolean result = mlClient.cancelEvaluation(evaluationId);
-
- Assert.assertTrue(result);
-
- evaluation = mlClient.getEvaluation(evaluationId);
- Assert
- .assertTrue(evaluation.getStatus() == Status.CANCELLED, "Cancellation failed" + evaluation.getStatus().name());
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/lens-site.xml b/lens-ml-lib/src/test/resources/lens-site.xml
index 2d32d20..3d5dbef 100644
--- a/lens-ml-lib/src/test/resources/lens-site.xml
+++ b/lens-ml-lib/src/test/resources/lens-site.xml
@@ -23,141 +23,140 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
- <property>
- <name>lens.server.drivers</name>
- <value>org.apache.lens.driver.hive.HiveDriver</value>
- </property>
-
- <property>
- <name>test.lens.site.key</name>
- <value>gsvalue</value>
- </property>
-
- <property>
- <name>lens.server.enable.console.metrics</name>
- <value>false</value>
- <description>Enable metrics to be reported on console</description>
- </property>
-
- <property>
- <name>lens.server.persist.location</name>
- <value>target/persist-dir</value>
- </property>
-
- <property>
- <name>lens.query.result.parent.dir</name>
- <value>target/lens-results</value>
- </property>
-
- <property>
- <name>hive.server2.query.log.dir</name>
- <value>target/query-logs</value>
- </property>
-
- <property>
- <name>hive.server2.authentication</name>
- <value>CUSTOM</value>
- </property>
-
- <property>
- <name>hive.server2.custom.authentication.class</name>
- <value>org.apache.lens.server.auth.FooBarAuthenticationProvider</value>
- </property>
-
-
- <property>
- <name>lens.query.enable.mail.notify</name>
- <value>false</value>
- <description>When a query ends, whether to notify the submitter by mail or not.</description>
- </property>
-
- <property>
- <name>lens.server.mail.from.address</name>
- <value>blah@company.com</value>
- <description>The from field in the notifier mail to the submitter.</description>
- </property>
-
- <property>
- <name>lens.server.mail.host</name>
- <value>localhost</value>
- <description>SMTP Host for sending mail</description>
- </property>
-
- <property>
- <name>lens.server.mail.port</name>
- <value>25000</value>
- <description>SMTP Port</description>
- </property>
-
- <property>
- <name>lens.server.ws.resourcenames</name>
- <value>session,metastore,query,quota,scheduler,index,test,ml</value>
- <description>These JAX-RS resources would be started in the specified order when lens-server starts up
- </description>
- </property>
-
- <property>
- <name>lens.server.test.ws.resource.impl</name>
- <value>org.apache.lens.server.TestResource</value>
- <description>Implementation class for Test Resource</description>
- </property>
-
- <property>
- <name>lens.server.ml.ws.resource.impl</name>
- <value>org.apache.lens.ml.server.MLServiceResource</value>
- <description>Implementation class for ML Service Resource</description>
- </property>
-
- <property>
- <name>lens.server.user.resolver.type</name>
- <value>FIXED</value>
- </property>
- <property>
- <name>lens.server.user.resolver.fixed.value</name>
- <value>testlensuser</value>
- </property>
- <property>
- <name>lens.server.domain</name>
- <value>localhost</value>
- </property>
-
- <property>
- <!-- Immediately insert all finished queries to DB -->
- <name>lens.server.max.finished.queries</name>
- <value>1</value>
- </property>
-
- <property>
- <name>lens.server.db.jdbc.url</name>
- <value>jdbc:hsqldb:target/queries.db</value>
- </property>
-
- <property>
- <name>lens.server.servicenames</name>
- <value>session,query,metastore,scheduler,quota,ml,mocknonlens</value>
- </property>
-
- <property>
- <name>lens.server.ml.service.impl</name>
- <value>org.apache.lens.ml.server.MLServiceImpl</value>
- <description>Implementation class for ML service</description>
- </property>
-
- <property>
- <name>lens.ml.drivers</name>
- <value>org.apache.lens.ml.algo.spark.SparkMLDriver</value>
- </property>
-
- <property>
- <name>lens.ml.sparkdriver.spark.master</name>
- <value>local</value>
- </property>
-
-
- <property>
- <name>lens.server.mocknonlens.service.impl</name>
- <value>org.apache.lens.server.MockNonLensService</value>
- <description>Implementation class for session service</description>
- </property>
+ <property>
+ <name>lens.server.drivers</name>
+ <value>org.apache.lens.driver.hive.HiveDriver</value>
+ </property>
+
+ <property>
+ <name>test.lens.site.key</name>
+ <value>gsvalue</value>
+ </property>
+
+ <property>
+ <name>lens.server.enable.console.metrics</name>
+ <value>false</value>
+ <description>Enable metrics to be reported on console</description>
+ </property>
+
+ <property>
+ <name>lens.server.persist.location</name>
+ <value>target/persist-dir</value>
+ </property>
+
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/lens-results</value>
+ </property>
+
+ <property>
+ <name>hive.server2.query.log.dir</name>
+ <value>target/query-logs</value>
+ </property>
+
+ <property>
+ <name>hive.server2.authentication</name>
+ <value>CUSTOM</value>
+ </property>
+
+ <property>
+ <name>hive.server2.custom.authentication.class</name>
+ <value>org.apache.lens.server.auth.FooBarAuthenticationProvider</value>
+ </property>
+
+
+ <property>
+ <name>lens.query.enable.mail.notify</name>
+ <value>false</value>
+ <description>When a query ends, whether to notify the submitter by mail or not.</description>
+ </property>
+
+ <property>
+ <name>lens.server.mail.from.address</name>
+ <value>blah@company.com</value>
+ <description>The from field in the notifier mail to the submitter.</description>
+ </property>
+
+ <property>
+ <name>lens.server.mail.host</name>
+ <value>localhost</value>
+ <description>SMTP Host for sending mail</description>
+ </property>
+
+ <property>
+ <name>lens.server.mail.port</name>
+ <value>25000</value>
+ <description>SMTP Port</description>
+ </property>
+
+ <property>
+ <name>lens.server.ws.resourcenames</name>
+ <value>session,metastore,query,quota,scheduler,index,test,ml</value>
+ <description>These JAX-RS resources would be started in the specified order when lens-server starts up</description>
+ </property>
+
+ <property>
+ <name>lens.server.test.ws.resource.impl</name>
+ <value>org.apache.lens.server.TestResource</value>
+ <description>Implementation class for Test Resource</description>
+ </property>
+
+ <property>
+ <name>lens.server.ml.ws.resource.impl</name>
+ <value>org.apache.lens.ml.server.MLServiceResource</value>
+ <description>Implementation class for ML Service Resource</description>
+ </property>
+
+ <property>
+ <name>lens.server.user.resolver.type</name>
+ <value>FIXED</value>
+ </property>
+ <property>
+ <name>lens.server.user.resolver.fixed.value</name>
+ <value>testlensuser</value>
+ </property>
+ <property>
+ <name>lens.server.domain</name>
+ <value>localhost</value>
+ </property>
+
+ <property>
+ <!-- Immediately insert all finished queries to DB -->
+ <name>lens.server.max.finished.queries</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>lens.server.db.jdbc.url</name>
+ <value>jdbc:hsqldb:target/queries.db</value>
+ </property>
+
+ <property>
+ <name>lens.server.servicenames</name>
+ <value>session,query,metastore,scheduler,quota,ml,mocknonlens</value>
+ </property>
+
+ <property>
+ <name>lens.server.ml.service.impl</name>
+ <value>org.apache.lens.ml.server.MLServiceImpl</value>
+ <description>Implementation class for ML service</description>
+ </property>
+
+ <property>
+ <name>lens.ml.drivers</name>
+ <value>org.apache.lens.ml.algo.spark.SparkMLDriver</value>
+ </property>
+
+ <property>
+ <name>lens.ml.sparkdriver.spark.master</name>
+ <value>local</value>
+ </property>
+
+
+ <property>
+ <name>lens.server.mocknonlens.service.impl</name>
+ <value>org.apache.lens.server.MockNonLensService</value>
+ <description>Implementation class for session service</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/resources/log4j.properties b/lens-ml-lib/src/test/resources/log4j.properties
deleted file mode 100644
index afadc2f..0000000
--- a/lens-ml-lib/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-lensserver.root.logger=INFO, ROOT
-lensserver.request.logger=INFO, REQUEST
-lensserver.audit.logger=INFO, CONSOLE
-lensserver.querystatus.logger=INFO, CONSOLE
-
-log4j.rootLogger=${lensserver.root.logger}
-log4j.logger.org.apache.lens.server.LensServer.request=${lensserver.request.logger}
-log4j.additivity.org.apache.lens.server.LensServer.request=false
-log4j.logger.org.apache.lens.server.LensServer.audit=${lensserver.audit.logger}
-log4j.additivity.org.apache.lens.server.LensServer.audit=false
-log4j.logger.org.apache.lens.server.query.QueryExecutionServiceImpl$QueryStatusLogger=${lensserver.querystatus.logger}
-log4j.additivity.org.apache.lens.server.query.QueryExecutionServiceImpl$QueryStatusLogger=false
-log4j.logger.org.apache.lens.server.stats.event.query.QueryExecutionStatistics=DEBUG, QueryExecutionStatistics
-log4j.additivity.org.apache.lens.server.stats.event.query.QueryExecutionStatistics=false
-
-
-# CONSOLE is set to be a ConsoleAppender.
-log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
-
-# CONSOLE uses PatternLayout.
-log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
-log4j.appender.CONSOLE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
-
-log4j.appender.ROOT=org.apache.log4j.RollingFileAppender
-log4j.appender.ROOT.File=${lens.log.dir}/lensserver.log
-log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout
-log4j.appender.ROOT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
-
-log4j.appender.ROOT.MaxFileSize=100000KB
-# Keep 20 backup files
-log4j.appender.ROOT.MaxBackupIndex=20
-
-
-log4j.appender.AUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.AUDIT.File=${lens.log.dir}/lensserver-audit.log
-log4j.appender.AUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.AUDIT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
-
-log4j.appender.AUDIT.MaxFileSize=100000KB
-# Keep 20 backup files
-log4j.appender.AUDIT.MaxBackupIndex=20
-
-log4j.appender.REQUEST=org.apache.log4j.RollingFileAppender
-log4j.appender.REQUEST.File=${lens.log.dir}/lensserver-requests.log
-log4j.appender.REQUEST.layout=org.apache.log4j.PatternLayout
-log4j.appender.REQUEST.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
-
-log4j.appender.REQUEST.MaxFileSize=100000KB
-# Keep 20 backup files
-log4j.appender.REQUEST.MaxBackupIndex=20
-
-log4j.appender.STATUS=org.apache.log4j.RollingFileAppender
-log4j.appender.STATUS.File=${lens.log.dir}/lensserver-query-status.log
-log4j.appender.STATUS.layout=org.apache.log4j.PatternLayout
-log4j.appender.STATUS.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %x - %m%n
-
-log4j.appender.STATUS.MaxFileSize=100000KB
-# Keep 20 backup files
-log4j.appender.STATUS.MaxBackupIndex=20
-
-
-#Add query statistics logger with hourly rollup
-log4j.appender.QueryExecutionStatistics=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.QueryExecutionStatistics.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.QueryExecutionStatistics.File=${lens.log.dir}/query-stats.log
-log4j.appender.QueryExecutionStatistics.layout=org.apache.lens.server.stats.store.log.StatisticsLogLayout
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index f9c6d92..353c171 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -92,7 +92,7 @@ User CLI Commands
*--+--+
|close/bye|Releases all resources of the server session and exits the shell|
*--+--+
-|debug [[--enable] To print all logs on cli for debugging purpose]|prints all class level logs and verbose logs on cli for debugging purpose. 'debug false' to turn off all class level logging and verbose level logging |
+|debug [[--enable] To print all logs on cli for debugging purpose]|prints all class lelvel logs on cli for debugging purpose|
*--+--+
|get [--key] \<key\>|Fetches and prints session parameter specified with name <<<key>>> from lens server|
*--+--+
@@ -106,7 +106,7 @@ User CLI Commands
*--+--+
|show params|Fetches and prints all session parameter from lens server|
*--+--+
-|verbose [[--enable] Print the clilogger logs on cli]|Show cliLogger logs on cli'verbose false' turns off the cliLogger logs on console|
+|verbose [[--enable] Print the clilogger logs on cli]|Show cliLogger logs for command|
*--+--+
<<Lens Connection Commands>>
@@ -121,7 +121,7 @@ User CLI Commands
*--+--+
|create database [--db] \<database-name\> [--ignoreIfExists \<ignore-if-exists\>]|create a database with specified name. if <<<ignore-if-exists>>> is true, create will not be tried if already exists. Default is false|
*--+--+
-|drop database [--db] \<database-name\> [--cascade ]|drop a database with specified name|
+|drop database [--db] \<database-name\>|drop a database with specified name|
*--+--+
|show databases|displays list of all databases|
*--+--+
[35/50] [abbrv] incubator-lens git commit: LENS-665 : Use Slf4j
logger in lens-cube module
Posted by sh...@apache.org.
LENS-665 : Use Slf4j logger in lens-cube module
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/cb28e069
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/cb28e069
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/cb28e069
Branch: refs/heads/LENS-581
Commit: cb28e0690ed0b89b012cf9dbd4c367b34a4a82e5
Parents: 4c72526
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Jul 24 12:57:30 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jul 24 12:57:30 2015 +0530
----------------------------------------------------------------------
lens-cube/pom.xml | 4 -
.../lens/cube/metadata/AbstractBaseTable.java | 12 ++-
.../lens/cube/metadata/AbstractCubeTable.java | 9 +-
.../lens/cube/metadata/BaseDimAttribute.java | 10 +-
.../org/apache/lens/cube/metadata/Cube.java | 15 +--
.../lens/cube/metadata/CubeDimensionTable.java | 5 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 17 ++--
.../apache/lens/cube/metadata/Dimension.java | 7 +-
.../metadata/timeline/PartitionTimeline.java | 17 ++--
.../lens/cube/parse/AggregateResolver.java | 9 +-
.../apache/lens/cube/parse/CandidateFact.java | 3 -
.../lens/cube/parse/CandidateTableResolver.java | 90 +++++++++---------
.../lens/cube/parse/CubeQueryContext.java | 36 ++++----
.../org/apache/lens/cube/parse/DateUtil.java | 7 +-
.../cube/parse/DenormalizationResolver.java | 14 ++-
.../apache/lens/cube/parse/DimHQLContext.java | 4 -
.../lens/cube/parse/DimOnlyHQLContext.java | 4 -
.../apache/lens/cube/parse/FactHQLContext.java | 9 +-
.../apache/lens/cube/parse/GroupbyResolver.java | 10 +-
.../apache/lens/cube/parse/JoinResolver.java | 54 +++++------
.../lens/cube/parse/LeastPartitionResolver.java | 11 +--
.../cube/parse/LightestDimensionResolver.java | 10 +-
.../lens/cube/parse/LightestFactResolver.java | 11 +--
.../cube/parse/MaxCoveringFactResolver.java | 13 ++-
.../lens/cube/parse/MultiFactHQLContext.java | 4 -
.../lens/cube/parse/SimpleHQLContext.java | 11 +--
.../lens/cube/parse/SingleFactHQLContext.java | 4 -
.../parse/SingleFactMultiStorageHQLContext.java | 4 -
.../lens/cube/parse/StorageTableResolver.java | 96 ++++++++++----------
.../org/apache/lens/cube/parse/StorageUtil.java | 5 -
30 files changed, 234 insertions(+), 271 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml
index 56d7f10..36b508a 100644
--- a/lens-cube/pom.xml
+++ b/lens-cube/pom.xml
@@ -84,10 +84,6 @@
<artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java
index 29f6c02..a60d2e0 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java
@@ -27,13 +27,15 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import com.google.common.base.Preconditions;
+
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
/**
* Abstract table with expressions
*/
-
-
+@Getter
+@Slf4j
public abstract class AbstractBaseTable extends AbstractCubeTable {
private final Set<ExprColumn> expressions;
private static final List<FieldSchema> COLUMNS = new ArrayList<FieldSchema>();
@@ -171,7 +173,7 @@ public abstract class AbstractBaseTable extends AbstractCubeTable {
// Replace measure if already existing
if (exprMap.containsKey(expr.getName().toLowerCase())) {
expressions.remove(getExpressionByName(expr.getName()));
- LOG.info("Replacing expression " + getExpressionByName(expr.getName()) + " with " + expr);
+ log.info("Replacing expression {} with {}", getExpressionByName(expr.getName()), expr);
}
expressions.add(expr);
@@ -187,7 +189,7 @@ public abstract class AbstractBaseTable extends AbstractCubeTable {
*/
public void removeExpression(String exprName) {
if (exprMap.containsKey(exprName.toLowerCase())) {
- LOG.info("Removing expression " + getExpressionByName(exprName));
+ log.info("Removing expression {}", getExpressionByName(exprName));
expressions.remove(getExpressionByName(exprName));
exprMap.remove(exprName.toLowerCase());
MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getExpressionListKey(getName()), expressions);
@@ -234,7 +236,7 @@ public abstract class AbstractBaseTable extends AbstractCubeTable {
// Replace dimension if already existing
if (chainMap.containsKey(joinchain.getName().toLowerCase())) {
joinChains.remove(getChainByName(joinchain.getName()));
- LOG.info("Replacing joinchain " + getChainByName(joinchain.getName()) + " with " + joinchain);
+ log.info("Replacing joinchain {} with {}", getChainByName(joinchain.getName()), joinchain);
}
joinChains.add(joinchain);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
index 9384021..da3a7e5 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractCubeTable.java
@@ -23,10 +23,11 @@ import java.util.*;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.log4j.Logger;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public abstract class AbstractCubeTable implements Named {
- public static final Logger LOG = Logger.getLogger(AbstractCubeTable.class);
private final String name;
private final List<FieldSchema> columns;
private final Map<String, String> properties = new HashMap<String, String>();
@@ -132,8 +133,8 @@ public abstract class AbstractCubeTable implements Named {
i++;
}
if (alterPos != -1) {
- LOG.info("In " + getName() + " replacing column " + toReplace.getName() + ":" + toReplace.getType() + " to "
- + column.getName() + ":" + column.getType());
+ log.info("In {} replacing column {}:{} to {}:{}", getName(), toReplace.getName(), toReplace.getType(),
+ column.getName(), column.getType());
columns.add(alterPos, column);
} else {
columns.add(column);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java
index ebbb8b6..f5a84d8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/BaseDimAttribute.java
@@ -27,11 +27,13 @@ import java.util.Map;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import com.google.common.base.Optional;
+
import lombok.Getter;
import lombok.ToString;
-import lombok.extern.apachecommons.CommonsLog;
+import lombok.extern.slf4j.Slf4j;
-@CommonsLog @ToString(callSuper=true, includeFieldNames=true)
+@Slf4j
+@ToString(callSuper=true, includeFieldNames=true)
public class BaseDimAttribute extends CubeDimAttribute {
@Getter private final String type;
@Getter private Optional<Long> numOfDistinctValues = Optional.absent();
@@ -87,8 +89,8 @@ public class BaseDimAttribute extends CubeDimAttribute {
try {
return Optional.of(Long.parseLong((props.get(MetastoreUtil.getDimNumOfDistinctValuesPropertyKey(name)))));
} catch (NumberFormatException ne) {
- log.error("NumberFormat exception while parsing the num of distinct vlaues "
- + props.get(MetastoreUtil.getDimNumOfDistinctValuesPropertyKey(name)));
+ log.error("NumberFormat exception while parsing the num of distinct vlaues {}",
+ props.get(MetastoreUtil.getDimNumOfDistinctValuesPropertyKey(name)));
}
}
return Optional.absent();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
index d9af602..98fb92e 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Cube.java
@@ -25,6 +25,9 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class Cube extends AbstractBaseTable implements CubeInterface {
private final Set<CubeMeasure> measures;
private final Set<CubeDimAttribute> dimensions;
@@ -234,7 +237,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
// Replace measure if already existing
if (measureMap.containsKey(measure.getName().toLowerCase())) {
measures.remove(getMeasureByName(measure.getName()));
- LOG.info("Replacing measure " + getMeasureByName(measure.getName()) + " with " + measure);
+ log.info("Replacing measure {} with {}", getMeasureByName(measure.getName()), measure);
}
measures.add(measure);
@@ -250,7 +253,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
*/
public boolean removeJoinChain(String chainName) {
if (super.removeJoinChain(chainName)) {
- LOG.info("Removing dimension " + getDimAttributeByName(chainName));
+ log.info("Removing dimension {}", getDimAttributeByName(chainName));
return true;
}
return false;
@@ -270,7 +273,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
// Replace dimension if already existing
if (dimMap.containsKey(dimension.getName().toLowerCase())) {
dimensions.remove(getDimAttributeByName(dimension.getName()));
- LOG.info("Replacing dimension " + getDimAttributeByName(dimension.getName()) + " with " + dimension);
+ log.info("Replacing dimension {} with {}", getDimAttributeByName(dimension.getName()), dimension);
}
dimensions.add(dimension);
@@ -287,7 +290,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
*/
public void removeDimension(String dimName) {
if (dimMap.containsKey(dimName.toLowerCase())) {
- LOG.info("Removing dimension " + getDimAttributeByName(dimName));
+ log.info("Removing dimension {}", getDimAttributeByName(dimName));
dimensions.remove(getDimAttributeByName(dimName));
dimMap.remove(dimName.toLowerCase());
MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeDimensionListKey(getName()), dimensions);
@@ -301,7 +304,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
*/
public void removeMeasure(String msrName) {
if (measureMap.containsKey(msrName.toLowerCase())) {
- LOG.info("Removing measure " + getMeasureByName(msrName));
+ log.info("Removing measure {}", getMeasureByName(msrName));
measures.remove(getMeasureByName(msrName));
measureMap.remove(msrName.toLowerCase());
MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getCubeMeasureListKey(getName()), measures);
@@ -324,7 +327,7 @@ public class Cube extends AbstractBaseTable implements CubeInterface {
timeDims = new LinkedHashSet<String>();
}
if (timeDims.contains(timedDimension)) {
- LOG.info("Timed dimension " + timedDimension + " is" + " already present in cube " + getName());
+ log.info("Timed dimension {} is already present in cube {}", timedDimension, getName());
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java
index 47a30dd..cd80d64 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeDimensionTable.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public final class CubeDimensionTable extends AbstractCubeTable {
private String dimName; // dimension name the dimtabe belongs to
private final Map<String, UpdatePeriod> snapshotDumpPeriods = new HashMap<String, UpdatePeriod>();
@@ -210,7 +213,7 @@ public final class CubeDimensionTable extends AbstractCubeTable {
}
if (snapshotDumpPeriods.containsKey(storage)) {
- LOG.info("Updating dump period for " + storage + " from " + snapshotDumpPeriods.get(storage) + " to " + period);
+ log.info("Updating dump period for {} from {} to {}", storage, snapshotDumpPeriods.get(storage), period);
}
snapshotDumpPeriods.put(storage, period);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index c8abca0..0ebc707 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -42,12 +42,13 @@ import org.apache.thrift.TException;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import lombok.extern.apachecommons.CommonsLog;
+
+import lombok.extern.slf4j.Slf4j;
/**
* Wrapper class around Hive metastore to do cube metastore operations.
*/
-@CommonsLog
+@Slf4j
public class CubeMetastoreClient {
private final HiveConf config;
private final boolean enableCaching;
@@ -299,7 +300,7 @@ public class CubeMetastoreClient {
}
}
}
- log.info("timeline for " + storageTableName + " is: " + get(storageTableName));
+ log.info("timeline for {} is: {}", storageTableName, get(storageTableName));
}
// return the final value from memory
return get(storageTableName);
@@ -311,7 +312,7 @@ public class CubeMetastoreClient {
// First make sure all combinations of update period and partition column have an entry even
// if no partitions exist
String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
- log.info("loading from all partitions: " + storageTableName);
+ log.info("loading from all partitions: {}", storageTableName);
Table storageTable = getTable(storageTableName);
if (getCubeFact(fact).getUpdatePeriods() != null && getCubeFact(fact).getUpdatePeriods().get(
storage) != null) {
@@ -328,8 +329,8 @@ public class CubeMetastoreClient {
UpdatePeriod period = deduceUpdatePeriod(partition);
List<String> values = partition.getValues();
if (values.contains(StorageConstants.LATEST_PARTITION_VALUE)) {
- log.info("dropping latest partition from fact storage table: " + storageTableName
- + ". Spec: " + partition.getSpec());
+ log.info("dropping latest partition from fact storage table: {}. Spec: {}", storageTableName,
+ partition.getSpec());
getClient().dropPartition(storageTableName, values, false);
continue;
}
@@ -347,7 +348,7 @@ public class CubeMetastoreClient {
private void loadTimelinesFromTableProperties(String fact, String storage) throws HiveException, LensException {
// found in table properties, load from there.
String storageTableName = MetastoreUtil.getStorageTableName(fact, Storage.getPrefix(storage));
- log.info("loading from table properties: " + storageTableName);
+ log.info("loading from table properties: {}", storageTableName);
for (UpdatePeriod updatePeriod : getCubeFact(fact).getUpdatePeriods().get(storage)) {
for (String partCol : getTimePartColNamesOfTable(storageTableName)) {
ensureEntry(storageTableName, updatePeriod, partCol).init(getTable(storageTableName));
@@ -371,7 +372,7 @@ public class CubeMetastoreClient {
partition));
} catch (LensException e) {
// to take care of the case where partition name is something like `latest`
- log.error("Couldn't parse partition: " + partition + " with update period: " + updatePeriod + ", skipping.", e);
+ log.error("Couldn't parse partition: {} with update period: {}, skipping.", partition, updatePeriod, e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
index 9cba3a7..27cbc30 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Dimension.java
@@ -27,6 +27,9 @@ import java.util.Set;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class Dimension extends AbstractBaseTable {
private final Set<CubeDimAttribute> attributes;
@@ -192,7 +195,7 @@ public class Dimension extends AbstractBaseTable {
// Replace dimension if already existing
if (attributeMap.containsKey(attribute.getName().toLowerCase())) {
attributes.remove(getAttributeByName(attribute.getName()));
- LOG.info("Replacing attribute " + getAttributeByName(attribute.getName()) + " with " + attribute);
+ log.info("Replacing attribute {} with {}", getAttributeByName(attribute.getName()), attribute);
}
attributes.add(attribute);
@@ -208,7 +211,7 @@ public class Dimension extends AbstractBaseTable {
*/
public void removeAttribute(String attrName) {
if (attributeMap.containsKey(attrName.toLowerCase())) {
- LOG.info("Removing attribute " + getAttributeByName(attrName));
+ log.info("Removing attribute {}", getAttributeByName(attrName));
attributes.remove(getAttributeByName(attrName));
attributeMap.remove(attrName.toLowerCase());
MetastoreUtil.addNameStrings(getProperties(), MetastoreUtil.getDimAttributeListKey(getName()), attributes);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
index 4ede3db..1642482 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/timeline/PartitionTimeline.java
@@ -31,9 +31,10 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
+
import lombok.Data;
import lombok.NonNull;
-import lombok.extern.apachecommons.CommonsLog;
+import lombok.extern.slf4j.Slf4j;
/**
* Represents the in-memory data structure that represents timeline of all existing partitions for a given storage
@@ -43,7 +44,7 @@ import lombok.extern.apachecommons.CommonsLog;
* @see org.apache.lens.cube.metadata.timeline.StoreAllPartitionTimeline
*/
@Data
-@CommonsLog
+@Slf4j
public abstract class PartitionTimeline implements Iterable<TimePartition> {
private final String storageTableName;
private final UpdatePeriod updatePeriod;
@@ -87,10 +88,10 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> {
props.put(entry.getKey().substring(prefix.length()), entry.getValue());
}
}
- log.info("initializing timeline from table properties: "
- + getStorageTableName() + ", " + getUpdatePeriod() + ", " + getPartCol());
+ log.info("initializing timeline from table properties: {},{},{}",
+ getStorageTableName(), getUpdatePeriod(), getPartCol());
initFromProperties(props);
- log.info("initialized to: " + this);
+ log.info("initialized to: {}", this);
}
/**
@@ -115,11 +116,11 @@ public abstract class PartitionTimeline implements Iterable<TimePartition> {
if (getAll() == null) {
return true;
}
- log.info("initializing timeline from batch addition: "
- + getStorageTableName() + ", " + getUpdatePeriod() + ", " + getPartCol());
+ log.info("initializing timeline from batch addition: {},{},{}",
+ getStorageTableName(), getUpdatePeriod(), getPartCol());
boolean result = add(getAll());
all = null;
- log.info("initialized to: " + this);
+ log.info("initialized to: {}", this);
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
index edf0f74..f880495 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/AggregateResolver.java
@@ -28,8 +28,6 @@ import org.apache.lens.cube.metadata.CubeMeasure;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -39,15 +37,16 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.antlr.runtime.CommonToken;
+import lombok.extern.slf4j.Slf4j;
+
/**
* <p> Replace select and having columns with default aggregate functions on them, if default aggregate is defined and
* if there isn't already an aggregate function specified on the columns. </p> <p/> <p> Expressions which already
* contain aggregate sub-expressions will not be changed. </p> <p/> <p> At this point it's assumed that aliases have
* been added to all columns. </p>
*/
+@Slf4j
class AggregateResolver implements ContextRewriter {
- public static final Log LOG = LogFactory.getLog(AggregateResolver.class.getName());
-
public AggregateResolver(Configuration conf) {
}
@@ -82,7 +81,7 @@ class AggregateResolver implements ContextRewriter {
}
}
nonDefaultAggregates = true;
- LOG.info("Query has non default aggregates, no aggregate resolution will be done");
+ log.info("Query has non default aggregates, no aggregate resolution will be done");
}
cubeql.pruneCandidateFactSet(CandidateTablePruneCode.MISSING_DEFAULT_AGGREGATE);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 86c420b..6c88fb3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -30,8 +30,6 @@ import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor;
import org.apache.lens.cube.parse.HQLParser.TreeNode;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -51,7 +49,6 @@ import lombok.Getter;
* Holds context of a candidate fact table.
*/
public class CandidateFact implements CandidateTable {
- public static final Log LOG = LogFactory.getLog(CandidateFact.class.getName());
final CubeFactTable fact;
@Getter
private Set<String> storageTables;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index f861ec7..50a4d53 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -28,8 +28,6 @@ import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext;
import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -37,6 +35,8 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
/**
* This resolver prunes the candidate tables for following cases
* <p/>
@@ -47,9 +47,9 @@ import com.google.common.collect.Sets;
* candidate tables 5. Required denormalized fields are not part of refered tables, there by all the candidates which
* are using denormalized fields.
*/
+@Slf4j
class CandidateTableResolver implements ContextRewriter {
- private static final Log LOG = LogFactory.getLog(CandidateTableResolver.class.getName());
private boolean checkForQueriedColumns = true;
public CandidateTableResolver(Configuration conf) {
@@ -58,7 +58,7 @@ class CandidateTableResolver implements ContextRewriter {
@Override
public void rewriteContext(CubeQueryContext cubeql) throws SemanticException {
if (checkForQueriedColumns) {
- LOG.debug("Dump queried columns:" + cubeql.getTblAliasToColumns());
+ log.debug("Dump queried columns:{}", cubeql.getTblAliasToColumns());
populateCandidateTables(cubeql);
resolveCandidateFactTables(cubeql);
resolveCandidateDimTables(cubeql);
@@ -68,7 +68,7 @@ class CandidateTableResolver implements ContextRewriter {
} else {
// populate optional tables
for (Dimension dim : cubeql.getOptionalDimensions()) {
- LOG.info("Populating optional dim:" + dim);
+ log.info("Populating optional dim:{}", dim);
populateDimTables(dim, cubeql, true);
}
if (cubeql.getAutoJoinCtx() != null) {
@@ -99,7 +99,7 @@ class CandidateTableResolver implements ContextRewriter {
CandidateFact cfact = new CandidateFact(fact, cubeql.getCube());
cubeql.getCandidateFacts().add(cfact);
}
- LOG.info("Populated candidate facts:" + cubeql.getCandidateFacts());
+ log.info("Populated candidate facts: {}", cubeql.getCandidateFacts());
}
if (cubeql.getDimensions().size() != 0) {
@@ -125,7 +125,7 @@ class CandidateTableResolver implements ContextRewriter {
throw new SemanticException(ErrorMsg.NO_CANDIDATE_DIM_AVAILABLE, dim.getName(),
"Dimension tables do not exist");
} else {
- LOG.info("Not considering optional dimension " + dim + " as," + " No dimension tables exist");
+ log.info("Not considering optional dimension {} as, No dimension tables exist", dim);
removeOptionalDim(cubeql, dim);
}
}
@@ -133,7 +133,7 @@ class CandidateTableResolver implements ContextRewriter {
CandidateDim cdim = new CandidateDim(dimtable, dim);
candidates.add(cdim);
}
- LOG.info("Populated candidate dims:" + cubeql.getCandidateDimTables().get(dim) + " for " + dim);
+ log.info("Populated candidate dims: {} for {}", cubeql.getCandidateDimTables().get(dim), dim);
} catch (HiveException e) {
throw new SemanticException(e);
}
@@ -154,8 +154,8 @@ class CandidateTableResolver implements ContextRewriter {
while (iter.hasNext()) {
CandidateTable candidate = iter.next();
if (!allCandidates.contains(candidate)) {
- LOG.info("Removing candidate " + candidate + " from requiredForCandidates of " + dim + ", as it is no more"
- + " candidate");
+ log.info("Removing candidate {} from requiredForCandidates of {}, as it is no more candidate", candidate,
+ dim);
iter.remove();
removedCandidates.add(candidate);
}
@@ -167,7 +167,7 @@ class CandidateTableResolver implements ContextRewriter {
OptionalDimCtx optdim = optdimEntry.getValue();
candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates);
if ((!optdim.colQueried.isEmpty() && optdim.requiredForCandidates.isEmpty()) && !optdim.isRequiredInJoinChain) {
- LOG.info("Not considering optional dimension " + dim + " as all requiredForCandidates are removed");
+ log.info("Not considering optional dimension {} as all requiredForCandidates are removed", dim);
tobeRemoved.add(dim);
}
}
@@ -181,12 +181,12 @@ class CandidateTableResolver implements ContextRewriter {
// remove all the depending candidate table as well
for (CandidateTable candidate : optdim.requiredForCandidates) {
if (candidate instanceof CandidateFact) {
- LOG.info("Not considering fact:" + candidate + " as refered table does not have any valid dimtables");
+ log.info("Not considering fact:{} as refered table does not have any valid dimtables", candidate);
cubeql.getCandidateFacts().remove(candidate);
cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact, new CandidateTablePruneCause(
CandidateTablePruneCode.INVALID_DENORM_TABLE));
} else {
- LOG.info("Not considering dimtable:" + candidate + " as refered table does not have any valid dimtables");
+ log.info("Not considering dimtable:{} as refered table does not have any valid dimtables", candidate);
cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
new CandidateTablePruneCause(CandidateTablePruneCode.INVALID_DENORM_TABLE));
@@ -212,7 +212,7 @@ class CandidateTableResolver implements ContextRewriter {
if (validFactTables != null) {
if (!validFactTables.contains(cfact.getName().toLowerCase())) {
- LOG.info("Not considering fact table:" + cfact + " as it is" + " not a valid fact");
+ log.info("Not considering fact table:{} as it is not a valid fact", cfact);
cubeql
.addFactPruningMsgs(cfact.fact, new CandidateTablePruneCause(CandidateTablePruneCode.INVALID));
i.remove();
@@ -230,7 +230,7 @@ class CandidateTableResolver implements ContextRewriter {
if (!cfact.getColumns().contains(col.toLowerCase())) {
// check if it available as reference, if not remove the candidate
if (!cubeql.getDeNormCtx().addRefUsage(cfact, col, cubeql.getCube().getName())) {
- LOG.info("Not considering fact table:" + cfact + " as column " + col + " is not available");
+ log.info("Not considering fact table:{} as column {} is not available", cfact, col);
cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(col));
toRemove = true;
break;
@@ -245,8 +245,8 @@ class CandidateTableResolver implements ContextRewriter {
// check if chain is optional or not
if (optdim == null || optdim.isRequiredInJoinChain
|| (optdim != null && optdim.requiredForCandidates.contains(cfact))) {
- LOG.info("Not considering fact table:" + cfact + " as columns " + chain.getSourceColumns()
- + " are not available");
+ log.info("Not considering fact table:{} as columns {} are not available", cfact,
+ chain.getSourceColumns());
cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(chain.getSourceColumns()));
toRemove = true;
break;
@@ -261,7 +261,7 @@ class CandidateTableResolver implements ContextRewriter {
cubeql.getExprCtx().updateEvaluables(expr, cfact);
if (!cubeql.getQueriedExprsWithMeasures().contains(expr) && !cubeql.getExprCtx().isEvaluable(expr, cfact)) {
// if expression has no measures, prune facts which cannot evaluate expression
- LOG.info("Not considering fact table:" + cfact + " as expression " + expr + " is not evaluatable");
+ log.info("Not considering fact table:{} as expression {} is not evaluatable", cfact, expr);
cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.expressionNotEvaluable(expr));
toRemove = true;
break;
@@ -273,7 +273,7 @@ class CandidateTableResolver implements ContextRewriter {
if (!checkForColumnExists(cfact, queriedMsrs)
&& (cubeql.getQueriedExprsWithMeasures().isEmpty()
|| cubeql.getExprCtx().allNotEvaluable(cubeql.getQueriedExprsWithMeasures(), cfact))) {
- LOG.info("Not considering fact table:" + cfact + " as columns " + queriedMsrs + " is not available");
+ log.info("Not considering fact table:{} as columns {} is not available", cfact, queriedMsrs);
cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(queriedMsrs,
cubeql.getQueriedExprsWithMeasures()));
toRemove = true;
@@ -305,7 +305,7 @@ class CandidateTableResolver implements ContextRewriter {
List<CandidateFact> cfacts = new ArrayList<CandidateFact>(cubeql.getCandidateFacts());
cfactset = findCoveringSets(cubeql, cfacts, queriedMsrs,
cubeql.getQueriedExprsWithMeasures());
- LOG.info("Measure covering fact sets :" + cfactset);
+ log.info("Measure covering fact sets :{}", cfactset);
String msrString = (!queriedMsrs.isEmpty() ? queriedMsrs.toString() : "")
+ (!cubeql.getQueriedExprsWithMeasures().isEmpty() ? cubeql.getQueriedExprsWithMeasures().toString() : "");
if (cfactset.isEmpty()) {
@@ -353,8 +353,8 @@ class CandidateTableResolver implements ContextRewriter {
cfactset.add(set);
}
} else {
- LOG.info("Couldnt find any set containing remaining measures:" + remainingMsrs + " " + remainingExprs
- + " in " + cfactsPassed);
+ log.info("Couldnt find any set containing remaining measures:{} {} in {}", remainingMsrs, remainingExprs,
+ cfactsPassed);
}
}
}
@@ -388,8 +388,8 @@ class CandidateTableResolver implements ContextRewriter {
|| (optdim != null && optdim.requiredForCandidates.contains(cdim))) {
i.remove();
removed = true;
- LOG.info("Not considering dimtable:" + dimtable + " as its columns are"
- + " not part of any join paths. Join columns:" + colSet);
+ log.info("Not considering dimtable:{} as its columns are not part of any join paths. Join columns:{}",
+ dimtable, colSet);
cubeql.addDimPruningMsgs(dim, dimtable, CandidateTablePruneCause.noColumnPartOfAJoinPath(colSet));
break;
}
@@ -408,8 +408,8 @@ class CandidateTableResolver implements ContextRewriter {
|| (optdim != null && optdim.requiredForCandidates.contains(cdim))) {
i.remove();
removed = true;
- LOG.info("Not considering dimtable:" + dimtable + " as its columns are"
- + " not part of any join paths. Join columns:" + colSet);
+ log.info("Not considering dimtable:{} as its columns are not part of any join paths. Join columns:{}",
+ dimtable, colSet);
cubeql.addDimPruningMsgs(dim, dimtable, CandidateTablePruneCause.noColumnPartOfAJoinPath(colSet));
break;
}
@@ -421,8 +421,8 @@ class CandidateTableResolver implements ContextRewriter {
if (cubeql.getOptionalDimensionMap().get(dim) != null
&& !checkForColumnExists(cdim, cubeql.getOptionalDimensionMap().get(dim).colQueried)) {
i.remove();
- LOG.info("Not considering optional dimtable:" + dimtable + " as its denorm fields do not exist."
- + " Denorm fields:" + cubeql.getOptionalDimensionMap().get(dim).colQueried);
+ log.info("Not considering optional dimtable:{} as its denorm fields do not exist. Denorm fields:{}",
+ dimtable, cubeql.getOptionalDimensionMap().get(dim).colQueried);
cubeql.addDimPruningMsgs(dim, dimtable,
CandidateTablePruneCause.noColumnPartOfAJoinPath(cubeql.getOptionalDimensionMap().get(dim).colQueried));
}
@@ -436,9 +436,8 @@ class CandidateTableResolver implements ContextRewriter {
.getAllJoinPathColumnsOfTable(dim).toString());
} else {
// remove it from optional tables
- LOG.info("Not considering optional dimension " + dim + " as,"
- + " No dimension table has the queried columns:" + optdim.colQueried
- + " Clearing the required for candidates:" + optdim.requiredForCandidates);
+ log.info("Not considering optional dimension {} as, No dimension table has the queried columns:{}"
+ + " Clearing the required for candidates:{}", dim, optdim.colQueried, optdim.requiredForCandidates);
removeOptionalDim(cubeql, dim);
}
}
@@ -467,8 +466,8 @@ class CandidateTableResolver implements ContextRewriter {
if (optdim == null || optdim.isRequiredInJoinChain
|| (optdim != null && optdim.requiredForCandidates.contains(cfact))) {
i.remove();
- LOG.info("Not considering fact table:" + fact + " as it does not have columns"
- + " in any of the join paths. Join columns:" + colSet);
+ log.info("Not considering fact table:{} as it does not have columns in any of the join paths."
+ + " Join columns:{}", fact, colSet);
cubeql.addFactPruningMsgs(fact, CandidateTablePruneCause.noColumnPartOfAJoinPath(colSet));
break;
}
@@ -492,7 +491,7 @@ class CandidateTableResolver implements ContextRewriter {
if (cubeql.getAutoJoinCtx() == null) {
Set<Dimension> optionaldims = new HashSet<Dimension>(cubeql.getOptionalDimensions());
for (Dimension dim : optionaldims) {
- LOG.info("Not considering optional dimension " + dim + " as," + " automatic join resolver is disbled ");
+ log.info("Not considering optional dimension {} as, automatic join resolver is disbled ", dim);
removeOptionalDim(cubeql, dim);
}
return;
@@ -510,14 +509,13 @@ class CandidateTableResolver implements ContextRewriter {
CandidateTable candidate = iter.next();
boolean remove = false;
if (cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim) == null) {
- LOG.info("Removing candidate " + candidate + " from requiredForCandidates of " + dim + ", as no join paths"
- + " exist");
+ log.info("Removing candidate {} from requiredForCandidates of {}, as no join paths exist", candidate, dim);
remove = true;
} else {
List<String> colSet = cubeql.getAutoJoinCtx().getJoinPathFromColumns().get(dim).get(candidate.getBaseTable());
if (!checkForColumnExists(candidate, colSet)) {
- LOG.info("Removing candidate " + candidate + " from requiredForCandidates of " + dim + ", as columns:"
- + colSet + " do not exist");
+ log.info("Removing candidate {} from requiredForCandidates of {}, as columns:{} do not exist", candidate,
+ dim, colSet);
remove = true;
}
}
@@ -554,13 +552,13 @@ class CandidateTableResolver implements ContextRewriter {
if (!candidatesReachableThroughRefs.contains(candidate)) {
if (candidate instanceof CandidateFact) {
if (cubeql.getCandidateFacts().contains(candidate)) {
- LOG.info("Not considering fact:" + candidate + " as its required optional dims are not reachable");
+ log.info("Not considering fact:{} as its required optional dims are not reachable", candidate);
cubeql.getCandidateFacts().remove(candidate);
cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
CandidateTablePruneCause.columnNotFound(col));
}
} else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
- LOG.info("Not considering dimtable:" + candidate + " as its required optional dims are not reachable");
+ log.info("Not considering dimtable:{} as its required optional dims are not reachable", candidate);
cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(),
(CubeDimensionTable) candidate.getTable(),
@@ -605,13 +603,13 @@ class CandidateTableResolver implements ContextRewriter {
}
if (candidate instanceof CandidateFact) {
if (cubeql.getCandidateFacts().contains(candidate)) {
- LOG.info("Not considering fact:" + candidate + " as is not reachable through any optional dim");
+ log.info("Not considering fact:{} as is not reachable through any optional dim", candidate);
cubeql.getCandidateFacts().remove(candidate);
cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol()));
}
} else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
- LOG.info("Not considering dimtable:" + candidate + " as is not reachable through any optional dim");
+ log.info("Not considering dimtable:{} as is not reachable through any optional dim", candidate);
cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol()));
@@ -627,7 +625,7 @@ class CandidateTableResolver implements ContextRewriter {
Dimension dim = optdimEntry.getKey();
OptionalDimCtx optdim = optdimEntry.getValue();
if ((!optdim.colQueried.isEmpty() && optdim.requiredForCandidates.isEmpty()) && !optdim.isRequiredInJoinChain) {
- LOG.info("Not considering optional dimension " + dim + " as all requiredForCandidates are removed");
+ log.info("Not considering optional dimension {} as all requiredForCandidates are removed", dim);
tobeRemoved.add(dim);
}
}
@@ -649,9 +647,9 @@ class CandidateTableResolver implements ContextRewriter {
// check if the column is an expression
if (cdim.getBaseTable().getExpressionNames().contains(col)) {
cubeql.getExprCtx().updateEvaluables(col, cdim);
- // check if the expression is evaluatable
+ // check if the expression is evaluable
if (!cubeql.getExprCtx().isEvaluable(col, cdim)) {
- LOG.info("Not considering dimtable:" + cdim + " as expression " + col + " is not evaluatable");
+ log.info("Not considering dimtable: {} as expression {} is not evaluable", cdim, col);
cubeql.addDimPruningMsgs(dim, cdim.getTable(), CandidateTablePruneCause.expressionNotEvaluable(
col));
i.remove();
@@ -660,7 +658,7 @@ class CandidateTableResolver implements ContextRewriter {
} else if (!cubeql.getDeNormCtx().addRefUsage(cdim, col, dim.getName())) {
// check if it available as reference, if not remove the
// candidate
- LOG.info("Not considering dimtable:" + cdim + " as column " + col + " is not available");
+ log.info("Not considering dimtable: {} as column {} is not available", cdim, col);
cubeql.addDimPruningMsgs(dim, cdim.getTable(), CandidateTablePruneCause.columnNotFound(col));
i.remove();
break;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 39b13cc..65c240a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -33,8 +33,6 @@ import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
@@ -49,18 +47,17 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.AllArgsConstructor;
-
import lombok.Data;
-
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class CubeQueryContext implements TrackQueriedColumns {
public static final String TIME_RANGE_FUNC = "time_range_in";
public static final String NOW = "now";
public static final String DEFAULT_TABLE = "_default_";
- public static final Log LOG = LogFactory.getLog(CubeQueryContext.class.getName());
private final ASTNode ast;
@Getter
private final QB qb;
@@ -225,7 +222,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
// try adding them as joinchains
boolean added = addJoinChain(alias, false);
if (!added) {
- LOG.info("Queried tables do not exist. Missing table:" + alias);
+ log.info("Queried tables do not exist. Missing table:{}", alias);
throw new SemanticException(ErrorMsg.NEITHER_CUBE_NOR_DIMENSION);
}
}
@@ -260,10 +257,10 @@ public class CubeQueryContext implements TrackQueriedColumns {
String destTable = joinchain.getDestTable();
boolean added = addQueriedTable(alias, destTable, isOptional, true);
if (!added) {
- LOG.info("Queried tables do not exist. Missing tables:" + destTable);
+ log.info("Queried tables do not exist. Missing tables:{}", destTable);
throw new SemanticException(ErrorMsg.NEITHER_CUBE_NOR_DIMENSION);
}
- LOG.info("Added join chain for " + destTable);
+ log.info("Added join chain for {}", destTable);
return true;
}
@@ -444,8 +441,8 @@ public class CubeQueryContext implements TrackQueriedColumns {
if (!optDim.isRequiredInJoinChain) {
optDim.isRequiredInJoinChain = isRequiredInJoin;
}
- LOG.info("Adding optional dimension:" + dim + " optDim:" + optDim
- + (cubeCol == null ? "" : " for column:" + cubeCol + " isRef:" + isRef));
+ log.info("Adding optional dimension:{} optDim:{} {} isRef:{}", dim , optDim,
+ (cubeCol == null ? "" : " for column:" + cubeCol), isRef);
} catch (HiveException e) {
throw new SemanticException(e);
}
@@ -581,7 +578,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
builder.append("\n Destination:");
builder.append("\n\t dest expr:" + qb.getParseInfo().getDestForClause(clause).dump());
}
- LOG.info(builder.toString());
+ log.info(builder.toString());
}
void printJoinTree(QBJoinTree joinTree, StringBuilder builder) {
@@ -777,8 +774,8 @@ public class CubeQueryContext implements TrackQueriedColumns {
for (Dimension dim : dimensions) {
if (candidateDims.get(dim) != null && candidateDims.get(dim).size() > 0) {
CandidateDim cdim = candidateDims.get(dim).iterator().next();
- LOG.info("Available candidate dims are:" + candidateDims.get(dim) + ", picking up " + cdim.dimtable
- + " for querying");
+ log.info("Available candidate dims are:{}, picking up {} for querying.", candidateDims.get(dim),
+ cdim.dimtable);
dimsToQuery.put(dim, cdim);
} else {
String reason = "";
@@ -814,7 +811,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
if (hasCubeInQuery()) {
if (candidateFactSets.size() > 0) {
facts = candidateFactSets.iterator().next();
- LOG.info("Available candidate facts:" + candidateFactSets + ", picking up " + facts + " for querying");
+ log.info("Available candidate facts:{}, picking up {} for querying", candidateFactSets, facts);
} else {
String reason = "";
if (!factPruningMsgs.isEmpty()) {
@@ -917,7 +914,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
}
dimsToQuery.putAll(pickCandidateDimsToQuery(joiningTables));
}
- LOG.info("Picked Fact:" + cfacts + " dimsToQuery:" + dimsToQuery);
+ log.info("Picked Fact:{} dimsToQuery: {}" + dimsToQuery, cfacts);
pickedDimTables = dimsToQuery.values();
pickedFacts = cfacts;
if (cfacts != null) {
@@ -952,8 +949,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
ParseDriver pd = new ParseDriver();
ASTNode tree;
try {
- LOG.info("HQL:" + hql);
- System.out.println("Rewritten HQL:" + hql);
+ log.info("HQL:{}", hql);
tree = pd.parse(hql, ctx);
} catch (ParseException e) {
throw new SemanticException(e);
@@ -1195,8 +1191,8 @@ public class CubeQueryContext implements TrackQueriedColumns {
for (Iterator<Set<CandidateFact>> i = candidateFactSets.iterator(); i.hasNext();) {
Set<CandidateFact> cfacts = i.next();
if (!candidateFacts.containsAll(cfacts)) {
- LOG.info("Not considering fact table set:" + cfacts
- + " as they have non candidate tables and facts missing because of " + pruneCause);
+ log.info("Not considering fact table set:{} as they have non candidate tables and facts missing because of {}",
+ cfacts, pruneCause);
i.remove();
}
}
@@ -1225,7 +1221,7 @@ public class CubeQueryContext implements TrackQueriedColumns {
for (Iterator<CandidateFact> i = candidateFacts.iterator(); i.hasNext();) {
CandidateFact cfact = i.next();
if (!allCoveringFacts.contains(cfact)) {
- LOG.info("Not considering fact table:" + cfact + " as " + pruneCause);
+ log.info("Not considering fact table:{} as {}", cfact, pruneCause);
addFactPruningMsgs(cfact.fact, pruneCause);
i.remove();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
index 9931e7c..9e295d3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
@@ -35,18 +35,17 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.log4j.Logger;
import lombok.Data;
import lombok.EqualsAndHashCode;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public final class DateUtil {
private DateUtil() {
}
- public static final Logger LOG = Logger.getLogger(DateUtil.class);
-
/*
* NOW -> new java.util.Date() NOW-7DAY -> a date one week earlier NOW (+-)
* <NUM>UNIT or Hardcoded dates in DD-MM-YYYY hh:mm:ss,sss
@@ -143,7 +142,7 @@ public final class DateUtil {
try {
return ABSDATE_PARSER.get().parse(getAbsDateFormatString(str));
} catch (ParseException e) {
- LOG.error("Invalid date format. expected only " + ABSDATE_FMT + " date provided:" + str, e);
+ log.error("Invalid date format. expected only {} date provided:{}", ABSDATE_FMT, str, e);
throw new SemanticException(e, ErrorMsg.WRONG_TIME_RANGE_FORMAT, ABSDATE_FMT, str);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
index 075cdb4..4a95d5a 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DenormalizationResolver.java
@@ -28,8 +28,6 @@ import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCo
import org.apache.lens.cube.parse.ExpressionResolver.ExprSpecContext;
import org.apache.lens.cube.parse.ExpressionResolver.ExpressionContext;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.ASTNode;
@@ -39,6 +37,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.antlr.runtime.CommonToken;
import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
/**
* This class resolves all the reference columns that are queried.
@@ -46,10 +45,9 @@ import lombok.ToString;
* Keeps track of the context that if any candidate needs to use columns through tables referenced and replaces the
* columns from referenced tables in all the ASTs
*/
+@Slf4j
public class DenormalizationResolver implements ContextRewriter {
- private static final Log LOG = LogFactory.getLog(DenormalizationResolver.class);
-
public DenormalizationResolver(Configuration conf) {
}
@@ -143,7 +141,7 @@ public class DenormalizationResolver implements ContextRewriter {
// should not be required here. Join resolution will figure out if
// there is no path
// to the source table
- LOG.info("Adding denormalized column for column:" + col + " for table:" + table);
+ log.info("Adding denormalized column for column:{} for table:{}", col, table);
Set<ReferencedQueriedColumn> refCols = tableToRefCols.get(table.getName());
if (refCols == null) {
refCols = new HashSet<ReferencedQueriedColumn>();
@@ -185,7 +183,7 @@ public class DenormalizationResolver implements ContextRewriter {
if (pickedReferences.containsKey(col)) {
for (PickedReference ref : pickedReferences.get(col)) {
if (ref.srcAlias.equalsIgnoreCase(srcAlias)) {
- LOG.info("Picked reference for " + col + " ref:" + pickedReferences.get(col));
+ log.info("Picked reference for {} ref:{}", col, pickedReferences.get(col));
return ref;
}
}
@@ -378,7 +376,7 @@ public class DenormalizationResolver implements ContextRewriter {
if (denormCtx.tableToRefCols.containsKey(cfact.getName())) {
for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cfact.getName())) {
if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) {
- LOG.info("Not considering fact table:" + cfact + " as column " + refcol.col + " is not available");
+ log.info("Not considering fact table:{} as column {} is not available", cfact, refcol.col);
cubeql.addFactPruningMsgs(cfact.fact, CandidateTablePruneCause.columnNotFound(refcol.col.getName()));
i.remove();
}
@@ -398,7 +396,7 @@ public class DenormalizationResolver implements ContextRewriter {
if (denormCtx.tableToRefCols.containsKey(cdim.getName())) {
for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cdim.getName())) {
if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) {
- LOG.info("Not considering dim table:" + cdim + " as column " + refcol.col + " is not available");
+ log.info("Not considering dim table:{} as column {} is not available", cdim, refcol.col);
cubeql.addDimPruningMsgs(dim, cdim.dimtable,
CandidateTablePruneCause.columnNotFound(refcol.col.getName()));
i.remove();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
index 63aaa12..3814cf6 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimHQLContext.java
@@ -26,8 +26,6 @@ import java.util.Set;
import org.apache.lens.cube.metadata.Dimension;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
@@ -38,8 +36,6 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
*/
abstract class DimHQLContext extends SimpleHQLContext {
- public static final Log LOG = LogFactory.getLog(DimHQLContext.class.getName());
-
private final Map<Dimension, CandidateDim> dimsToQuery;
private final Set<Dimension> queriedDims;
private String where;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
index 9a2a44a..922501d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DimOnlyHQLContext.java
@@ -23,8 +23,6 @@ import java.util.Set;
import org.apache.lens.cube.metadata.Dimension;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
@@ -35,8 +33,6 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
*/
class DimOnlyHQLContext extends DimHQLContext {
- public static final Log LOG = LogFactory.getLog(DimOnlyHQLContext.class.getName());
-
DimOnlyHQLContext(Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext query) throws SemanticException {
super(query, dimsToQuery, dimsToQuery.keySet(), query.getSelectTree(),
query.getWhereTree(), query.getGroupByTree(), query.getOrderByTree(),
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java
index 4bd89a7..623c58b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/FactHQLContext.java
@@ -23,17 +23,16 @@ import java.util.Set;
import org.apache.lens.cube.metadata.Dimension;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import lombok.extern.slf4j.Slf4j;
+
/**
* HQL context class which passes all query strings from the fact and works with required dimensions for the fact.
*/
+@Slf4j
public class FactHQLContext extends DimHQLContext {
- public static final Log LOG = LogFactory.getLog(FactHQLContext.class.getName());
-
private final CandidateFact fact;
private final Set<Dimension> factDims;
@@ -43,7 +42,7 @@ public class FactHQLContext extends DimHQLContext {
.getHavingTree(), null);
this.fact = fact;
this.factDims = factDims;
- LOG.info("factDims:" + factDims + " for fact:" + fact);
+ log.info("factDims:{} for fact:{}", factDims, fact);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
index 3201c09..4d2692b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/GroupbyResolver.java
@@ -27,8 +27,6 @@ import java.util.List;
import org.apache.lens.cube.metadata.AbstractBaseTable;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
@@ -38,11 +36,13 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.Tree;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Promotes groupby to select and select to groupby.
*/
+@Slf4j
class GroupbyResolver implements ContextRewriter {
- private static final Log LOG = LogFactory.getLog(GroupbyResolver.class.getName());
private final boolean selectPromotionEnabled;
private final boolean groupbyPromotionEnabled;
@@ -62,7 +62,7 @@ class GroupbyResolver implements ContextRewriter {
}
if (!groupByExprs.isEmpty()) {
- LOG.info("Not promoting select expression to groupby," + " since there are already group by expressions");
+ log.info("Not promoting select expression to groupby, since there are already group by expressions");
return;
}
@@ -132,7 +132,7 @@ class GroupbyResolver implements ContextRewriter {
for (String expr : selectExprs) {
expr = getExpressionWithoutAlias(cubeql, expr);
if (!cubeql.isAggregateExpr(expr)) {
- LOG.info("Not promoting groupby expression to select, since there are expression projected");
+ log.info("Not promoting groupby expression to select, since there are expression projected");
return;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
index 97c7a46..7d04d19 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/JoinResolver.java
@@ -28,22 +28,20 @@ import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCo
import org.apache.lens.cube.parse.CubeQueryContext.OptionalDimCtx;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.*;
import lombok.*;
+import lombok.extern.slf4j.Slf4j;
/**
* JoinResolver.
*/
+@Slf4j
class JoinResolver implements ContextRewriter {
- private static final Log LOG = LogFactory.getLog(JoinResolver.class);
-
@ToString
public static class JoinClause implements Comparable<JoinClause> {
private final int cost;
@@ -339,9 +337,9 @@ class JoinResolver implements ContextRewriter {
this.autoJoinTarget = autoJoinTarget;
this.joinTypeCfg = joinTypeCfg;
this.joinsResolved = joinsResolved;
- LOG.debug("All join paths:" + allPaths);
- LOG.debug("Join path from columns:" + joinPathFromColumns);
- LOG.debug("Join path to columns:" + joinPathToColumns);
+ log.debug("All join paths:{}", allPaths);
+ log.debug("Join path from columns:{}", joinPathFromColumns);
+ log.debug("Join path to columns:{}", joinPathToColumns);
}
public AbstractCubeTable getAutoJoinTarget() {
@@ -413,10 +411,6 @@ class JoinResolver implements ContextRewriter {
}
}
- public void printAllPaths(String src) {
- LOG.info(src + " All paths" + allPaths);
- }
-
public void removeJoinedTable(Dimension dim) {
allPaths.remove(Aliased.create(dim));
joinPathFromColumns.remove(dim);
@@ -429,7 +423,7 @@ class JoinResolver implements ContextRewriter {
public String getFromString(String fromTable, CandidateFact fact, Set<Dimension> qdims,
Map<Dimension, CandidateDim> dimsToQuery, CubeQueryContext cubeql) throws SemanticException {
String fromString = fromTable;
- LOG.info("All paths dump:" + cubeql.getAutoJoinCtx().getAllPaths());
+ log.info("All paths dump:{}", cubeql.getAutoJoinCtx().getAllPaths());
if (qdims == null || qdims.isEmpty()) {
return fromString;
}
@@ -630,7 +624,7 @@ class JoinResolver implements ContextRewriter {
// This path requires some columns from the cube which are not
// present in the candidate fact
// Remove this path
- LOG.info("Removing join path:" + jp + " as columns :" + cubeCols + " dont exist");
+ log.info("Removing join path:{} as columns :{} dont exist", jp, cubeCols);
paths.remove(i);
i--;
}
@@ -666,7 +660,7 @@ class JoinResolver implements ContextRewriter {
if (cols != null && (dimColumns.get(refTable) == null || !dimColumns.get(refTable).containsAll(cols))) {
// This path requires some columns from the cube which are not present in any candidate dim
// Remove this path
- LOG.info("Removing join path:" + jp + " as columns :" + cols + " dont exist");
+ log.info("Removing join path:{} as columns :{} dont exist", jp, cols);
paths.remove(i);
i--;
break;
@@ -703,7 +697,7 @@ class JoinResolver implements ContextRewriter {
// This path requires some columns from the cube which are not
// present in the candidate fact
// Remove this path
- LOG.info("Removing join path:" + jp + " as columns :" + cubeCols + " dont exist");
+ log.info("Removing join path:{} as columns :{} dont exist", jp, cubeCols);
paths.remove(i);
i--;
}
@@ -727,7 +721,7 @@ class JoinResolver implements ContextRewriter {
// This path requires some columns from the dimension which are
// not present in the candidate dim
// Remove this path
- LOG.info("Removing join path:" + jp + " as columns :" + candidateDimCols + " dont exist");
+ log.info("Removing join path:{} as columns :{} dont exist", jp, candidateDimCols);
paths.remove(i);
i--;
}
@@ -752,9 +746,9 @@ class JoinResolver implements ContextRewriter {
allPaths = new LinkedHashMap<Aliased<Dimension>, List<SchemaGraph.JoinPath>>(this.allPaths);
}
// prune allPaths with qdims
- LOG.info("pruning allPaths before generating all permutations.");
- LOG.info("allPaths: " + allPaths);
- LOG.info("qdims: " + qdims);
+ log.info("pruning allPaths before generating all permutations.");
+ log.info("allPaths: {}", allPaths);
+ log.info("qdims: {}", qdims);
pruneAllPathsWithQueriedDims(allPaths, qdims);
// Number of paths in each path set
@@ -828,7 +822,7 @@ class JoinResolver implements ContextRewriter {
while (iter.hasNext()) {
Map.Entry<Aliased<Dimension>, List<SchemaGraph.JoinPath>> cur = iter.next();
if (!qdims.contains(cur.getKey().getObject())) {
- LOG.info("removing from allPaths: " + cur);
+ log.info("removing from allPaths: {}", cur);
iter.remove();
}
}
@@ -855,7 +849,7 @@ class JoinResolver implements ContextRewriter {
throw new SemanticException(ErrorMsg.NO_JOIN_PATH, qdims.toString(), autoJoinTarget.getName());
}
- LOG.info("Fact:" + fact + " minCostClause:" + minCostClause);
+ log.info("Fact: {} minCostClause:{}", fact, minCostClause);
if (fact != null) {
cubeql.getAutoJoinCtx().getFactClauses().put(fact, minCostClause);
} else {
@@ -875,8 +869,8 @@ class JoinResolver implements ContextRewriter {
CubeDimensionTable dimtable = cdim.dimtable;
if (!cdim.getColumns().containsAll(minCostClause.chainColumns.get(dim))) {
i.remove();
- LOG.info("Not considering dimtable:" + dimtable + " as its columns are"
- + " not part of any join paths. Join columns:" + minCostClause.chainColumns.get(dim));
+ log.info("Not considering dimtable:{} as its columns are not part of any join paths. Join columns:{}",
+ dimtable, minCostClause.chainColumns.get(dim));
cubeql.addDimPruningMsgs(dim, cdim.dimtable,
CandidateTablePruneCause.noColumnPartOfAJoinPath(minCostClause.chainColumns.get(dim)));
}
@@ -997,7 +991,7 @@ class JoinResolver implements ContextRewriter {
String targetDimAlias = cubeql.getQb().getTabAliases().iterator().next();
String targetDimTable = cubeql.getQb().getTabNameForAlias(targetDimAlias);
if (targetDimTable == null) {
- LOG.warn("Null table for alias " + targetDimAlias);
+ log.warn("Null table for alias {}", targetDimAlias);
return;
}
target = cubeql.getMetastoreClient().getDimension(targetDimTable);
@@ -1005,7 +999,7 @@ class JoinResolver implements ContextRewriter {
}
searchDimensionTables(cubeql.getMetastoreClient(), joinClause);
if (target == null) {
- LOG.warn("Can't resolve joins for null target");
+ log.warn("Can't resolve joins for null target");
return;
}
@@ -1024,7 +1018,7 @@ class JoinResolver implements ContextRewriter {
dimTables.remove(target);
if (dimTables.isEmpty() && cubeql.getJoinchains().isEmpty()) {
// Joins not required
- LOG.info("No dimension tables to resolve and no join chains present!");
+ log.info("No dimension tables to resolve and no join chains present!");
return;
}
@@ -1045,10 +1039,10 @@ class JoinResolver implements ContextRewriter {
addOptionalTables(cubeql, multipleJoinPaths.get(aliasedJoinee), cubeql.getDimensions().contains(joinee));
} else {
// No link to cube from this dim, can't proceed with query
- if (LOG.isDebugEnabled()) {
+ if (log.isDebugEnabled()) {
graph.print();
}
- LOG.warn("No join path between " + joinee.getName() + " and " + target.getName());
+ log.warn("No join path between {} and {}", joinee.getName(), target.getName());
if (cubeql.getDimensions().contains(joinee)) {
throw new SemanticException(ErrorMsg.NO_JOIN_PATH, joinee.getName(), target.getName());
} else {
@@ -1057,13 +1051,13 @@ class JoinResolver implements ContextRewriter {
for (CandidateTable candidate : candidates) {
if (candidate instanceof CandidateFact) {
if (cubeql.getCandidateFacts().contains(candidate)) {
- LOG.info("Not considering fact:" + candidate + " as there is no join path to " + joinee);
+ log.info("Not considering fact:{} as there is no join path to {}", candidate, joinee);
cubeql.getCandidateFacts().remove(candidate);
cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact, new CandidateTablePruneCause(
CandidateTablePruneCode.COLUMN_NOT_FOUND));
}
} else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
- LOG.info("Not considering dimtable:" + candidate + " as there is no join path to " + joinee);
+ log.info("Not considering dimtable:{} as there is no join path to {}", candidate, joinee);
cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
cubeql.addDimPruningMsgs(
(Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
index 6e4e948..7f02ae8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LeastPartitionResolver.java
@@ -22,17 +22,16 @@ import java.util.*;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Prune candidate fact sets which require more partitions than minimum parts.
*/
+@Slf4j
class LeastPartitionResolver implements ContextRewriter {
- public static final Log LOG = LogFactory.getLog(LeastPartitionResolver.class.getName());
-
public LeastPartitionResolver(Configuration conf) {
}
@@ -54,8 +53,8 @@ class LeastPartitionResolver implements ContextRewriter {
for (Iterator<Set<CandidateFact>> i = cubeql.getCandidateFactSets().iterator(); i.hasNext();) {
Set<CandidateFact> facts = i.next();
if (factPartCount.get(facts) > minPartitions) {
- LOG.info("Not considering facts:" + facts + " from candidate fact tables as it requires more partitions to"
- + " be queried:" + factPartCount.get(facts) + " minimum:" + minPartitions);
+ log.info("Not considering facts:{} from candidate fact tables as it requires more partitions to be"
+ + " queried:{} minimum:{}", facts, factPartCount.get(facts), minPartitions);
i.remove();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestDimensionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestDimensionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestDimensionResolver.java
index b4c603b..4ae6226 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestDimensionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestDimensionResolver.java
@@ -23,16 +23,16 @@ import java.util.*;
import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Prune dimension tables having more weight than minimum
*/
+@Slf4j
class LightestDimensionResolver implements ContextRewriter {
- public static final Log LOG = LogFactory.getLog(LightestDimensionResolver.class.getName());
public LightestDimensionResolver(Configuration conf) {
}
@@ -55,8 +55,8 @@ class LightestDimensionResolver implements ContextRewriter {
for (Iterator<CandidateDim> i = entry.getValue().iterator(); i.hasNext();) {
CandidateDim dim = i.next();
if (dimWeightMap.get(dim) > minWeight) {
- LOG.info("Not considering dimtable:" + dim + " from candidate dimension tables as it has more weight:"
- + dimWeightMap.get(dim) + " minimum:" + minWeight);
+ log.info("Not considering dimtable:{} from candidate dimension tables as it has more weight:{} minimum:{}",
+ dim, dimWeightMap.get(dim), minWeight);
cubeql.addDimPruningMsgs(entry.getKey(), dim.dimtable, new CandidateTablePruneCause(
CandidateTablePruneCode.MORE_WEIGHT));
i.remove();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
index fc06f08..fba682d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/LightestFactResolver.java
@@ -23,17 +23,16 @@ import java.util.*;
import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Prune fact tables having more weight than minimum.
*/
+@Slf4j
public class LightestFactResolver implements ContextRewriter {
- public static final Log LOG = LogFactory.getLog(LightestFactResolver.class.getName());
-
public LightestFactResolver(Configuration conf) {
}
@@ -51,8 +50,8 @@ public class LightestFactResolver implements ContextRewriter {
for (Iterator<Set<CandidateFact>> i = cubeql.getCandidateFactSets().iterator(); i.hasNext();) {
Set<CandidateFact> facts = i.next();
if (factWeightMap.get(facts) > minWeight) {
- LOG.info("Not considering facts:" + facts + " from candidate fact tables as it has more fact weight:"
- + factWeightMap.get(facts) + " minimum:" + minWeight);
+ log.info("Not considering facts:{} from candidate fact tables as it has more fact weight:{} minimum:{}",
+ facts, factWeightMap.get(facts), minWeight);
i.remove();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
index 9a92bee..4d8cbf3 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MaxCoveringFactResolver.java
@@ -28,18 +28,18 @@ import org.apache.lens.cube.metadata.UpdatePeriod;
import org.apache.lens.cube.metadata.timeline.RangesPartitionTimeline;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import com.google.common.collect.Maps;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Prune candidate fact sets so that the facts except the ones that are covering maximum of range are pruned
*/
+@Slf4j
class MaxCoveringFactResolver implements ContextRewriter {
- public static final Log LOG = LogFactory.getLog(MaxCoveringFactResolver.class.getName());
private final boolean failOnPartialData;
public MaxCoveringFactResolver(Configuration conf) {
@@ -82,9 +82,8 @@ class MaxCoveringFactResolver implements ContextRewriter {
timeCoveredLong = 0L;
}
if (timeCoveredLong < maxTimeCovered) {
- LOG.info(
- "Not considering facts:" + facts + " from candidate fact tables as it covers less time than the max"
- + " for partition column: " + partColQueried + " which is: " + timeCovered);
+ log.info("Not considering facts:{} from candidate fact tables as it covers less time than the max"
+ + " for partition column: {} which is: {}", facts, partColQueried, timeCovered);
iter.remove();
}
}
@@ -115,7 +114,7 @@ class MaxCoveringFactResolver implements ContextRewriter {
try {
partitionRangesForPartitionColumns.add(part);
} catch (LensException e) {
- LOG.error("invalid partition: " + e);
+ log.error("invalid partition: ", e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
index 0228fb8..b5f5adc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/MultiFactHQLContext.java
@@ -22,8 +22,6 @@ import java.util.*;
import org.apache.lens.cube.metadata.Dimension;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -36,8 +34,6 @@ import com.google.common.collect.Lists;
*/
class MultiFactHQLContext extends SimpleHQLContext {
- public static final Log LOG = LogFactory.getLog(MultiFactHQLContext.class.getName());
-
private Map<Dimension, CandidateDim> dimsToQuery;
private Set<CandidateFact> facts;
private CubeQueryContext query;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
index 76f976a..f3f3f78 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SimpleHQLContext.java
@@ -23,19 +23,18 @@ import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Accepts strings of all expressions and constructs HQL query.
* <p></p>
* Making this as an abstract class because it provides constructors without all expressions being set.
*/
+@Slf4j
public abstract class SimpleHQLContext implements HQLContextInterface {
- public static final Log LOG = LogFactory.getLog(SimpleHQLContext.class.getName());
-
private String select;
private String from;
private String where;
@@ -79,8 +78,8 @@ public abstract class SimpleHQLContext implements HQLContextInterface {
setMissingExpressions();
String qfmt = getQueryFormat();
Object[] queryTreeStrings = getQueryTreeStrings();
- if (LOG.isDebugEnabled()) {
- LOG.debug("qfmt:" + qfmt + " Query strings: " + Arrays.toString(queryTreeStrings));
+ if (log.isDebugEnabled()) {
+ log.debug("qfmt: {} Query strings: {}", qfmt, Arrays.toString(queryTreeStrings));
}
String baseQuery = String.format(qfmt, queryTreeStrings);
return baseQuery;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
index 6f6d2e4..b63111b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactHQLContext.java
@@ -23,8 +23,6 @@ import java.util.Map;
import org.apache.lens.cube.metadata.Dimension;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -37,8 +35,6 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
*/
class SingleFactHQLContext extends DimOnlyHQLContext {
- public static final Log LOG = LogFactory.getLog(SingleFactHQLContext.class.getName());
-
private final CandidateFact fact;
private String storageAlias;
[33/50] [abbrv] incubator-lens git commit: LENS-682 : Make XML
marshalling errors descriptive on CLI
Posted by sh...@apache.org.
LENS-682 : Make XML marshalling errors descriptive on CLI
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/4c725266
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/4c725266
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/4c725266
Branch: refs/heads/LENS-581
Commit: 4c7252669fb07f5b38e6a6653cc26830ccfe6329
Parents: 44ec2af
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jul 24 12:56:18 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jul 24 12:56:18 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/api/APIResult.java | 48 ++++-
.../apache/lens/client/LensMetadataClient.java | 193 +++++--------------
2 files changed, 90 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4c725266/lens-api/src/main/java/org/apache/lens/api/APIResult.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/APIResult.java b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
index d048373..6b734a9 100644
--- a/lens-api/src/main/java/org/apache/lens/api/APIResult.java
+++ b/lens-api/src/main/java/org/apache/lens/api/APIResult.java
@@ -23,10 +23,7 @@ import java.io.StringWriter;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Marshaller;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.*;
import lombok.AccessLevel;
import lombok.Getter;
@@ -43,6 +40,7 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor(access = AccessLevel.PROTECTED)
public class APIResult {
+ private static final APIResult SUCCESS = new APIResult(Status.SUCCEEDED, "");
/**
* The status.
*/
@@ -117,4 +115,46 @@ public class APIResult {
return e.getMessage();
}
}
+
+ public static APIResult partial(int actual, int expected) {
+ return new APIResult(Status.PARTIAL, actual + " out of " + expected);
+ }
+
+ public static APIResult successOrPartialOrFailure(int actual, int expected) {
+ return successOrPartialOrFailure(actual, expected, null);
+ }
+
+ public static APIResult successOrPartialOrFailure(int actual, int expected, Exception e) {
+ if (actual == 0 && expected != 0) {
+ return failure(e);
+ }
+ if (actual < expected) {
+ return partial(actual, expected);
+ } else {
+ return success();
+ }
+ }
+
+ public static APIResult success() {
+ return SUCCESS;
+ }
+
+ public static APIResult failure(Exception e) {
+ String cause = extractCause(e);
+ return new APIResult(Status.FAILED, cause);
+ }
+
+ public static APIResult partial(Exception e) {
+ String cause = extractCause(e);
+ return new APIResult(Status.PARTIAL, cause);
+ }
+
+ private static String extractCause(Throwable e) {
+ String cause = null;
+ while ((cause == null || cause.isEmpty()) && e != null) {
+ cause = e.getMessage();
+ e = e.getCause();
+ }
+ return cause;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4c725266/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index 4bec6ad..31666a0 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -24,7 +24,6 @@ import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.nio.charset.Charset;
import java.util.Date;
import java.util.List;
@@ -46,8 +45,6 @@ import org.glassfish.jersey.media.multipart.*;
import org.xml.sax.SAXException;
import com.google.common.base.Joiner;
-import com.google.common.io.Files;
-
import lombok.extern.slf4j.Slf4j;
@Slf4j
@@ -204,12 +201,8 @@ public class LensMetadataClient {
public APIResult createCube(String cubeSpec) {
try {
return createCube(this.<XCube>readFromXML(cubeSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -225,12 +218,8 @@ public class LensMetadataClient {
public APIResult updateCube(String cubeName, String cubeSpec) {
try {
return updateCube(cubeName, this.<XCube>readFromXML(cubeSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -298,12 +287,8 @@ public class LensMetadataClient {
public APIResult createDimension(String dimSpec) {
try {
return createDimension(this.<XDimension>readFromXML(dimSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -319,12 +304,8 @@ public class LensMetadataClient {
public APIResult updateDimension(String dimName, String dimSpec) {
try {
return updateDimension(dimName, this.<XDimension>readFromXML(dimSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -367,12 +348,8 @@ public class LensMetadataClient {
public APIResult createNewStorage(String storage) {
try {
return createNewStorage(this.<XStorage>readFromXML(storage));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -397,12 +374,8 @@ public class LensMetadataClient {
public APIResult updateStorage(String storageName, String storage) {
try {
return updateStorage(storageName, this.<XStorage>readFromXML(storage));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -489,38 +462,11 @@ public class LensMetadataClient {
public APIResult createFactTable(String factSpec) {
try {
return createFactTable(this.<XFactTable>readFromXML(factSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
- }
- }
-
- private String getContent(String path) {
- try {
- List<String> content = null;
- if (path.startsWith("/")) {
- // path provided is absolute path
- content = Files.readLines(new File(path), Charset.defaultCharset());
- } else {
- // load from classpath
- URI uri = Thread.currentThread().getContextClassLoader().getResource(path).toURI();
- content = Files.readLines(new File(uri),
- Charset.defaultCharset());
- }
- return Joiner.on("\n").skipNulls().join(content);
- } catch (IOException e) {
- log.error("IO Error reading content ", e);
- throw new IllegalStateException(e);
- } catch (URISyntaxException e) {
- log.error("URI Error reading content ", e);
- throw new IllegalStateException(e);
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
-
public APIResult updateFactTable(String factName, XFactTable table) {
WebTarget target = getMetastoreWebTarget();
APIResult result = target.path("facts").path(factName)
@@ -533,12 +479,8 @@ public class LensMetadataClient {
public APIResult updateFactTable(String factName, String table) {
try {
return updateFactTable(factName, this.<XFactTable>readFromXML(table));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -586,12 +528,8 @@ public class LensMetadataClient {
public APIResult addStorageToFactTable(String factname, String storageSpec) {
try {
return addStorageToFactTable(factname, this.<XStorageTableElement>readFromXML(storageSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -703,12 +641,8 @@ public class LensMetadataClient {
public APIResult createDimensionTable(String tableXml) {
try {
return createDimensionTable(this.<XDimensionTable>readFromXML(tableXml));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -728,14 +662,9 @@ public class LensMetadataClient {
XDimensionTable dimensionTable = readFromXML(dimSpec);
dimensionTable.setTableName(dimTblName);
return updateDimensionTable(dimensionTable);
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
-
}
public APIResult dropDimensionTable(String table, boolean cascade) {
@@ -784,12 +713,8 @@ public class LensMetadataClient {
public APIResult addStorageToDimTable(String dimTblName, String table) {
try {
return addStorageToDimTable(dimTblName, this.<XStorageTableElement>readFromXML(table));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -885,12 +810,8 @@ public class LensMetadataClient {
String partitionSpec) {
try {
return addPartitionToDimensionTable(dimTblName, storage, (XPartition) readFromXML(partitionSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -909,12 +830,8 @@ public class LensMetadataClient {
String partitionsSpec) {
try {
return addPartitionsToDimensionTable(dimTblName, storage, (XPartitionList) readFromXML(partitionsSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -933,12 +850,8 @@ public class LensMetadataClient {
String partitionSpec) {
try {
return addPartitionToFactTable(fact, storage, (XPartition) readFromXML(partitionSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -957,14 +870,11 @@ public class LensMetadataClient {
String partitionsSpec) {
try {
return addPartitionsToFactTable(fact, storage, (XPartitionList) readFromXML(partitionsSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
+
public APIResult updatePartitionOfDimensionTable(String dimTblName, String storage,
XPartition partition) {
WebTarget target = getMetastoreWebTarget();
@@ -980,12 +890,8 @@ public class LensMetadataClient {
String partitionSpec) {
try {
return updatePartitionOfDimensionTable(dimTblName, storage, (XPartition) readFromXML(partitionSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -1004,12 +910,8 @@ public class LensMetadataClient {
String partitionsSpec) {
try {
return updatePartitionsOfDimensionTable(dimTblName, storage, (XPartitionList) readFromXML(partitionsSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -1028,12 +930,8 @@ public class LensMetadataClient {
String partitionSpec) {
try {
return updatePartitionOfFactTable(fact, storage, (XPartition) readFromXML(partitionSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -1052,12 +950,8 @@ public class LensMetadataClient {
String partitionsSpec) {
try {
return updatePartitionsOfFactTable(fact, storage, (XPartitionList) readFromXML(partitionsSpec));
- } catch (JAXBException e) {
- log.error("Unmarshalling error:", e);
- return new APIResult(Status.FAILED, "Unmarshalling failed");
- } catch (IOException e) {
- log.error("File error:", e);
- return new APIResult(Status.FAILED, "File not found");
+ } catch (JAXBException | IOException e) {
+ return failureAPIResult(e);
}
}
@@ -1079,4 +973,9 @@ public class LensMetadataClient {
.request(MediaType.APPLICATION_XML)
.get(StringList.class).getElements();
}
+
+ private APIResult failureAPIResult(Exception e) {
+ log.error("Failed", e);
+ return APIResult.failure(e);
+ }
}
[12/50] [abbrv] incubator-lens git commit: LENS-24 : Add support for
User Query Rewriter chain before Cube Rewrites
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 59066f9..7e486ae 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -73,7 +73,6 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import org.glassfish.jersey.test.TestProperties;
-import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@@ -210,7 +209,7 @@ public class TestQueryService extends LensJerseyTest {
final WebTarget target = target().path("queryapi/queries");
Response rs = target.path("random").queryParam("sessionid", lensSessionId).request().get();
- Assert.assertEquals(rs.getStatus(), 400);
+ assertEquals(rs.getStatus(), 400);
}
/**
@@ -236,7 +235,7 @@ public class TestQueryService extends LensJerseyTest {
new GenericType<LensAPIResult<QueryHandle>>() {
}).getData();
- Assert.assertNotNull(handle);
+ assertNotNull(handle);
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
@@ -253,8 +252,8 @@ public class TestQueryService extends LensJerseyTest {
assertEquals(ctx.getDriverStartTime(), 0);
assertEquals(ctx.getDriverFinishTime(), 0);
assertTrue(ctx.getFinishTime() > 0);
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
- Assert.assertTrue(metricsSvc.getTotalFailedQueries() >= failedQueries + 1);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
+ assertTrue(metricsSvc.getTotalFailedQueries() >= failedQueries + 1);
}
// test with execute async post, get all queries, get query context,
@@ -287,13 +286,13 @@ public class TestQueryService extends LensJerseyTest {
new GenericType<LensAPIResult<QueryHandle>>() {
}).getData();
- Assert.assertNotNull(handle);
+ assertNotNull(handle);
// Get all queries
// XML
List<QueryHandle> allQueriesXML = target.queryParam("sessionid", lensSessionId).request(MediaType.APPLICATION_XML)
.get(new GenericType<List<QueryHandle>>() {});
- Assert.assertTrue(allQueriesXML.size() >= 1);
+ assertTrue(allQueriesXML.size() >= 1);
// JSON
// List<QueryHandle> allQueriesJSON = target.request(
@@ -304,8 +303,8 @@ public class TestQueryService extends LensJerseyTest {
List<QueryHandle> allQueries = (List<QueryHandle>) target.queryParam("sessionid", lensSessionId).request()
.get(new GenericType<List<QueryHandle>>() {
});
- Assert.assertTrue(allQueries.size() >= 1);
- Assert.assertTrue(allQueries.contains(handle));
+ assertTrue(allQueries.size() >= 1);
+ assertTrue(allQueries.contains(handle));
// Get query
// Invocation.Builder builderjson = target.path(handle.toString()).request(MediaType.APPLICATION_JSON);
@@ -316,7 +315,7 @@ public class TestQueryService extends LensJerseyTest {
System.out.println("query XML:" + queryXML);
Response response = target.path(handle.toString() + "001").queryParam("sessionid", lensSessionId).request().get();
- Assert.assertEquals(response.getStatus(), 404);
+ assertEquals(response.getStatus(), 404);
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
@@ -340,7 +339,7 @@ public class TestQueryService extends LensJerseyTest {
}
assertTrue(ctx.getSubmissionTime() > 0);
assertTrue(ctx.getFinishTime() > 0);
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
// Update conf for query
final FormDataMultiPart confpart = new FormDataMultiPart();
@@ -352,7 +351,7 @@ public class TestQueryService extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
APIResult updateConf = target.path(handle.toString()).request()
.put(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
- Assert.assertEquals(updateConf.getStatus(), APIResult.Status.FAILED);
+ assertEquals(updateConf.getStatus(), APIResult.Status.FAILED);
}
// Test explain query
@@ -377,9 +376,9 @@ public class TestQueryService extends LensJerseyTest {
final QueryPlan plan = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryPlan>>() {}).getData();
- Assert.assertEquals(plan.getTablesQueried().size(), 1);
- Assert.assertTrue(plan.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase()));
- Assert.assertNull(plan.getPrepareHandle());
+ assertEquals(plan.getTablesQueried().size(), 1);
+ assertTrue(plan.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase()));
+ assertNull(plan.getPrepareHandle());
// Test explain and prepare
final WebTarget ptarget = target().path("queryapi/preparedqueries");
@@ -395,9 +394,9 @@ public class TestQueryService extends LensJerseyTest {
final QueryPlan plan2 = ptarget.request().post(Entity.entity(mp2, MediaType.MULTIPART_FORM_DATA_TYPE),
QueryPlan.class);
- Assert.assertEquals(plan2.getTablesQueried().size(), 1);
- Assert.assertTrue(plan2.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase()));
- Assert.assertNotNull(plan2.getPrepareHandle());
+ assertEquals(plan2.getTablesQueried().size(), 1);
+ assertTrue(plan2.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase()));
+ assertNotNull(plan2.getPrepareHandle());
}
// Test explain failure
@@ -423,10 +422,10 @@ public class TestQueryService extends LensJerseyTest {
final QueryPlan plan = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryPlan>>() {}).getData();
- Assert.assertTrue(plan.isError());
- Assert.assertNotNull(plan.getErrorMsg());
- Assert.assertTrue(plan.getErrorMsg()
- .contains("Invalid table alias or column reference 'NO_ID': " + "(possible column names are: id, idstr)"));
+ assertTrue(plan.isError());
+ assertNotNull(plan.getErrorMsg());
+ assertTrue(plan.getErrorMsg()
+ .contains("Invalid table alias or column reference 'NO_ID': " + "(possible column names are: id, idstr)"));
// Test explain and prepare
final WebTarget ptarget = target().path("queryapi/preparedqueries");
@@ -442,10 +441,10 @@ public class TestQueryService extends LensJerseyTest {
final QueryPlan plan2 = ptarget.request().post(Entity.entity(mp2, MediaType.MULTIPART_FORM_DATA_TYPE),
QueryPlan.class);
- Assert.assertTrue(plan2.isError());
- Assert.assertNotNull(plan2.getErrorMsg());
- Assert.assertNull(plan2.getPrepareHandle());
- Assert.assertTrue(plan2.getErrorMsg().contains("Invalid table alias or column reference 'NO_ID': "
+ assertTrue(plan2.isError());
+ assertNotNull(plan2.getErrorMsg());
+ assertNull(plan2.getPrepareHandle());
+ assertTrue(plan2.getErrorMsg().contains("Invalid table alias or column reference 'NO_ID': "
+ "(possible column names are: id, idstr)"));
}
@@ -482,16 +481,16 @@ public class TestQueryService extends LensJerseyTest {
List<QueryPrepareHandle> allQueries = (List<QueryPrepareHandle>) target.queryParam("sessionid", lensSessionId)
.queryParam("queryName", "testQuery1").request().get(new GenericType<List<QueryPrepareHandle>>() {
});
- Assert.assertTrue(allQueries.size() >= 1);
- Assert.assertTrue(allQueries.contains(pHandle));
+ assertTrue(allQueries.size() >= 1);
+ assertTrue(allQueries.contains(pHandle));
LensPreparedQuery ctx = target.path(pHandle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensPreparedQuery.class);
- Assert.assertTrue(ctx.getUserQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
- Assert.assertTrue(ctx.getDriverQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
- Assert.assertEquals(ctx.getSelectedDriverClassName(),
+ assertTrue(ctx.getUserQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
+ assertTrue(ctx.getDriverQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
+ assertEquals(ctx.getSelectedDriverClassName(),
org.apache.lens.driver.hive.HiveDriver.class.getCanonicalName());
- Assert.assertNull(ctx.getConf().getProperties().get("my.property"));
+ assertNull(ctx.getConf().getProperties().get("my.property"));
// Update conf for prepared query
final FormDataMultiPart confpart = new FormDataMultiPart();
@@ -503,10 +502,10 @@ public class TestQueryService extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
APIResult updateConf = target.path(pHandle.toString()).request()
.put(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
- Assert.assertEquals(updateConf.getStatus(), APIResult.Status.SUCCEEDED);
+ assertEquals(updateConf.getStatus(), APIResult.Status.SUCCEEDED);
ctx = target.path(pHandle.toString()).queryParam("sessionid", lensSessionId).request().get(LensPreparedQuery.class);
- Assert.assertEquals(ctx.getConf().getProperties().get("my.property"), "myvalue");
+ assertEquals(ctx.getConf().getProperties().get("my.property"), "myvalue");
QueryHandle handle1 = target.path(pHandle.toString()).request()
.post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class);
@@ -516,11 +515,11 @@ public class TestQueryService extends LensJerseyTest {
// do post once again
QueryHandle handle2 = target.path(pHandle.toString()).request()
.post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class);
- Assert.assertNotEquals(handle1, handle2);
+ assertNotEquals(handle1, handle2);
LensQuery ctx1 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId)
.request().get(LensQuery.class);
- Assert.assertEquals(ctx1.getQueryName().toLowerCase(), "testquery1");
+ assertEquals(ctx1.getQueryName().toLowerCase(), "testquery1");
// wait till the query finishes
QueryStatus stat = ctx1.getStatus();
while (!stat.finished()) {
@@ -529,12 +528,12 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx1.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
LensQuery ctx2 = target().path("queryapi/queries").path(handle2.toString()).queryParam("sessionid", lensSessionId)
.request().get(LensQuery.class);
- Assert.assertNotNull(ctx2);
- Assert.assertEquals(ctx2.getQueryName().toLowerCase(), "testqueryname2");
+ assertNotNull(ctx2);
+ assertEquals(ctx2.getQueryName().toLowerCase(), "testqueryname2");
// wait till the query finishes
stat = ctx2.getStatus();
while (!stat.finished()) {
@@ -543,17 +542,17 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx2.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
// destroy prepared
APIResult result = target.path(pHandle.toString()).queryParam("sessionid", lensSessionId).request()
.delete(APIResult.class);
- Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
// Post on destroyed query
Response response = target.path(pHandle.toString()).request()
.post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), Response.class);
- Assert.assertEquals(response.getStatus(), 404);
+ assertEquals(response.getStatus(), 404);
}
/**
@@ -575,17 +574,17 @@ public class TestQueryService extends LensJerseyTest {
final QueryPlan plan = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), QueryPlan.class);
- Assert.assertEquals(plan.getTablesQueried().size(), 1);
- Assert.assertTrue(plan.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase()));
- Assert.assertNotNull(plan.getPrepareHandle());
+ assertEquals(plan.getTablesQueried().size(), 1);
+ assertTrue(plan.getTablesQueried().get(0).endsWith(TEST_TABLE.toLowerCase()));
+ assertNotNull(plan.getPrepareHandle());
LensPreparedQuery ctx = target.path(plan.getPrepareHandle().toString()).queryParam("sessionid", lensSessionId)
.request().get(LensPreparedQuery.class);
- Assert.assertTrue(ctx.getUserQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
- Assert.assertTrue(ctx.getDriverQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
- Assert.assertEquals(ctx.getSelectedDriverClassName(),
- org.apache.lens.driver.hive.HiveDriver.class.getCanonicalName());
- Assert.assertNull(ctx.getConf().getProperties().get("my.property"));
+ assertTrue(ctx.getUserQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
+ assertTrue(ctx.getDriverQuery().equalsIgnoreCase("select ID from " + TEST_TABLE));
+ assertEquals(ctx.getSelectedDriverClassName(),
+ org.apache.lens.driver.hive.HiveDriver.class.getCanonicalName());
+ assertNull(ctx.getConf().getProperties().get("my.property"));
// Update conf for prepared query
final FormDataMultiPart confpart = new FormDataMultiPart();
@@ -597,11 +596,11 @@ public class TestQueryService extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
APIResult updateConf = target.path(plan.getPrepareHandle().toString()).request()
.put(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
- Assert.assertEquals(updateConf.getStatus(), APIResult.Status.SUCCEEDED);
+ assertEquals(updateConf.getStatus(), APIResult.Status.SUCCEEDED);
ctx = target.path(plan.getPrepareHandle().toString()).queryParam("sessionid", lensSessionId).request()
.get(LensPreparedQuery.class);
- Assert.assertEquals(ctx.getConf().getProperties().get("my.property"), "myvalue");
+ assertEquals(ctx.getConf().getProperties().get("my.property"), "myvalue");
QueryHandle handle1 = target.path(plan.getPrepareHandle().toString()).request()
.post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class);
@@ -609,7 +608,7 @@ public class TestQueryService extends LensJerseyTest {
// do post once again
QueryHandle handle2 = target.path(plan.getPrepareHandle().toString()).request()
.post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), QueryHandle.class);
- Assert.assertNotEquals(handle1, handle2);
+ assertNotEquals(handle1, handle2);
LensQuery ctx1 = target().path("queryapi/queries").path(handle1.toString()).queryParam("sessionid", lensSessionId)
.request().get(LensQuery.class);
@@ -621,7 +620,7 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx1.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
LensQuery ctx2 = target().path("queryapi/queries").path(handle2.toString()).queryParam("sessionid", lensSessionId)
.request().get(LensQuery.class);
@@ -633,17 +632,17 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx2.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx1.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
// destroy prepared
APIResult result = target.path(plan.getPrepareHandle().toString()).queryParam("sessionid", lensSessionId).request()
.delete(APIResult.class);
- Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
// Post on destroyed query
Response response = target.path(plan.getPrepareHandle().toString()).request()
.post(Entity.entity(confpart, MediaType.MULTIPART_FORM_DATA_TYPE), Response.class);
- Assert.assertEquals(response.getStatus(), 404);
+ assertEquals(response.getStatus(), 404);
}
@@ -675,24 +674,25 @@ public class TestQueryService extends LensJerseyTest {
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- Assert.assertNotNull(handle);
+ assertNotNull(handle);
// Get query
- LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
+ LensQuery lensQuery = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
- Assert.assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
- || ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING)
- || ctx.getStatus().getStatus().equals(Status.SUCCESSFUL));
+ assertTrue(lensQuery.getStatus().getStatus().equals(Status.QUEUED)
+ || lensQuery.getStatus().getStatus().equals(Status.LAUNCHED)
+ || lensQuery.getStatus().getStatus().equals(Status.RUNNING)
+ || lensQuery.getStatus().getStatus().equals(Status.SUCCESSFUL));
// wait till the query finishes
- QueryStatus stat = ctx.getStatus();
+ QueryStatus stat = lensQuery.getStatus();
while (!stat.finished()) {
- ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
- stat = ctx.getStatus();
+ lensQuery = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
+ stat = lensQuery.getStatus();
switch (stat.getStatus()) {
case RUNNING:
assertEquals(metricsSvc.getRunningQueries(), runningQueries + 1,
- "Asserting queries for " + ctx.getQueryHandle());
+ "Asserting queries for " + lensQuery.getQueryHandle());
break;
case QUEUED:
assertEquals(metricsSvc.getQueuedQueries(), queuedQueries + 1);
@@ -701,12 +701,15 @@ public class TestQueryService extends LensJerseyTest {
}
Thread.sleep(1000);
}
- assertTrue(ctx.getSubmissionTime() > 0);
- assertTrue(ctx.getLaunchTime() > 0);
- assertTrue(ctx.getDriverStartTime() > 0);
- assertTrue(ctx.getDriverFinishTime() > 0);
- assertTrue(ctx.getFinishTime() > 0);
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertTrue(lensQuery.getSubmissionTime() > 0);
+ assertTrue(lensQuery.getLaunchTime() > 0);
+ assertTrue(lensQuery.getDriverStartTime() > 0);
+ assertTrue(lensQuery.getDriverFinishTime() > 0);
+ assertTrue(lensQuery.getFinishTime() > 0);
+ QueryContext ctx = queryService.getQueryContext(lensQuery.getQueryHandle());
+ assertNotNull(ctx.getPhase1RewrittenQuery());
+ assertEquals(ctx.getPhase1RewrittenQuery(), ctx.getUserQuery()); //Since there is no rewriter in this test
+ assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
validatePersistedResult(handle, target(), lensSessionId, new String[][]{{"ID", "INT"}, {"IDSTR", "STRING"}}, true);
@@ -714,20 +717,20 @@ public class TestQueryService extends LensJerseyTest {
final QueryHandle handle2 = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- Assert.assertNotNull(handle2);
+ assertNotNull(handle2);
APIResult result = target.path(handle2.toString()).queryParam("sessionid", lensSessionId).request()
.delete(APIResult.class);
// cancel would fail query is already successful
LensQuery ctx2 = target.path(handle2.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
if (result.getStatus().equals(APIResult.Status.FAILED)) {
- Assert.assertEquals(ctx2.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL,
+ assertEquals(ctx2.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL,
"cancel failed without query having been succeeded");
} else if (result.getStatus().equals(APIResult.Status.SUCCEEDED)) {
- Assert.assertEquals(ctx2.getStatus().getStatus(), QueryStatus.Status.CANCELED,
+ assertEquals(ctx2.getStatus().getStatus(), QueryStatus.Status.CANCELED,
"cancel succeeded but query wasn't cancelled");
} else {
- Assert.fail("unexpected cancel status: " + result.getStatus());
+ fail("unexpected cancel status: " + result.getStatus());
}
// Test http download end point
@@ -747,12 +750,12 @@ public class TestQueryService extends LensJerseyTest {
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
// Get query
- ctx = target.path(handle3.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
+ lensQuery = target.path(handle3.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
// wait till the query finishes
- stat = ctx.getStatus();
+ stat = lensQuery.getStatus();
while (!stat.finished()) {
- ctx = target.path(handle3.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
- stat = ctx.getStatus();
+ lensQuery = target.path(handle3.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
+ stat = lensQuery.getStatus();
Thread.sleep(1000);
}
validateHttpEndPoint(target(), null, handle3, null);
@@ -799,17 +802,17 @@ public class TestQueryService extends LensJerseyTest {
*/
public static List<String> readResultSet(PersistentQueryResult resultset, QueryHandle handle, boolean isDir)
throws IOException {
- Assert.assertTrue(resultset.getPersistedURI().contains(handle.toString()));
+ assertTrue(resultset.getPersistedURI().contains(handle.toString()));
Path actualPath = new Path(resultset.getPersistedURI());
FileSystem fs = actualPath.getFileSystem(new Configuration());
List<String> actualRows = new ArrayList<String>();
if (fs.getFileStatus(actualPath).isDir()) {
- Assert.assertTrue(isDir);
+ assertTrue(isDir);
for (FileStatus fstat : fs.listStatus(actualPath)) {
addRowsFromFile(actualRows, fs, fstat.getPath());
}
} else {
- Assert.assertFalse(isDir);
+ assertFalse(isDir);
addRowsFromFile(actualRows, fs, actualPath);
}
return actualRows;
@@ -873,7 +876,7 @@ public class TestQueryService extends LensJerseyTest {
"5[][\"nothing\"]",
};
for (int i = 0; i < actualRows.size(); i++) {
- Assert.assertEquals(
+ assertEquals(
expected1[i].indexOf(actualRows.get(i)) == 0 || expected2[i].indexOf(actualRows.get(i)) == 0, true);
}
}
@@ -893,7 +896,7 @@ public class TestQueryService extends LensJerseyTest {
Response response = parent.path("queryapi/queries/" + handle.toString() + "/httpresultset")
.queryParam("sessionid", lensSessionId).request().get();
- Assert.assertTrue(response.getHeaderString("content-disposition").contains(handle.toString()));
+ assertTrue(response.getHeaderString("content-disposition").contains(handle.toString()));
if (redirectUrl == null) {
InputStream in = (InputStream) response.getEntity();
@@ -906,8 +909,8 @@ public class TestQueryService extends LensJerseyTest {
List<String> actualRows = Arrays.asList(result.split("\n"));
validatePersistentResult(actualRows);
} else {
- Assert.assertEquals(Response.Status.SEE_OTHER.getStatusCode(), response.getStatus());
- Assert.assertTrue(response.getHeaderString("Location").contains(redirectUrl));
+ assertEquals(Response.Status.SEE_OTHER.getStatusCode(), response.getStatus());
+ assertTrue(response.getHeaderString("Location").contains(redirectUrl));
}
}
@@ -923,9 +926,9 @@ public class TestQueryService extends LensJerseyTest {
Response response = parent.path("queryapi/queries/" + handle.toString() + "/httpresultset")
.queryParam("sessionid", lensSessionId).request().get();
if (Response.Status.NOT_FOUND.getStatusCode() != response.getStatus()) {
- Assert.fail("Expected not found excepiton, but got:" + response.getStatus());
+ fail("Expected not found excepiton, but got:" + response.getStatus());
}
- Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
+ assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
} catch (NotFoundException e) {
// expected
log.error("Resource not found.", e);
@@ -960,12 +963,12 @@ public class TestQueryService extends LensJerseyTest {
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- Assert.assertNotNull(handle);
+ assertNotNull(handle);
// Get query
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
- Assert.assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
+ assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
|| ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING)
|| ctx.getStatus().getStatus().equals(Status.SUCCESSFUL));
@@ -976,7 +979,7 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
// fetch results
validateResultSetMetadata(handle, "",
@@ -1015,12 +1018,12 @@ public class TestQueryService extends LensJerseyTest {
final QueryHandle dropHandle = target.request().post(Entity.entity(drop, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- Assert.assertNotNull(dropHandle);
+ assertNotNull(dropHandle);
// Get query
LensQuery ctx = target.path(dropHandle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
- Assert.assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
+ assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
|| ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING)
|| ctx.getStatus().getStatus().equals(Status.SUCCESSFUL));
@@ -1031,7 +1034,7 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
final FormDataMultiPart mp = new FormDataMultiPart();
conf = new LensConf();
@@ -1046,11 +1049,11 @@ public class TestQueryService extends LensJerseyTest {
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- Assert.assertNotNull(handle);
+ assertNotNull(handle);
// Get query
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
- Assert.assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
+ assertTrue(ctx.getStatus().getStatus().equals(Status.QUEUED)
|| ctx.getStatus().getStatus().equals(Status.LAUNCHED) || ctx.getStatus().getStatus().equals(Status.RUNNING)
|| ctx.getStatus().getStatus().equals(Status.SUCCESSFUL));
@@ -1061,7 +1064,7 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
String select = "SELECT * FROM temp_output";
final FormDataMultiPart fetch = new FormDataMultiPart();
@@ -1074,7 +1077,7 @@ public class TestQueryService extends LensJerseyTest {
final QueryHandle handle2 = target.request().post(Entity.entity(fetch, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- Assert.assertNotNull(handle2);
+ assertNotNull(handle2);
// Get query
ctx = target.path(handle2.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
@@ -1086,7 +1089,7 @@ public class TestQueryService extends LensJerseyTest {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL);
// fetch results
validateResultSetMetadata(handle2, "temp_output.", new String[][]{{"ID", "INT"}, {"IDSTR", "STRING"}},
@@ -1124,7 +1127,7 @@ public class TestQueryService extends LensJerseyTest {
QueryResultSetMetadata metadata = target.path(handle.toString()).path("resultsetmetadata")
.queryParam("sessionid", lensSessionId).request().get(QueryResultSetMetadata.class);
- Assert.assertEquals(metadata.getColumns().size(), columns.length);
+ assertEquals(metadata.getColumns().size(), columns.length);
for (int i = 0; i < columns.length; i++) {
assertTrue(
metadata.getColumns().get(i).getName().toLowerCase().equals(outputTablePfx + columns[i][0].toLowerCase())
@@ -1140,20 +1143,20 @@ public class TestQueryService extends LensJerseyTest {
* @param resultset the resultset
*/
private void validateInmemoryResult(InMemoryQueryResult resultset) {
- Assert.assertEquals(resultset.getRows().size(), 5);
- Assert.assertEquals(resultset.getRows().get(0).getValues().get(0), 1);
- Assert.assertEquals((String) resultset.getRows().get(0).getValues().get(1), "one");
+ assertEquals(resultset.getRows().size(), 5);
+ assertEquals(resultset.getRows().get(0).getValues().get(0), 1);
+ assertEquals((String) resultset.getRows().get(0).getValues().get(1), "one");
- Assert.assertNull(resultset.getRows().get(1).getValues().get(0));
- Assert.assertEquals((String) resultset.getRows().get(1).getValues().get(1), "two");
+ assertNull(resultset.getRows().get(1).getValues().get(0));
+ assertEquals((String) resultset.getRows().get(1).getValues().get(1), "two");
- Assert.assertEquals(resultset.getRows().get(2).getValues().get(0), 3);
- Assert.assertNull(resultset.getRows().get(2).getValues().get(1));
+ assertEquals(resultset.getRows().get(2).getValues().get(0), 3);
+ assertNull(resultset.getRows().get(2).getValues().get(1));
- Assert.assertNull(resultset.getRows().get(3).getValues().get(0));
- Assert.assertNull(resultset.getRows().get(3).getValues().get(1));
- Assert.assertEquals(resultset.getRows().get(4).getValues().get(0), 5);
- Assert.assertEquals(resultset.getRows().get(4).getValues().get(1), "");
+ assertNull(resultset.getRows().get(3).getValues().get(0));
+ assertNull(resultset.getRows().get(3).getValues().get(1));
+ assertEquals(resultset.getRows().get(4).getValues().get(0), 5);
+ assertEquals(resultset.getRows().get(4).getValues().get(1), "");
}
// test execute with timeout, fetch results
@@ -1182,8 +1185,8 @@ public class TestQueryService extends LensJerseyTest {
QueryHandleWithResultSet result = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
- Assert.assertNotNull(result.getQueryHandle());
- Assert.assertNotNull(result.getResult());
+ assertNotNull(result.getQueryHandle());
+ assertNotNull(result.getResult());
validatePersistentResult((PersistentQueryResult) result.getResult(), result.getQueryHandle(), true);
final FormDataMultiPart mp2 = new FormDataMultiPart();
@@ -1201,8 +1204,8 @@ public class TestQueryService extends LensJerseyTest {
result = target.request().post(Entity.entity(mp2, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
- Assert.assertNotNull(result.getQueryHandle());
- Assert.assertNotNull(result.getResult());
+ assertNotNull(result.getQueryHandle());
+ assertNotNull(result.getResult());
validateInmemoryResult((InMemoryQueryResult) result.getResult());
}
@@ -1228,9 +1231,9 @@ public class TestQueryService extends LensJerseyTest {
QueryHandleWithResultSet result = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
- Assert.assertNotNull(result.getQueryHandle());
- Assert.assertNull(result.getResult());
- Assert.assertEquals(result.getStatus().getStatus(), Status.FAILED);
+ assertNotNull(result.getQueryHandle());
+ assertNull(result.getResult());
+ assertEquals(result.getStatus().getStatus(), Status.FAILED);
}
/**
@@ -1245,23 +1248,23 @@ public class TestQueryService extends LensJerseyTest {
Configuration conf = queryService.getLensConf(lensSessionId, queryConf);
// session specific conf
- Assert.assertEquals(conf.get("test.session.key"), "svalue");
+ assertEquals(conf.get("test.session.key"), "svalue");
// query specific conf
- Assert.assertEquals(conf.get("test.query.conf"), "qvalue");
+ assertEquals(conf.get("test.query.conf"), "qvalue");
// lenssession default should be loaded
- Assert.assertNotNull(conf.get("lens.query.enable.persistent.resultset"));
+ assertNotNull(conf.get("lens.query.enable.persistent.resultset"));
// lens site should be loaded
- Assert.assertEquals(conf.get("test.lens.site.key"), "gsvalue");
+ assertEquals(conf.get("test.lens.site.key"), "gsvalue");
// hive default variables should not be set
- Assert.assertNull(conf.get("hive.exec.local.scratchdir"));
+ assertNull(conf.get("hive.exec.local.scratchdir"));
// hive site variables should not be set
- Assert.assertNull(conf.get("hive.metastore.warehouse.dir"));
+ assertNull(conf.get("hive.metastore.warehouse.dir"));
// core default should not be loaded
- Assert.assertNull(conf.get("fs.default.name"));
+ assertNull(conf.get("fs.default.name"));
// Test server config. Hive configs overriden should be set
- Assert.assertFalse(Boolean.parseBoolean(queryService.getHiveConf().get("hive.server2.log.redirection.enabled")));
- Assert.assertEquals(queryService.getHiveConf().get("hive.server2.query.log.dir"), "target/query-logs");
+ assertFalse(Boolean.parseBoolean(queryService.getHiveConf().get("hive.server2.log.redirection.enabled")));
+ assertEquals(queryService.getHiveConf().get("hive.server2.query.log.dir"), "target/query-logs");
final String query = "select ID from " + TEST_TABLE;
QueryContext ctx = new QueryContext(query, null, queryConf, conf, queryService.getDrivers());
@@ -1272,27 +1275,27 @@ public class TestQueryService extends LensJerseyTest {
ctx.setDriverQueries(driverQueries);
// This still holds since current database is default
- Assert.assertEquals(queryService.getSession(lensSessionId).getCurrentDatabase(), "default");
- Assert.assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader(), ctx.getConf()
+ assertEquals(queryService.getSession(lensSessionId).getCurrentDatabase(), "default");
+ assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader(), ctx.getConf()
.getClassLoader());
- Assert.assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader(),
+ assertEquals(queryService.getSession(lensSessionId).getHiveConf().getClassLoader(),
ctx.getDriverContext().getDriverConf(queryService.getDrivers().iterator().next()).getClassLoader());
- Assert.assertTrue(ctx.isDriverQueryExplicitlySet());
+ assertTrue(ctx.isDriverQueryExplicitlySet());
for (LensDriver driver : queryService.getDrivers()) {
Configuration dconf = ctx.getDriverConf(driver);
- Assert.assertEquals(dconf.get("test.session.key"), "svalue");
+ assertEquals(dconf.get("test.session.key"), "svalue");
// query specific conf
- Assert.assertEquals(dconf.get("test.query.conf"), "qvalue");
+ assertEquals(dconf.get("test.query.conf"), "qvalue");
// lenssession default should be loaded
- Assert.assertNotNull(dconf.get("lens.query.enable.persistent.resultset"));
+ assertNotNull(dconf.get("lens.query.enable.persistent.resultset"));
// lens site should be loaded
- Assert.assertEquals(dconf.get("test.lens.site.key"), "gsvalue");
+ assertEquals(dconf.get("test.lens.site.key"), "gsvalue");
// hive default variables should not be set
- Assert.assertNull(conf.get("hive.exec.local.scratchdir"));
+ assertNull(conf.get("hive.exec.local.scratchdir"));
// driver site should be loaded
- Assert.assertEquals(dconf.get("lens.driver.test.key"), "set");
+ assertEquals(dconf.get("lens.driver.test.key"), "set");
// core default should not be loaded
- Assert.assertNull(dconf.get("fs.default.name"));
+ assertNull(dconf.get("fs.default.name"));
}
}
@@ -1317,9 +1320,9 @@ public class TestQueryService extends LensJerseyTest {
final QueryCostTO result = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryCostTO>>() {}).getData();
- Assert.assertNotNull(result);
- Assert.assertEquals(result.getEstimatedExecTimeMillis(), null);
- Assert.assertEquals(result.getEstimatedResourceUsage(), Double.MAX_VALUE);
+ assertNotNull(result);
+ assertEquals(result.getEstimatedExecTimeMillis(), null);
+ assertEquals(result.getEstimatedResourceUsage(), Double.MAX_VALUE);
}
@@ -1344,8 +1347,8 @@ public class TestQueryService extends LensJerseyTest {
LensSessionImpl session = sessionService.getSession(sessionHandle);
// Jars should be pending until query is run
- Assert.assertEquals(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS).size(), 1);
- Assert.assertEquals(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS_2).size(), 1);
+ assertEquals(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS).size(), 1);
+ assertEquals(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS_2).size(), 1);
final String tableInDBWithJars = "testHiveDriverGetsDBJars";
try {
@@ -1362,7 +1365,7 @@ public class TestQueryService extends LensJerseyTest {
LensTestUtil.DB_WITH_JARS);
}
}
- Assert.assertTrue(addedToHiveDriver);
+ assertTrue(addedToHiveDriver);
// Switch database
log.info("@@@# database switch test");
@@ -1371,18 +1374,18 @@ public class TestQueryService extends LensJerseyTest {
+ "ROW FORMAT SERDE \"DatabaseJarSerde\"");
// All db jars should have been added
- Assert.assertTrue(session.getDBResources(LensTestUtil.DB_WITH_JARS_2).isEmpty());
- Assert.assertTrue(session.getDBResources(LensTestUtil.DB_WITH_JARS).isEmpty());
+ assertTrue(session.getDBResources(LensTestUtil.DB_WITH_JARS_2).isEmpty());
+ assertTrue(session.getDBResources(LensTestUtil.DB_WITH_JARS).isEmpty());
// All session resources must have been added to both DBs
- Assert.assertFalse(session.getLensSessionPersistInfo().getResources().isEmpty());
+ assertFalse(session.getLensSessionPersistInfo().getResources().isEmpty());
for (LensSessionImpl.ResourceEntry resource : session.getLensSessionPersistInfo().getResources()) {
- Assert.assertTrue(resource.isAddedToDatabase(LensTestUtil.DB_WITH_JARS_2));
- Assert.assertTrue(resource.isAddedToDatabase(LensTestUtil.DB_WITH_JARS));
+ assertTrue(resource.isAddedToDatabase(LensTestUtil.DB_WITH_JARS_2));
+ assertTrue(resource.isAddedToDatabase(LensTestUtil.DB_WITH_JARS));
}
- Assert.assertTrue(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS).isEmpty());
- Assert.assertTrue(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS_2).isEmpty());
+ assertTrue(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS).isEmpty());
+ assertTrue(session.getPendingSessionResourcesForDatabase(LensTestUtil.DB_WITH_JARS_2).isEmpty());
} finally {
log.info("@@@ TEST_OVER");
@@ -1432,11 +1435,11 @@ public class TestQueryService extends LensJerseyTest {
Map<LensDriver, AbstractQueryContext.DriverEstimateRunnable> estimateRunnables = ctx.getDriverEstimateRunnables();
for (LensDriver driver : estimateRunnables.keySet()) {
estimateRunnables.get(driver).run();
- Assert.assertFalse(estimateRunnables.get(driver).isSucceeded(), driver + " estimate should have been skipped");
+ assertFalse(estimateRunnables.get(driver).isSucceeded(), driver + " estimate should have been skipped");
}
for (LensDriver driver : queryService.getDrivers()) {
- Assert.assertNull(ctx.getDriverQueryCost(driver));
+ assertNull(ctx.getDriverQueryCost(driver));
}
}
@@ -1473,11 +1476,11 @@ public class TestQueryService extends LensJerseyTest {
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryCostTO>>() {
}).getData();
- Assert.assertNotNull(queryCostTO);
+ assertNotNull(queryCostTO);
MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
- Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
+ assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-DRIVER_SELECTION",
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-CUBE_REWRITE",
"lens.MethodMetricGauge.TestQueryService-testEstimateGauges-HiveDriver-DRIVER_ESTIMATE",
@@ -1502,6 +1505,6 @@ public class TestQueryService extends LensJerseyTest {
MediaType.APPLICATION_XML_TYPE));
Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE));
- Assert.assertEquals(response.getStatus(), 400);
+ assertEquals(response.getStatus(), 400);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
new file mode 100644
index 0000000..9628865
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestRewriting.java
@@ -0,0 +1,461 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.rewrite;
+
+import java.util.*;
+
+import org.apache.lens.api.LensConf;
+import org.apache.lens.cube.parse.CubeQueryContext;
+import org.apache.lens.cube.parse.CubeQueryRewriter;
+import org.apache.lens.cube.parse.HQLParser;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.driver.MockDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.metrics.LensMetricsRegistry;
+import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.parse.*;
+
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.testng.PowerMockObjectFactory;
+import org.testng.Assert;
+import org.testng.IObjectFactory;
+import org.testng.annotations.ObjectFactory;
+import org.testng.annotations.Test;
+
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * The Class TestRewriting.
+ */
+@PrepareForTest(RewriteUtil.class)
+@PowerMockIgnore({"org.apache.log4j.*", "javax.management.*", "javax.xml.*",
+ "com.sun.org.apache.xerces.internal.jaxp.*", "ch.qos.logback.*", "org.slf4j.*", "org.w3c.dom*"})
+public class TestRewriting {
+ /**
+ * We need a special {@link IObjectFactory}.
+ *
+ * @return {@link PowerMockObjectFactory}.
+ */
+ @ObjectFactory
+ public IObjectFactory getObjectFactory() {
+ return new PowerMockObjectFactory();
+ }
+
+ private HiveConf hconf = new HiveConf();
+
+ static int i = 0;
+ // number of successful queries through mock rewriter
+ // we use this number to mock failures after successful queries
+ // change the number, if more tests for success needs to be added
+ static final int NUM_SUCCESS = 36;
+
+ private CubeQueryRewriter getMockedRewriter() throws SemanticException, ParseException, LensException {
+ CubeQueryRewriter mockwriter = Mockito.mock(CubeQueryRewriter.class);
+ Mockito.when(mockwriter.rewrite(Matchers.any(String.class))).thenAnswer(new Answer<CubeQueryContext>() {
+ @Override
+ public CubeQueryContext answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ i++;
+ // return query for first NUM_SUCCESS calls and fail later
+ if (i <= NUM_SUCCESS) {
+ return getMockedCubeContext((String) args[0]);
+ } else {
+ throw new RuntimeException("Mock fail");
+ }
+ }
+ });
+ Mockito.when(mockwriter.rewrite(Matchers.any(ASTNode.class))).thenAnswer(new Answer<CubeQueryContext>() {
+ @Override
+ public CubeQueryContext answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return getMockedCubeContext((ASTNode) args[0]);
+ }
+ });
+ return mockwriter;
+ }
+
+ /**
+ * Gets the mocked cube context.
+ *
+ * @param query the query
+ * @return the mocked cube context
+ * @throws SemanticException the semantic exception
+ * @throws ParseException the parse exception
+ */
+ private CubeQueryContext getMockedCubeContext(String query)
+ throws SemanticException, ParseException, LensException {
+ CubeQueryContext context = Mockito.mock(CubeQueryContext.class);
+ Mockito.when(context.toHQL()).thenReturn(query.substring(4));
+ Mockito.when(context.toAST(Matchers.any(Context.class))).thenReturn(HQLParser.parseHQL(query.substring(4), hconf));
+ return context;
+ }
+
+ /**
+ * Gets the mocked cube context.
+ *
+ * @param ast the ast
+ * @return the mocked cube context
+ * @throws SemanticException the semantic exception
+ * @throws ParseException the parse exception
+ */
+ private CubeQueryContext getMockedCubeContext(ASTNode ast) throws SemanticException, ParseException {
+ CubeQueryContext context = Mockito.mock(CubeQueryContext.class);
+ if (ast.getToken().getType() == HiveParser.TOK_QUERY) {
+ if (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) {
+ // remove cube child from AST
+ for (int i = 0; i < ast.getChildCount() - 1; i++) {
+ ast.setChild(i, ast.getChild(i + 1));
+ }
+ ast.deleteChild(ast.getChildCount() - 1);
+ }
+ }
+ StringBuilder builder = new StringBuilder();
+ HQLParser.toInfixString(ast, builder);
+ Mockito.when(context.toHQL()).thenReturn(builder.toString());
+ Mockito.when(context.toAST(Matchers.any(Context.class))).thenReturn(ast);
+ return context;
+ }
+
+ private void runRewrites(Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnableMap) {
+ for (LensDriver driver : runnableMap.keySet()) {
+ RewriteUtil.DriverRewriterRunnable r = runnableMap.get(driver);
+ r.run();
+ Assert.assertTrue(r.getDriver() == driver);
+
+ // Failure cause should be set only when rewrite fails
+ if (r.isSucceeded()) {
+ Assert.assertNull(r.getFailureCause(), driver
+ + " succeeded but failure cause is set to " + r.getFailureCause());
+ Assert.assertNotNull(r.getRewrittenQuery(), driver + " succeeded but rewritten query is not set");
+ } else {
+ Assert.assertNotNull(r.getFailureCause(), driver + " failed but failure cause is not set");
+ Assert.assertNull(r.getRewrittenQuery(),
+ driver + " failed but rewritten query is set to " + r.getRewrittenQuery());
+ }
+
+ }
+ }
+
+ /**
+ * Test cube query.
+ *
+ * @throws ParseException the parse exception
+ * @throws SemanticException the semantic exception
+ * @throws LensException the lens exception
+ */
+ @Test
+ public void testCubeQuery() throws ParseException, SemanticException, LensException {
+ List<LensDriver> drivers = new ArrayList<LensDriver>();
+ MockDriver driver = new MockDriver();
+ LensConf lensConf = new LensConf();
+ Configuration conf = new Configuration();
+ driver.configure(conf);
+ drivers.add(driver);
+
+ CubeQueryRewriter mockWriter = getMockedRewriter();
+ PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getCubeRewriter")).toReturn(mockWriter);
+ String q1 = "select name from table";
+ Assert.assertFalse(RewriteUtil.isCubeQuery(q1));
+ List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(q1, hconf);
+ Assert.assertEquals(cubeQueries.size(), 0);
+ QueryContext ctx = new QueryContext(q1, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName());
+ driver.configure(conf);
+ String q2 = "cube select name from table";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
+
+ Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
+ "lens.MethodMetricGauge.TestRewriting-MockDriver-RewriteUtil-rewriteQuery",
+ "lens.MethodMetricGauge.TestRewriting-MockDriver-1-RewriteUtil-rewriteQuery-toHQL")));
+ conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
+
+ q2 = "insert overwrite directory '/tmp/rewrite' cube select name from table";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "insert overwrite local directory '/tmp/rewrite' cube select name from table";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "insert overwrite local directory '/tmp/example-output' cube select id,name from dim_table";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select id,name from dim_table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "explain cube select name from table";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from (cube select name from table) a";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "insert overwrite directory '/tmp/rewrite' select * from (cube select name from table) a";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from (cube select name from table)a";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from ( cube select name from table ) a";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from ( cube select name from table where"
+ + " (name = 'ABC'||name = 'XYZ')&&(key=100) ) a";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from"
+ + " table where (name = 'ABC' OR name = 'XYZ') AND (key=100)");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName() + "-multiple");
+ q2 = "select * from (cube select name from table) a join (cube select" + " name2 from table2) b";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ reg = LensMetricsRegistry.getStaticRegistry();
+ Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
+ "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-1-RewriteUtil-rewriteQuery-toHQL",
+ "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-2-RewriteUtil-rewriteQuery-toHQL",
+ "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-RewriteUtil-rewriteQuery")));
+ conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
+
+ q2 = "select * from (cube select name from table) a full outer join"
+ + " (cube select name2 from table2) b on a.name=b.name2";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from (cube select name from table) a join (select name2 from table2) b";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from (cube select name from table union all cube select name2 from table2) u";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+
+ q2 = "insert overwrite directory '/tmp/rewrite' "
+ + "select * from (cube select name from table union all cube select name2 from table2) u";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+
+ q2 = "select u.* from (select name from table union all cube select name2 from table2) u";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select u.* from (select name from table union all cube select name2 from table2)u";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from (cube select name from table union all cube select name2"
+ + " from table2 union all cube select name3 from table3) u";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 3);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3");
+
+ q2 = "select * from ( cube select name from table union all cube"
+ + " select name2 from table2 union all cube select name3 from table3 ) u";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 3);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3");
+
+ q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+
+ q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+
+ q2 = "create table temp1 as cube select name from table";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+
+ q2 = "create table temp1 as select * from (cube select name from table union all cube select"
+ + " name2 from table2) u group by u.name";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 2);
+ Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
+ Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
+
+ q2 = "create table temp1 as cube select name from table where"
+ + " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')";
+ Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
+ cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ runRewrites(RewriteUtil.rewriteQuery(ctx));
+ Assert.assertEquals(cubeQueries.size(), 1);
+ Assert.assertEquals(cubeQueries.get(0).query,
+ "cube select name from table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')");
+
+ // failing query for second driver
+ MockDriver driver2 = new MockDriver();
+ driver2.configure(conf);
+ drivers.add(driver2);
+
+ Assert.assertEquals(drivers.size(), 2);
+ Assert.assertTrue(drivers.contains(driver) && drivers.contains(driver2));
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ Map<LensDriver, RewriteUtil.DriverRewriterRunnable> dQueries = RewriteUtil.rewriteQuery(ctx);
+ Assert.assertEquals(dQueries.size(), 2);
+
+ // Check both driver runnables are present
+ Assert.assertNotNull(dQueries.get(driver), driver + " runnable not found");
+ Assert.assertEquals(dQueries.get(driver).getDriver(), driver);
+
+ Assert.assertNotNull(dQueries.get(driver2), driver2 + " runnable not found");
+ Assert.assertEquals(dQueries.get(driver2).getDriver(), driver2);
+
+ runRewrites(dQueries);
+
+ // We have to verify that query failed for the second driver
+ // First driver passes
+ Iterator<LensDriver> itr = dQueries.keySet().iterator();
+ itr.next();
+ LensDriver failedDriver = itr.next();
+ Assert.assertFalse(dQueries.get(failedDriver).isSucceeded(), failedDriver + " rewrite should have failed");
+ Assert.assertNull(dQueries.get(failedDriver).getRewrittenQuery(),
+ failedDriver + " rewritten query should not be set");
+ // running again will fail on both drivers
+ ctx = new QueryContext(q2, null, lensConf, conf, drivers);
+ Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnables = RewriteUtil.rewriteQuery(ctx);
+ runRewrites(runnables);
+
+ Assert.assertFalse(runnables.get(driver).isSucceeded());
+ Assert.assertNotNull(runnables.get(driver).getFailureCause());
+ Assert.assertNull(runnables.get(driver).getRewrittenQuery());
+ Assert.assertNotNull(ctx.getDriverRewriteError(driver));
+
+ Assert.assertFalse(runnables.get(driver2).isSucceeded());
+ Assert.assertNotNull(runnables.get(driver2).getFailureCause());
+ Assert.assertNull(runnables.get(driver2).getRewrittenQuery());
+ Assert.assertNotNull(ctx.getDriverRewriteError(driver2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/src/test/java/org/apache/lens/server/rewrite/TestUserQueryToCubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/rewrite/TestUserQueryToCubeQueryRewriter.java b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestUserQueryToCubeQueryRewriter.java
new file mode 100644
index 0000000..38f8d71
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/rewrite/TestUserQueryToCubeQueryRewriter.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.rewrite;
+
+import static org.testng.Assert.assertEquals;
+
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.rewrite.Phase1Rewriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+public class TestUserQueryToCubeQueryRewriter {
+ public static class Rewriter1 implements Phase1Rewriter {
+ public static final String PREFIX = "a";
+ public static final String SUFFIX = "b";
+
+ @Override
+ public String rewrite(String query, Configuration queryConf, HiveConf metastoreConf) throws LensException {
+ return PREFIX + query + SUFFIX;
+ }
+
+ @Override
+ public void init(Configuration rewriteConf) {
+
+ }
+ }
+
+ public static class Rewriter2 implements Phase1Rewriter {
+ public static final String PREFIX = "c";
+ public static final String SUFFIX = "d";
+
+ @Override
+ public String rewrite(String query, Configuration queryConf, HiveConf metastoreConf) throws LensException {
+ return PREFIX + query + SUFFIX;
+ }
+
+ @Override
+ public void init(Configuration rewriteConf) {
+
+ }
+ }
+
+ UserQueryToCubeQueryRewriter rewriter;
+
+ @BeforeTest
+ public void setup() throws LensException {
+ Configuration conf = new Configuration();
+ conf.set(LensConfConstants.QUERY_PHASE1_REWRITERS, Rewriter1.class.getName() + "," + Rewriter2.class.getName());
+ rewriter = new UserQueryToCubeQueryRewriter(conf);
+ }
+
+ @Test
+ public void testRewriter() throws LensException {
+ String query = "user query";
+ assertEquals(rewriter.rewriteToCubeQuery(query, null, null),
+ Rewriter2.PREFIX + Rewriter1.PREFIX + query + Rewriter1.SUFFIX + Rewriter2.SUFFIX);
+ }
+}
[41/50] [abbrv] incubator-lens git commit: LENS-16 : Add command to
show current sessionId from lens-cli
Posted by sh...@apache.org.
LENS-16 : Add command to show current sessionId from lens-cli
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/3ce889fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/3ce889fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/3ce889fc
Branch: refs/heads/LENS-581
Commit: 3ce889fcb4c350b4c306c6b84a15b2b21fbba2aa
Parents: 211ef3b
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Jul 27 16:57:28 2015 +0530
Committer: arshad-matin <ar...@apache.org>
Committed: Mon Jul 27 16:57:28 2015 +0530
----------------------------------------------------------------------
.../cli/commands/LensConnectionCommands.java | 33 ++++++++++++++------
.../lens/cli/TestLensConnectionCliCommands.java | 21 +++++++++++--
src/site/apt/user/cli.apt | 2 ++
3 files changed, 43 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3ce889fc/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
index 8b4face..c059f30 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensConnectionCommands.java
@@ -24,8 +24,8 @@ import java.util.List;
import javax.ws.rs.ProcessingException;
import org.apache.lens.api.APIResult;
+import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
-
import org.apache.lens.client.LensClient;
import org.apache.log4j.*;
@@ -36,7 +36,6 @@ import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
import com.google.common.base.Joiner;
-
import lombok.extern.slf4j.Slf4j;
@@ -158,14 +157,28 @@ public class LensConnectionCommands extends BaseLensCommand {
}
/**
+ * Get the current session handle
+ */
+ @CliCommand(value = {"session"}, help = "Print the current session handle")
+ public String getSessionHandle() {
+ LensSessionHandle sessionHandle = getClient().getConnection().getSessionHandle();
+ if (sessionHandle != null) {
+ return "Session Handle: " + sessionHandle.getPublicId();
+ } else {
+ return "Session not established";
+ }
+ }
+
+ /**
* Enables to show all class level logs on cli
* @param enable
*/
- @CliCommand(value = {"debug"}, help = "prints all class level logs and verbose logs on cli for debugging purpose."
- + " 'debug false' to turn off all class level logging and verbose level logging ")
+ @CliCommand(value = {"debug"},
+ help = "prints all class level logs and verbose logs on cli for debugging purpose."
+ + " 'debug false' to turn off all class level logging and verbose level logging ")
public void debug(@CliOption(key = {"", "enable"},
- mandatory = false, unspecifiedDefaultValue = "true",
- help = "To print all logs on cli for debugging purpose") boolean enable) {
+ mandatory = false, unspecifiedDefaultValue = "true",
+ help = "To print all logs on cli for debugging purpose") boolean enable) {
Logger logger = LoggerUtil.getRootLogger();
Logger cliLogger = LoggerUtil.getCliLogger();
if (enable) {
@@ -181,11 +194,11 @@ public class LensConnectionCommands extends BaseLensCommand {
* Enables to show only cliLogger(verbose) logs on cli
* @param enable
*/
- @CliCommand(value = {"verbose"}, help = "Show cliLogger logs on cli"
- + "'verbose false' turns off the cliLogger logs on console")
+ @CliCommand(value = {"verbose"},
+ help = "Show cliLogger logs on cli. 'verbose false' turns off the cliLogger logs on console")
public void verbose(@CliOption(key = {"", "enable"},
- mandatory = false, unspecifiedDefaultValue = "true",
- help = "Print the clilogger logs on cli") boolean enable) {
+ mandatory = false, unspecifiedDefaultValue = "true",
+ help = "Print the clilogger logs on cli") boolean enable) {
Logger cliLogger = LoggerUtil.getCliLogger();
if (enable) {
LoggerUtil.addConsoleAppenderIfNotPresent(cliLogger);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3ce889fc/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
index 7503221..ae04b3f 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensConnectionCliCommands.java
@@ -24,6 +24,7 @@ import java.net.URI;
import javax.ws.rs.BadRequestException;
+import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.cli.commands.LensConnectionCommands;
import org.apache.lens.client.LensClient;
@@ -89,10 +90,10 @@ public class TestLensConnectionCliCommands extends LensCliApplicationTest {
return f;
}
- private String getFilePathFromUri(String uripath){
+ private String getFilePathFromUri(String uripath) {
try {
return new URI(uripath).getPath();
- } catch (Exception e){
+ } catch (Exception e) {
return null;
}
}
@@ -172,7 +173,7 @@ public class TestLensConnectionCliCommands extends LensCliApplicationTest {
/* Tests input file pattern file: and file:// */
String filenameA = "file:" + projectdir + "/target/tempdata_a.txt";
- String filenameB = "file://" + projectdir +"/target/tempdata_b.txt";
+ String filenameB = "file://" + projectdir + "/target/tempdata_b.txt";
String fileRegex = "file:" + projectdir + "/target/tempdata_*.txt";
@@ -327,4 +328,18 @@ public class TestLensConnectionCliCommands extends LensCliApplicationTest {
commands.quitShell();
}
}
+
+ /**
+ * Test CLI command to get session handle
+ */
+ @Test
+ public void testGetSessionHandle() {
+ LensClient client = new LensClient();
+ LensConnectionCommands commands = new LensConnectionCommands();
+ commands.setClient(client);
+ LensSessionHandle sessionHandle = client.getConnection().getSessionHandle();
+ Assert.assertNotNull(sessionHandle);
+ String output = commands.getSessionHandle();
+ Assert.assertTrue(output.contains(sessionHandle.getPublicId().toString()), "session handle output: " + output);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/3ce889fc/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index 51257e0..a28d8a4 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -102,6 +102,8 @@ User CLI Commands
*--+--+
|remove jar [--path] \<path-to-jar-on-server-side\>|Removes a jar resource from session|
*--+--+
+|session|Print the current session handle|
+*--+--+
|set \<key\>=\<value\>|Assign <<<value>>> to session parameter specified with <<<key>>> on lens server|
*--+--+
|show params|Fetches and prints all session parameter from lens server|
[30/50] [abbrv] incubator-lens git commit: LENS-619: Applying Query
Launching Constraints before allowing a query to be launched (Also contains
LENS-687 and LENS-688)
Posted by sh...@apache.org.
LENS-619: Applying Query Launching Constraints before allowing a query to be launched (Also contains LENS-687 and LENS-688)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c879f991
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c879f991
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c879f991
Branch: refs/heads/LENS-581
Commit: c879f991102f5f02c30c1f1cd428f4c603f2e0ec
Parents: 90415a1
Author: Himanshu Gahlaut <ga...@gmail.com>
Authored: Wed Jul 22 14:51:27 2015 +0530
Committer: Himanshu Gahlaut <hi...@apache.org>
Committed: Wed Jul 22 16:40:43 2015 +0530
----------------------------------------------------------------------
.../main/java/org/apache/lens/api/Addable.java | 32 --
.../org/apache/lens/api/query/LensQuery.java | 4 +
.../org/apache/lens/api/query/QueryStatus.java | 14 +-
.../apache/lens/cli/TestLensQueryCommands.java | 19 +-
.../org/apache/lens/client/LensStatement.java | 6 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 22 ++
.../src/main/resources/hivedriver-default.xml | 18 +
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 31 +-
.../driver/jdbc/JDBCDriverConfConstants.java | 6 +
.../src/main/resources/jdbcdriver-default.xml | 25 ++
lens-server-api/pom.xml | 11 +
.../lens/server/api/LensConfConstants.java | 18 +
.../ConfigBasedObjectCreationFactory.java | 33 ++
.../lens/server/api/driver/LensDriver.java | 23 +-
.../server/api/driver/MinQueryCostSelector.java | 27 +-
.../lens/server/api/metrics/MetricsService.java | 7 +
.../server/api/query/AbstractQueryContext.java | 20 +-
.../api/query/DriverSelectorQueryContext.java | 38 +-
.../server/api/query/FinishedLensQuery.java | 11 +
.../lens/server/api/query/QueryContext.java | 44 ++-
.../server/api/query/QueryExecutionService.java | 7 +
...erSpecificWaitingQueriesSelectionPolicy.java | 42 +++
...ficWaitingQueriesSelectionPolicyFactory.java | 33 ++
.../EstimatedImmutableQueryCollection.java | 65 ++++
.../query/collect/ImmutableQueryCollection.java | 53 +++
.../collect/WaitingQueriesSelectionPolicy.java | 39 ++
.../MaxConcurrentDriverQueriesConstraint.java | 47 +++
...oncurrentDriverQueriesConstraintFactory.java | 39 ++
.../constraint/QueryLaunchingConstraint.java | 34 ++
.../query/cost/FactPartitionBasedQueryCost.java | 5 +-
.../lens/server/api/query/cost/QueryCost.java | 6 +-
.../apache/lens/server/api/util/LensUtil.java | 36 ++
.../lens/server/api/driver/MockDriver.java | 15 +
...ecificWaitingQueriesSelectionPolicyTest.java | 51 +++
...axConcurrentDriverQueriesConstraintTest.java | 57 +++
.../cost/TestFactPartitionBasedQueryCost.java | 14 +-
lens-server/pom.xml | 8 +-
.../lens/server/metrics/MetricsServiceImpl.java | 18 +-
.../server/query/QueryExecutionServiceImpl.java | 377 +++++++++++++++----
.../DefaultEstimatedQueryCollection.java | 169 +++++++++
.../query/collect/DefaultQueryCollection.java | 104 +++++
.../query/collect/EstimatedQueryCollection.java | 30 ++
.../IntersectingWaitingQueriesSelector.java | 107 ++++++
.../query/collect/MutableQueryCollection.java | 65 ++++
.../server/query/collect/QueryCollection.java | 31 ++
.../ThreadSafeEstimatedQueryCollection.java | 91 +++++
.../collect/ThreadSafeQueryCollection.java | 75 ++++
...erSpecificWaitingQueriesSelectionPolicy.java | 44 +++
...ficWaitingQueriesSelectionPolicyFactory.java | 34 ++
.../query/collect/WaitingQueriesSelector.java | 43 +++
...DefaultQueryLaunchingConstraintsChecker.java | 68 ++++
.../QueryLaunchingConstraintsChecker.java | 33 ++
.../TotalQueryCostCeilingConstraint.java | 74 ++++
.../TotalQueryCostCeilingConstraintFactory.java | 47 +++
.../server/util/FairPriorityBlockingQueue.java | 107 ++++++
.../apache/lens/server/util/UtilityMethods.java | 1 +
.../src/main/resources/lensserver-default.xml | 37 ++
.../org/apache/lens/server/LensTestUtil.java | 20 +-
.../lens/server/query/TestQueryService.java | 8 +-
.../DefaultEstimatedQueryCollectionTest.java | 127 +++++++
.../collect/DefaultQueryCollectionTest.java | 107 ++++++
.../IntersectingWaitingQueriesSelectorTest.java | 185 +++++++++
.../server/query/collect/QueryCollectUtil.java | 102 +++++
.../server/query/collect/ThreadSafetyTest.java | 37 ++
...ecificWaitingQueriesSelectionPolicyTest.java | 52 +++
...ultQueryLaunchingConstraintsCheckerTest.java | 149 ++++++++
...alQueryCostCeilingConstraintFactoryTest.java | 56 +++
.../TotalQueryCostCeilingConstraintTest.java | 75 ++++
lens-server/src/test/resources/lens-site.xml | 1 -
lens-server/src/test/resources/log4j.properties | 2 +-
src/site/apt/admin/config.apt | 100 ++---
src/site/apt/admin/hivedriver-config.apt | 4 +
src/site/apt/admin/jdbcdriver-config.apt | 58 +--
73 files changed, 3306 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-api/src/main/java/org/apache/lens/api/Addable.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/Addable.java b/lens-api/src/main/java/org/apache/lens/api/Addable.java
deleted file mode 100644
index 5a55691..0000000
--- a/lens-api/src/main/java/org/apache/lens/api/Addable.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- *
- */
-package org.apache.lens.api;
-
-public interface Addable<T extends Addable<T>> {
- /**
- * Add another object of same type object of the same type.
- * Implement this if add operation makes sense for the class
- * @param other
- * @return new T object after logical addition
- */
- T add(T other);
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
index f594133..af439ff 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
@@ -220,4 +220,8 @@ public class LensQuery {
public String getQueryHandleString() {
return (this.queryHandle != null) ? this.queryHandle.getHandleIdString() : null;
}
+
+ public boolean queued() {
+ return this.status.queued();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index 539f0b0..9614caa 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -186,6 +186,18 @@ public class QueryStatus implements Serializable {
return status.equals(Status.SUCCESSFUL) || status.equals(Status.FAILED) || status.equals(Status.CANCELED);
}
+ public boolean launched() {
+ return status.equals(Status.LAUNCHED);
+ }
+
+ public boolean running() {
+ return status.equals(Status.RUNNING);
+ }
+
+ public boolean queued() {
+ return status.equals(Status.QUEUED);
+ }
+
/**
* Checks if is valid transition.
*
@@ -254,7 +266,7 @@ public class QueryStatus implements Serializable {
* @param newState the new state
* @return true, if is validate transition
*/
- public boolean isValidateTransition(Status newState) {
+ public boolean isValidTransition(final Status newState) {
return isValidTransition(this.status, newState);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index 46e28a9..319077a 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -32,6 +32,7 @@ import javax.xml.datatype.DatatypeFactory;
import org.apache.lens.api.APIResult;
import org.apache.lens.api.metastore.*;
import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.cli.commands.LensCubeCommands;
import org.apache.lens.cli.commands.LensQueryCommands;
import org.apache.lens.client.LensClient;
@@ -234,19 +235,23 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String[] resultSplits = result.split("\n");
// assert on the number of queries
assertEquals(String.valueOf(resultSplits.length - 1), resultSplits[resultSplits.length - 1].split(": ")[1]);
- String details = qCom.getDetails(qh);
- assertTrue(details.contains("driverQuery"), details);
+
+ QueryStatus queryStatus = client.getQueryStatus(qh);
+ while (!queryStatus.finished()) {
+ if (queryStatus.launched()) {
+ String details = qCom.getDetails(qh);
+ assertTrue(details.contains("driverQuery"));
+ }
+ Thread.sleep(1000);
+ queryStatus = client.getQueryStatus(qh);
+ }
// Check that query name searching is 'ilike'
String result2 = qCom.getAllQueries("", "query", "all", -1, Long.MAX_VALUE);
assertTrue(result2.contains(qh), result2);
- while (!client.getQueryStatus(qh).finished()) {
- Thread.sleep(5000);
- }
-
assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL"));
- details = qCom.getDetails(qh);
+ String details = qCom.getDetails(qh);
assertTrue(details.contains("driverQuery"));
result = qCom.getQueryResults(qh, null, true);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
index f7305fb..1ecf145 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
@@ -195,7 +195,7 @@ public class LensStatement {
public void waitForQueryToComplete(QueryHandle handle) {
LensClient.getCliLooger().info("Query handle: " + handle);
query = getQuery(handle);
- while (isQueryInQUEUEDSate()) {
+ while (query.queued()) {
query = getQuery(handle);
LensClient.getCliLooger().debug("Query " + handle + " status: " + query.getStatus());
try {
@@ -222,10 +222,6 @@ public class LensStatement {
}
}
- private boolean isQueryInQUEUEDSate() {
- return query.getStatus().getStatus().equals(QueryStatus.Status.QUEUED);
- }
-
/**
* Gets the query web target.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 7c3e3e0..31c343a 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.driver.hive;
+import static org.apache.lens.server.api.util.LensUtil.getImplementations;
+
import java.io.*;
import java.util.*;
import java.util.concurrent.*;
@@ -38,6 +40,8 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.cost.QueryCostCalculator;
@@ -60,6 +64,8 @@ import org.apache.hive.service.cli.thrift.TSessionHandle;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
+import com.google.common.collect.ImmutableSet;
+import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/**
@@ -89,6 +95,12 @@ public class HiveDriver implements LensDriver {
public static final String HS2_PRIORITY_DEFAULT_RANGES = "VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW";
public static final String SESSION_KEY_DELIMITER = ".";
+ private static final String QUERY_LAUNCHIG_CONSTRAINT_FACTORIES_KEY
+ = "lens.driver.hive.query.launching.constraint.factories";
+
+ private static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY
+ = "lens.driver.hive.waiting.queries.selection.policy.factories";
+
/** The driver conf- which will merged with query conf */
private Configuration driverConf;
@@ -130,6 +142,9 @@ public class HiveDriver implements LensDriver {
boolean whetherCalculatePriority;
private DriverQueryHook queryHook;
+ @Getter
+ private ImmutableSet<QueryLaunchingConstraint> queryConstraints;
+ private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies;
private String sessionDbKey(String sessionHandle, String database) {
return sessionHandle + SESSION_KEY_DELIMITER + database;
@@ -349,6 +364,8 @@ public class HiveDriver implements LensDriver {
} catch (InstantiationException | IllegalAccessException e) {
throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
}
+ queryConstraints = getImplementations(QUERY_LAUNCHIG_CONSTRAINT_FACTORIES_KEY, driverConf);
+ selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, driverConf);
}
private QueryCost calculateQueryCost(AbstractQueryContext qctx) throws LensException {
@@ -765,6 +782,11 @@ public class HiveDriver implements LensDriver {
driverListeners.add(driverEventListener);
}
+ @Override
+ public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
+ return selectionPolicies;
+ }
+
protected CLIServiceClient getClient() throws LensException {
if (isEmbedded) {
if (embeddedConnection == null) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-driver-hive/src/main/resources/hivedriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml
index 300ce3f..f5fd3bb 100644
--- a/lens-driver-hive/src/main/resources/hivedriver-default.xml
+++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml
@@ -146,4 +146,22 @@
<value>org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator</value>
<description>Cost calculator class. By default calculating cost through fact partitions.</description>
</property>
+
+ <property>
+ <name>lens.driver.hive.query.launching.constraint.factories</name>
+ <value></value>
+ <description>Factories used to instantiate constraints enforced on queries by driver. A query will be launched
+ only if all constraints pass. Every Factory should be an implementation of
+ org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of
+ org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.</description>
+ </property>
+
+ <property>
+ <name>lens.driver.hive.waiting.queries.selection.policy.factories</name>
+ <value></value>
+ <description>Factories used to instantiate driver specific waiting queries selection policies. Every factory should
+ be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an
+ implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.</description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index e3a8799..777f73a 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -18,7 +18,12 @@
*/
package org.apache.lens.driver.jdbc;
+import static java.lang.Integer.parseInt;
+
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.*;
+import static org.apache.lens.server.api.util.LensUtil.getImplementations;
+
+import static com.google.common.base.Preconditions.checkState;
import java.io.IOException;
import java.io.ObjectInput;
@@ -40,9 +45,10 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.rewrite.QueryRewriter;
@@ -55,6 +61,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
+import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.NonNull;
import lombok.Setter;
@@ -93,6 +100,10 @@ public class JDBCDriver implements LensDriver {
private LogSegregationContext logSegregationContext;
private DriverQueryHook queryHook;
+ @Getter
+ private ImmutableSet<QueryLaunchingConstraint> queryConstraints;
+ private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies;
+
/**
* Data related to a query submitted to JDBCDriver.
*/
@@ -436,6 +447,13 @@ public class JDBCDriver implements LensDriver {
* @throws LensException the lens exception
*/
protected void init(Configuration conf) throws LensException {
+
+ final int maxPoolSize = parseInt(this.conf.get(JDBC_POOL_MAX_SIZE));
+ final int maxConcurrentQueries
+ = parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY));
+ checkState(maxPoolSize == maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:"
+ + maxConcurrentQueries);
+
queryContextMap = new ConcurrentHashMap<QueryHandle, JdbcQueryContext>();
asyncQueryPool = Executors.newCachedThreadPool(new ThreadFactory() {
@Override
@@ -456,6 +474,8 @@ public class JDBCDriver implements LensDriver {
throw new LensException(e);
}
this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
+ this.queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, this.conf);
+ this.selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, this.conf);
}
/**
@@ -1083,6 +1103,11 @@ public class JDBCDriver implements LensDriver {
}
+ @Override
+ public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
+ return this.selectionPolicies;
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
index ebcdaca..a26acde 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
@@ -95,4 +95,10 @@ public final class JDBCDriverConfConstants {
public static final String JDBC_FETCH_SIZE = JDBC_DRIVER_PFX + "fetch.size";
public static final int DEFAULT_JDBC_FETCH_SIZE = 1000;
+
+ public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = JDBC_DRIVER_PFX
+ + "query.launching.constraint.factories";
+
+ public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = JDBC_DRIVER_PFX
+ + "waiting.queries.selection.policy.factories";
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
index 99f6f20..e94c2f3 100644
--- a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
+++ b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml
@@ -207,4 +207,29 @@
<description>Flag to enable query syntactic and semantic validation using prepared statement.</description>
</property>
+ <property>
+ <name>lens.driver.jdbc.query.launching.constraint.factories</name>
+ <value>org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory</value>
+ <description>Factories used to instantiate constraints enforced on queries by driver. A query will be launched
+ only if all constraints pass. Every Factory should be an implementation of
+ org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of
+ org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.</description>
+ </property>
+
+ <property>
+ <name>lens.driver.jdbc.waiting.queries.selection.policy.factories</name>
+ <value>org.apache.lens.server.api.query.collect.DriverSpecificWaitingQueriesSelectionPolicyFactory</value>
+ <description>Factories used to instantiate driver specific waiting queries selection policies. Every factory should
+ be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an
+ implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.</description>
+ </property>
+
+ <property>
+ <name>driver.max.concurrent.launched.queries</name>
+ <value>15</value>
+ <description>Maximum queries which can be launched simultaneously on this driver. This should be equal to
+ lens.driver.jdbc.pool.max.size. This configuration value is only useful when MaxConcurrentDriverQueriesConstraint
+ is enabled by using org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory as
+ one of the factories in lens.driver.jdbc.query.constraint.factories property.</description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml
index b0b0fe0..6822976 100644
--- a/lens-server-api/pom.xml
+++ b/lens-server-api/pom.xml
@@ -66,5 +66,16 @@
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lens</groupId>
+ <artifactId>lens-api</artifactId>
+ <type>test-jar</type>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 2b3a480..4732f6d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -893,4 +893,22 @@ public final class LensConfConstants {
public static final int DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS = 60000; // 1 minute
public static final String QUERY_PHASE1_REWRITERS = SERVER_PFX + "query.phase1.rewriters";
+
+ /**
+ * Key to get the implementations of query constraint factories.
+ */
+ public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = SERVER_PFX
+ + "query.launching.constraint.factories";
+
+ /**
+ * Key to get the total query cost ceiling per user.
+ */
+ public static final String TOTAL_QUERY_COST_CEILING_PER_USER_KEY = SERVER_PFX
+ + "total.query.cost.ceiling.per.user";
+
+ /**
+ * Key to get the implementations of waiting queries selection policy factories.
+ */
+ public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = SERVER_PFX
+ + "waiting.queries.selection.policy.factories";
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/common/ConfigBasedObjectCreationFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/ConfigBasedObjectCreationFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/ConfigBasedObjectCreationFactory.java
new file mode 100644
index 0000000..5fbac34
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/common/ConfigBasedObjectCreationFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.common;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ *
+ * Uses {@link org.apache.hadoop.conf.Configuration} to create an instance of T
+ *
+ * @param <T> Type of the object created using {@link Configuration}
+ */
+public interface ConfigBasedObjectCreationFactory<T> {
+
+ T create(final Configuration conf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 52207d3..a5a60d7 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -24,13 +24,15 @@ import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.PreparedQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.hadoop.conf.Configuration;
+import com.google.common.collect.ImmutableSet;
+
/**
* The Interface LensDriver.
*/
@@ -182,4 +184,19 @@ public interface LensDriver extends Externalizable {
* @param driverEventListener the driver event listener
*/
void registerDriverEventListener(LensEventListener<DriverEvent> driverEventListener);
+
+ /**
+ *
+ * @return The {@link QueryLaunchingConstraint}s to be checked before launching a query on driver. If there are no
+ * driver level constraints, then an empty set is returned. null is never returned.
+ */
+ ImmutableSet<QueryLaunchingConstraint> getQueryConstraints();
+
+ /**
+ *
+ * @return The {@link WaitingQueriesSelectionPolicy}s to be used to select waiting queries eligible to be moved out
+ * of waiting state. If there are no driver level waiting query selection policies, then an empty set is returned.
+ * null is never returned.
+ */
+ ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
index a3fd6ac..8fdde1d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
@@ -18,8 +18,7 @@
*/
package org.apache.lens.server.api.driver;
-import java.util.Collections;
-import java.util.Comparator;
+import java.util.*;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
@@ -36,25 +35,17 @@ public class MinQueryCostSelector implements DriverSelector {
* @return the lens driver
*/
@Override
- public LensDriver select(final AbstractQueryContext ctx,
- final Configuration conf) {
- return Collections.min(ctx.getDriverContext().getDrivers(), new Comparator<LensDriver>() {
+ public LensDriver select(final AbstractQueryContext ctx, final Configuration conf) {
+
+ final Collection<LensDriver> drivers = ctx.getDriverContext().getDriversWithValidQueryCost();
+
+ return Collections.min(drivers, new Comparator<LensDriver>() {
@Override
public int compare(LensDriver d1, LensDriver d2) {
- return compareCosts(ctx.getDriverContext().getDriverQueryCost(d1), ctx
- .getDriverContext().getDriverQueryCost(d2));
+ final QueryCost c1 = ctx.getDriverContext().getDriverQueryCost(d1);
+ final QueryCost c2 = ctx.getDriverContext().getDriverQueryCost(d2);
+ return c1.compareTo(c2);
}
});
}
-
- int compareCosts(QueryCost c1, QueryCost c2) {
- if (c1 == null && c2 == null) {
- return 0;
- } else if (c1 == null && c2 != null) {
- return 1;
- } else if (c1 != null && c2 == null) {
- return -1;
- }
- return c1.compareTo(c2);
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
index 71128b7..72fe455 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
@@ -118,6 +118,11 @@ public interface MetricsService {
String RUNNING_QUERIES = "running-queries";
/**
+ * The Constant WAITING_QUERIES.
+ */
+ String WAITING_QUERIES = "waiting-queries";
+
+ /**
* The Constant FINISHED_QUERIES.
*/
String FINISHED_QUERIES = "finished-queries";
@@ -136,6 +141,8 @@ public interface MetricsService {
long getRunningQueries();
+ long getWaitingQueries();
+
long getFinishedQueries();
long getTotalAcceptedQueries();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 8046927..0c980a2 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -89,6 +89,13 @@ public abstract class AbstractQueryContext implements Serializable {
protected String selectedDriverQuery;
/**
+ * The selected Driver query cost
+ */
+ @Getter
+ @Setter
+ protected QueryCost selectedDriverQueryCost;
+
+ /**
* The submitted user.
*/
@Getter
@@ -370,19 +377,6 @@ public abstract class AbstractQueryContext implements Serializable {
}
/**
- * Get selected driver's cost
- *
- * @return QueryCostTO
- * @throws LensException
- */
- public QueryCost getSelectedDriverQueryCost() throws LensException {
- if (driverContext != null) {
- return driverContext.getSelectedDriverQueryCost();
- }
- return null;
- }
-
- /**
* Set exception during rewrite.
*
* @param driver
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 7a2da99..feac938 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -30,6 +30,7 @@ import org.apache.lens.server.api.util.LensUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import com.google.common.collect.Sets;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
@@ -238,32 +239,6 @@ public class DriverSelectorQueryContext {
return driverQueryCtxs.get(getSelectedDriver()).getDriverQueryPlan();
}
- /**
- * Return selected driver's query plan, but check for null conditions first.
- *
- * @return DriverQueryPlan of Selected Driver
- * @throws LensException
- */
- public QueryCost getSelectedDriverQueryCost() throws LensException {
- final Map<LensDriver, DriverQueryContext> driverQueryCtxs = getDriverQueryContextMap();
- if (driverQueryCtxs == null) {
- throw new LensException("No Driver query ctx. Check if re-write happened or not");
- }
- if (getSelectedDriver() == null) {
- throw new LensException("Selected Driver is NULL.");
- }
-
- if (driverQueryCtxs.get(getSelectedDriver()) == null) {
- throw new LensException("Could not find Driver Context for selected driver " + getSelectedDriver());
- }
-
- if (driverQueryCtxs.get(getSelectedDriver()).getDriverQueryCostEstimateError() != null) {
- throw new LensException("Driver Query Cost of the selected driver is null",
- driverQueryCtxs.get(getSelectedDriver()).getDriverQueryCostEstimateError());
- }
- return driverQueryCtxs.get(getSelectedDriver()).getDriverCost();
- }
-
public Configuration getSelectedDriverConf() {
return getSelectedDriver() == null ? null : driverQueryContextMap.get(getSelectedDriver()).getDriverSpecificConf();
}
@@ -290,6 +265,17 @@ public class DriverSelectorQueryContext {
return driverQueryContextMap.keySet();
}
+ public Collection<LensDriver> getDriversWithValidQueryCost() {
+
+ final Set<LensDriver> eligibleDrivers = Sets.newLinkedHashSet();
+ for (Map.Entry<LensDriver, DriverQueryContext> driverToDriverContext : this.driverQueryContextMap.entrySet()) {
+ if (driverToDriverContext.getValue().driverCost != null) {
+ eligibleDrivers.add(driverToDriverContext.getKey());
+ }
+ }
+ return Collections.unmodifiableCollection(eligibleDrivers);
+ }
+
public Collection<String> getDriverQueries() {
List<String> queries = new ArrayList<String>();
final Collection<DriverQueryContext> values = driverQueryContextMap.values();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index 6e7ab5d..0c78e01 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -25,9 +25,11 @@ import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
import org.apache.hadoop.conf.Configuration;
+import com.google.common.collect.ImmutableSet;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
@@ -156,6 +158,9 @@ public class FinishedLensQuery {
@Setter
private String queryName;
+ @Getter
+ private LensDriver selectedDriver;
+
/**
* Instantiates a new finished lens query.
*/
@@ -183,12 +188,14 @@ public class FinishedLensQuery {
if (ctx.getQueryName() != null) {
this.queryName = ctx.getQueryName().toLowerCase();
}
+ this.selectedDriver = ctx.getSelectedDriver();
}
public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) throws LensException {
QueryContext qctx = new QueryContext(userQuery, submitter, new LensConf(), conf, drivers, null, submissionTime,
false);
qctx.setQueryHandle(QueryHandle.fromString(handle));
+ qctx.setLaunchTime(this.startTime);
qctx.setEndTime(getEndTime());
qctx.setStatusSkippingTransitionTest(new QueryStatus(0.0, QueryStatus.Status.valueOf(getStatus()),
getErrorMessage() == null ? "" : getErrorMessage(), getResult() != null, null, null, null));
@@ -198,4 +205,8 @@ public class FinishedLensQuery {
qctx.setQueryName(getQueryName());
return qctx;
}
+
+ public ImmutableSet<WaitingQueriesSelectionPolicy> getDriverSelectionPolicies() {
+ return this.selectedDriver.getWaitingQuerySelectionPolicies();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index d72727e..86e3fa3 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -32,10 +32,13 @@ import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.DriverQueryStatus;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import lombok.Getter;
import lombok.Setter;
@@ -181,6 +184,7 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
.getDriverQueryContextMap().keySet(), prepared.getDriverContext().getSelectedDriver(), true);
setDriverContext(prepared.getDriverContext());
setSelectedDriverQuery(prepared.getSelectedDriverQuery());
+ setSelectedDriverQueryCost(prepared.getSelectedDriverQueryCost());
}
/**
@@ -337,15 +341,12 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
/*
* Introduced for Recovering finished query.
*/
- void setStatusSkippingTransitionTest(QueryStatus newStatus) throws LensException {
+ public void setStatusSkippingTransitionTest(final QueryStatus newStatus) throws LensException {
this.status = newStatus;
}
- public synchronized void setStatus(QueryStatus newStatus) throws LensException {
- if (!this.status.isValidateTransition(newStatus.getStatus())) {
- throw new LensException("Invalid state transition:[" + this.status.getStatus() + "->" + newStatus.getStatus()
- + "]");
- }
+ public synchronized void setStatus(final QueryStatus newStatus) throws LensException {
+ validateTransition(newStatus);
this.status = newStatus;
}
@@ -402,4 +403,35 @@ public class QueryContext extends AbstractQueryContext implements Comparable<Que
public String getQueryHandleString() {
return queryHandle.getHandleIdString();
}
+
+ public void validateTransition(final QueryStatus newStatus) throws LensException {
+ if (!this.status.isValidTransition(newStatus.getStatus())) {
+ throw new LensException("Invalid state transition:from[" + this.status.getStatus() + " to "
+ + newStatus.getStatus() + "]");
+ }
+ }
+
+ public boolean finished() {
+ return this.status.finished();
+ }
+
+ public boolean launched() {
+ return this.status.launched();
+ }
+
+ public boolean running() {
+ return this.status.running();
+ }
+
+ public boolean queued() {
+ return this.status.queued();
+ }
+
+ public ImmutableSet<QueryLaunchingConstraint> getSelectedDriverQueryConstraints() {
+ return getSelectedDriver().getQueryConstraints();
+ }
+
+ public ImmutableSet<WaitingQueriesSelectionPolicy> getSelectedDriverSelectionPolicies() {
+ return getSelectedDriver().getWaitingQuerySelectionPolicies();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
index 5d7163f..724e36a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecutionService.java
@@ -302,6 +302,13 @@ public interface QueryExecutionService {
long getRunningQueriesCount();
/**
+ * Get waiting queries count
+ *
+ * @return waiting queries count
+ */
+ long getWaitingQueriesCount();
+
+ /**
* Get finished queries count
*
* @return finished queries count
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicy.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicy.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicy.java
new file mode 100644
index 0000000..50c2785
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicy.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.QueryContext;
+
+public class DriverSpecificWaitingQueriesSelectionPolicy implements WaitingQueriesSelectionPolicy {
+
+ /**
+ *
+ * @param finishedQuery
+ * @param waitingQueries current waiting queries
+ * @return
+ */
+ @Override
+ public Set<QueryContext> selectQueries(final FinishedLensQuery finishedQuery,
+ final EstimatedImmutableQueryCollection waitingQueries) {
+
+ LensDriver selectedDriver = finishedQuery.getSelectedDriver();
+ return waitingQueries.getQueries(selectedDriver);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyFactory.java
new file mode 100644
index 0000000..eeb2c68
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query.collect;
+
+import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class DriverSpecificWaitingQueriesSelectionPolicyFactory
+ implements ConfigBasedObjectCreationFactory<WaitingQueriesSelectionPolicy> {
+
+ @Override
+ public WaitingQueriesSelectionPolicy create(Configuration conf) {
+ return new DriverSpecificWaitingQueriesSelectionPolicy();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/EstimatedImmutableQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/EstimatedImmutableQueryCollection.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/EstimatedImmutableQueryCollection.java
new file mode 100644
index 0000000..58f7d13
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/EstimatedImmutableQueryCollection.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+/**
+ *
+ * The implementations of this interface should make sure that add behaviours only accepts queries for which selected
+ * driver is not null and a valid selected driver query cost is present.
+ *
+ * IllegalStateException shall be thrown from add behaviours when selected driver is not set or a valid selected driver
+ * query cost is not present for the given query.
+ *
+ */
+public interface EstimatedImmutableQueryCollection extends ImmutableQueryCollection {
+
+ /**
+ *
+ * @param driver Driver for which queries have to be returned.
+ *
+ * @return A set of queries for which given driver is the selected driver. A new set is created and returned.
+ * Elements in the set are not cloned or copied. Multiple iterations over returned set are guaranteed to be in same
+ * order. If there are no queries, then an empty set is returned. null is never returned.
+ */
+ Set<QueryContext> getQueries(final LensDriver driver);
+
+ /**
+ *
+ * @param driver Driver for which count of queries have to be returned.
+ *
+ * @return count of queries of given driver.
+ */
+ int getQueriesCount(final LensDriver driver);
+
+ /**
+ *
+ * Get total query cost of all queries of given user
+ *
+ * @param user
+ * @return total query cost of all queries of user
+ */
+ QueryCost getTotalQueryCost(final String user);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/ImmutableQueryCollection.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/ImmutableQueryCollection.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/ImmutableQueryCollection.java
new file mode 100644
index 0000000..344ff10
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/ImmutableQueryCollection.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ *
+ * {@link ImmutableQueryCollection} interface defines immutable behaviours on queries existing in lens system.
+ *
+ */
+public interface ImmutableQueryCollection {
+
+ /**
+ *
+ * @return A new set copied from this collection of queries. Elements in the set are not cloned or copied.
+ * Multiple iterations over returned set are guaranteed to be in same order. If there are no queries, then an empty
+ * set is returned. null is never returned.
+ */
+ Set<QueryContext> getQueries();
+
+ /**
+ * @param user User for whom queries have to be returned.
+ * @return A set of queries submitted by the given user. A new set is created and returned. Elements in the set
+ * are not cloned or copied. Multiple iterations over returned set are guaranteed to be in same order. If there are
+ * no queries, then an empty set is returned. null is never returned.
+ */
+ Set<QueryContext> getQueries(final String user);
+
+ /**
+ *
+ * @return Count of existing queries
+ */
+ int getQueriesCount();
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/WaitingQueriesSelectionPolicy.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/WaitingQueriesSelectionPolicy.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/WaitingQueriesSelectionPolicy.java
new file mode 100644
index 0000000..81442ef
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/collect/WaitingQueriesSelectionPolicy.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.collect;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.QueryContext;
+
+public interface WaitingQueriesSelectionPolicy {
+
+ /**
+ *
+ * Selects a subset of waiting queries eligible to move out of waiting state, based on logic of this selection policy.
+ *
+ * @param finishedQuery
+ * @param waitingQueries
+ * @return Waiting queries eligible to move out of waiting state. If no queries are eligible, then an empty set is
+ * returned. null is never returned.
+ */
+ Set<QueryContext> selectQueries(
+ final FinishedLensQuery finishedQuery, final EstimatedImmutableQueryCollection waitingQueries);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
new file mode 100644
index 0000000..bae2e64
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query.constraint;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConstraint {
+
+ private final int maxConcurrentQueries;
+
+ public MaxConcurrentDriverQueriesConstraint(final int maxConcurrentQueries) {
+ this.maxConcurrentQueries = maxConcurrentQueries;
+ }
+
+ @Override
+ public boolean allowsLaunchOf(
+ final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries) {
+
+ final LensDriver selectedDriver = candidateQuery.getSelectedDriver();
+ final boolean canLaunch = (launchedQueries.getQueriesCount(selectedDriver) < maxConcurrentQueries);
+
+ log.debug("canLaunch:{}", canLaunch);
+ return canLaunch;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
new file mode 100644
index 0000000..b6e6c2f
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query.constraint;
+
+import static java.lang.Integer.parseInt;
+
+import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class MaxConcurrentDriverQueriesConstraintFactory
+ implements ConfigBasedObjectCreationFactory<QueryLaunchingConstraint> {
+
+ public static final String MAX_CONCURRENT_QUERIES_KEY = "driver.max.concurrent.launched.queries";
+
+ @Override
+ public QueryLaunchingConstraint create(final Configuration conf) {
+
+ int maxConcurrentQueries = parseInt(conf.get(MAX_CONCURRENT_QUERIES_KEY));
+ return new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java
new file mode 100644
index 0000000..12d9562
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.constraint;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+
+public interface QueryLaunchingConstraint {
+
+ /**
+ * Returns whether this constraint allows candidate query to be launched.
+ *
+ * @param candidateQuery The query which is the next candidate to be launched.
+ * @param launchedQueries Current launched queries
+ * @return
+ */
+ boolean allowsLaunchOf(final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
index 7fa5cdb..792508b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
@@ -21,6 +21,8 @@
*/
package org.apache.lens.server.api.query.cost;
+import java.io.Serializable;
+
import org.apache.lens.api.query.QueryCostType;
import com.google.common.base.Preconditions;
@@ -29,7 +31,8 @@ import lombok.ToString;
@ToString
@EqualsAndHashCode
-public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBasedQueryCost> {
+public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBasedQueryCost>, Serializable {
+
private final double partitionCost;
public FactPartitionBasedQueryCost(final double partitionCost) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
index be72321..4712f11 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
@@ -22,14 +22,16 @@
package org.apache.lens.server.api.query.cost;
-import org.apache.lens.api.Addable;
import org.apache.lens.api.query.QueryCostType;
/**
* QueryCost Interface.
* @param <T>
*/
-public interface QueryCost<T extends QueryCost<T>> extends Comparable<T>, Addable<T> {
+public interface QueryCost<T extends QueryCost> extends Comparable<T> {
+
+ T add(final T queryCost);
+
QueryCostType getQueryCostType();
long getEstimatedExecTimeMillis() throws UnsupportedOperationException;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
index f926299..6dcb2d7 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
@@ -18,7 +18,15 @@
*/
package org.apache.lens.server.api.util;
+import java.util.Set;
+
+import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
+
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
/**
* Utility methods for Lens
@@ -47,4 +55,32 @@ public final class LensUtil {
return expMsg;
}
+ public static <T> ImmutableSet<T> getImplementations(final String factoriesKey, final Configuration conf) {
+
+ Set<T> implSet = Sets.newLinkedHashSet();
+ final String[] factoryNames = conf.getStrings(factoriesKey);
+
+ if (factoryNames == null) {
+ return ImmutableSet.copyOf(implSet);
+ }
+
+ for (String factoryName : factoryNames) {
+ if (StringUtils.isNotBlank(factoryName)) {
+ final T implementation = getImplementation(factoryName, conf);
+ implSet.add(implementation);
+ }
+ }
+ return ImmutableSet.copyOf(implSet);
+ }
+
+ public static <T> T getImplementation(final String factoryName, final Configuration conf) {
+
+ try {
+ ConfigBasedObjectCreationFactory<T> factory
+ = (ConfigBasedObjectCreationFactory<T>) Class.forName(factoryName).newInstance();
+ return factory.create(conf);
+ } catch (final ReflectiveOperationException e) {
+ throw new IllegalStateException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index dffcfd3..ea24336 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -34,12 +34,17 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.hadoop.conf.Configuration;
import org.apache.hive.service.cli.ColumnDescriptor;
+import com.beust.jcommander.internal.Sets;
+import com.google.common.collect.ImmutableSet;
+
/**
* The Class MockDriver.
*/
@@ -186,6 +191,16 @@ public class MockDriver implements LensDriver {
}
+ @Override
+ public ImmutableSet<QueryLaunchingConstraint> getQueryConstraints() {
+ return ImmutableSet.copyOf(Sets.<QueryLaunchingConstraint>newHashSet());
+ }
+
+ @Override
+ public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
+ return ImmutableSet.copyOf(Sets.<WaitingQueriesSelectionPolicy>newHashSet());
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/test/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyTest.java
new file mode 100644
index 0000000..389ee33
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/collect/DriverSpecificWaitingQueriesSelectionPolicyTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query.collect;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.FinishedLensQuery;
+
+import org.testng.annotations.Test;
+
+public class DriverSpecificWaitingQueriesSelectionPolicyTest {
+
+ @Test
+ public void testSelectQueries() {
+
+ FinishedLensQuery mockFinishedQuery = mock(FinishedLensQuery.class);
+ EstimatedImmutableQueryCollection mockWaitingQueries = mock(EstimatedImmutableQueryCollection.class);
+ LensDriver mockSelectedDriver = mock(LensDriver.class);
+ Set expectedQueriesSet = mock(Set.class);
+
+ when(mockFinishedQuery.getSelectedDriver()).thenReturn(mockSelectedDriver);
+ when(mockWaitingQueries.getQueries(mockSelectedDriver)).thenReturn(expectedQueriesSet);
+
+ WaitingQueriesSelectionPolicy selectionPolicy = new DriverSpecificWaitingQueriesSelectionPolicy();
+ Set actualEligibleQueries = selectionPolicy.selectQueries(mockFinishedQuery, mockWaitingQueries);
+
+ assertEquals(actualEligibleQueries, expectedQueriesSet);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
new file mode 100644
index 0000000..55a2eea
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query.constraint;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class MaxConcurrentDriverQueriesConstraintTest {
+
+ @DataProvider
+ public Object[][] dpTestAllowsLaunchOfQuery() {
+ return new Object[][] { {2, true} , {10, false}, {11, false}};
+ }
+
+ @Test(dataProvider = "dpTestAllowsLaunchOfQuery")
+ public void testAllowsLaunchOfQuery(final int currentDriverLaunchedQueries, final boolean expectedCanLaunch) {
+
+ int maxConcurrentQueries = 10;
+
+ QueryContext mockCandidateQuery = mock(QueryContext.class);
+ EstimatedImmutableQueryCollection mockLaunchedQueries = mock(EstimatedImmutableQueryCollection.class);
+ LensDriver mockDriver = mock(LensDriver.class);
+
+ when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver);
+ when(mockLaunchedQueries.getQueriesCount(mockDriver)).thenReturn(currentDriverLaunchedQueries);
+
+ QueryLaunchingConstraint constraint = new MaxConcurrentDriverQueriesConstraint(maxConcurrentQueries);
+ boolean actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
+
+ assertEquals(actualCanLaunch, expectedCanLaunch);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
index 0ab0b0e..e047c91 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/cost/TestFactPartitionBasedQueryCost.java
@@ -24,15 +24,16 @@ package org.apache.lens.server.api.query.cost;
import static org.testng.Assert.*;
import org.apache.lens.api.query.QueryCostType;
+import org.apache.lens.api.serialize.SerializationTest;
import org.testng.annotations.Test;
public class TestFactPartitionBasedQueryCost {
- FactPartitionBasedQueryCost cost0 = new FactPartitionBasedQueryCost(0.0);
- FactPartitionBasedQueryCost cost1 = new FactPartitionBasedQueryCost(0.2);
- FactPartitionBasedQueryCost cost11 = new FactPartitionBasedQueryCost(0.2);
- FactPartitionBasedQueryCost cost2 = new FactPartitionBasedQueryCost(0.3);
+ QueryCost cost0 = new FactPartitionBasedQueryCost(0.0);
+ QueryCost cost1 = new FactPartitionBasedQueryCost(0.2);
+ QueryCost cost11 = new FactPartitionBasedQueryCost(0.2);
+ QueryCost cost2 = new FactPartitionBasedQueryCost(0.3);
@Test(expectedExceptions = {IllegalArgumentException.class})
public void testInvalid() {
@@ -74,4 +75,9 @@ public class TestFactPartitionBasedQueryCost {
assertTrue(cost11.equals(cost1));
assertFalse(cost1.equals(cost2));
}
+
+ @Test
+ public void testFactPartitionBasedQueryCostIsSerializable() {
+ new SerializationTest().verifySerializationAndDeserialization(new FactPartitionBasedQueryCost(Double.MAX_VALUE));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 6ac30a5..35e7718 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -223,10 +223,6 @@
<artifactId>commons-dbutils</artifactId>
</dependency>
<dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
- <dependency>
<groupId>org.hsqldb</groupId>
<artifactId>hsqldb</artifactId>
</dependency>
@@ -243,6 +239,10 @@
<artifactId>joda-time</artifactId>
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.lens</groupId>
<artifactId>lens-server-api</artifactId>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index 39f9f52..2455232 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -127,6 +127,9 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
/** The running queries. */
private Gauge<Long> runningQueries;
+ /** The waiting queries. */
+ private Gauge<Long> waitingQueries;
+
/** The finished queries. */
private Gauge<Long> finishedQueries;
@@ -330,6 +333,14 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
}
});
+ waitingQueries = metricRegistry.register(MetricRegistry.name(QueryExecutionService.class, WAITING_QUERIES),
+ new Gauge<Long>() {
+ @Override
+ public Long getValue() {
+ return getQuerySvc().getWaitingQueriesCount();
+ }
+ });
+
finishedQueries = metricRegistry.register(MetricRegistry.name(QueryExecutionService.class, FINISHED_QUERIES),
new Gauge<Long>() {
@Override
@@ -360,7 +371,7 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
+ OPENED_SESSIONS));
totalClosedSessions = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-"
- + CLOSED_SESSIONS));
+ + CLOSED_SESSIONS));
totalExpiredSessions = metricRegistry.counter(MetricRegistry.name(QueryExecutionService.class, "total-"
+ EXPIRED_SESSIONS));
@@ -508,6 +519,11 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
}
@Override
+ public long getWaitingQueries() {
+ return waitingQueries.getValue();
+ }
+
+ @Override
public long getFinishedQueries() {
return finishedQueries.getValue();
}
[15/50] [abbrv] incubator-lens git commit: LENS-649 : Improve
exception handling in UserConfigLoaderFactory
Posted by sh...@apache.org.
LENS-649 : Improve exception handling in UserConfigLoaderFactory
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/588baef0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/588baef0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/588baef0
Branch: refs/heads/LENS-581
Commit: 588baef059051a29d2e7d2abb70f4196468305a7
Parents: c7261d1
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Jul 20 13:45:45 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Jul 20 13:45:45 2015 +0530
----------------------------------------------------------------------
.../lens/server/user/UserConfigLoaderFactory.java | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/588baef0/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
index f92584b..eaf3848 100644
--- a/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
+++ b/lens-server/src/main/java/org/apache/lens/server/user/UserConfigLoaderFactory.java
@@ -29,9 +29,12 @@ import org.apache.lens.server.api.user.UserConfigLoaderException;
import org.apache.hadoop.hive.conf.HiveConf;
+import lombok.extern.slf4j.Slf4j;
+
/**
* A factory for creating UserConfigLoader objects.
*/
+@Slf4j
public final class UserConfigLoaderFactory {
private UserConfigLoaderFactory() {
@@ -133,6 +136,15 @@ public final class UserConfigLoaderFactory {
* @return the user config
*/
public static Map<String, String> getUserConfig(String loggedInUser) {
- return getUserConfigLoader().getUserConfig(loggedInUser);
+ try {
+ Map<String, String> config = getUserConfigLoader().getUserConfig(loggedInUser);
+ if (config == null) {
+ throw new UserConfigLoaderException("Got null User config for: " + loggedInUser);
+ }
+ return config;
+ } catch (RuntimeException e) {
+ log.error("Couldn't get user config for user: " + loggedInUser, e);
+ throw e;
+ }
}
}
[46/50] [abbrv] incubator-lens git commit: Revert "Lens-581 :
Rafactored all API to new api (Vikas Singh via sharad)"
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java
new file mode 100644
index 0000000..e5ec272
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.impl;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.*;
+
+import org.apache.lens.client.LensClient;
+import org.apache.lens.client.LensClientConfig;
+import org.apache.lens.client.LensMLClient;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+public class MLRunner {
+
+ private static final Log LOG = LogFactory.getLog(MLRunner.class);
+
+ private LensMLClient mlClient;
+ private String algoName;
+ private String database;
+ private String trainTable;
+ private String trainFile;
+ private String testTable;
+ private String testFile;
+ private String outputTable;
+ private String[] features;
+ private String labelColumn;
+ private HiveConf conf;
+
+ public void init(LensMLClient mlClient, String confDir) throws Exception {
+ File dir = new File(confDir);
+ File propFile = new File(dir, "ml.properties");
+ Properties props = new Properties();
+ props.load(new FileInputStream(propFile));
+ String feat = props.getProperty("features");
+ String trainFile = confDir + File.separator + "train.data";
+ String testFile = confDir + File.separator + "test.data";
+ init(mlClient, props.getProperty("algo"), props.getProperty("database"),
+ props.getProperty("traintable"), trainFile,
+ props.getProperty("testtable"), testFile,
+ props.getProperty("outputtable"), feat.split(","),
+ props.getProperty("labelcolumn"));
+ }
+
+ public void init(LensMLClient mlClient, String algoName,
+ String database, String trainTable, String trainFile,
+ String testTable, String testFile, String outputTable, String[] features,
+ String labelColumn) {
+ this.mlClient = mlClient;
+ this.algoName = algoName;
+ this.database = database;
+ this.trainTable = trainTable;
+ this.trainFile = trainFile;
+ this.testTable = testTable;
+ this.testFile = testFile;
+ this.outputTable = outputTable;
+ this.features = features;
+ this.labelColumn = labelColumn;
+ //hive metastore settings are loaded via lens-site.xml, so loading LensClientConfig
+ //is required
+ this.conf = new HiveConf(new LensClientConfig(), MLRunner.class);
+ }
+
+ public MLTask train() throws Exception {
+ LOG.info("Starting train & eval");
+
+ createTable(trainTable, trainFile);
+ createTable(testTable, testFile);
+ MLTask.Builder taskBuilder = new MLTask.Builder();
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn).outputTable(outputTable)
+ .client(mlClient).trainingTable(trainTable).testTable(testTable);
+
+ // Add features
+ for (String feature : features) {
+ taskBuilder.addFeatureColumn(feature);
+ }
+ MLTask task = taskBuilder.build();
+ LOG.info("Created task " + task.toString());
+ task.run();
+ return task;
+ }
+
+ public void createTable(String tableName, String dataFile) throws HiveException {
+
+ File filedataFile = new File(dataFile);
+ Path dataFilePath = new Path(filedataFile.toURI());
+ Path partDir = dataFilePath.getParent();
+
+ // Create table
+ List<FieldSchema> columns = new ArrayList<FieldSchema>();
+
+ // Label is optional. Not used for unsupervised models.
+ // If present, label will be the first column, followed by features
+ if (labelColumn != null) {
+ columns.add(new FieldSchema(labelColumn, "double", "Labelled Column"));
+ }
+
+ for (String feature : features) {
+ columns.add(new FieldSchema(feature, "double", "Feature " + feature));
+ }
+
+ Table tbl = Hive.get(conf).newTable(database + "." + tableName);
+ tbl.setTableType(TableType.MANAGED_TABLE);
+ tbl.getTTable().getSd().setCols(columns);
+ // tbl.getTTable().getParameters().putAll(new HashMap<String, String>());
+ tbl.setInputFormatClass(TextInputFormat.class);
+ tbl.setSerdeParam(serdeConstants.LINE_DELIM, "\n");
+ tbl.setSerdeParam(serdeConstants.FIELD_DELIM, " ");
+
+ List<FieldSchema> partCols = new ArrayList<FieldSchema>(1);
+ partCols.add(new FieldSchema("dummy_partition_col", "string", ""));
+ tbl.setPartCols(partCols);
+
+ Hive.get(conf).dropTable(database, tableName, false, true);
+ Hive.get(conf).createTable(tbl, true);
+ LOG.info("Created table " + tableName);
+
+ // Add partition for the data file
+ AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName,
+ false);
+ Map<String, String> partSpec = new HashMap<String, String>();
+ partSpec.put("dummy_partition_col", "dummy_val");
+ partitionDesc.addPartition(partSpec, partDir.toUri().toString());
+ Hive.get(conf).createPartitions(partitionDesc);
+ LOG.info(tableName + ": Added partition " + partDir.toUri().toString());
+ }
+
+ public static void main(String[] args) throws Exception {
+ if (args.length < 1) {
+ System.out.println("Usage: " + MLRunner.class.getName() + " <ml-conf-dir>");
+ System.exit(-1);
+ }
+ String confDir = args[0];
+ LensMLClient client = new LensMLClient(new LensClient());
+ MLRunner runner = new MLRunner();
+ runner.init(client, confDir);
+ runner.train();
+ System.out.println("Created the Model successfully. Output Table: " + runner.outputTable);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java
new file mode 100644
index 0000000..2867b90
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java
@@ -0,0 +1,285 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.impl;
+
+import java.util.*;
+
+import org.apache.lens.client.LensMLClient;
+import org.apache.lens.ml.api.LensML;
+import org.apache.lens.ml.api.MLTestReport;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import lombok.Getter;
+import lombok.ToString;
+
+/**
+ * Run a complete cycle of train and test (evaluation) for an ML algorithm
+ */
+@ToString
+public class MLTask implements Runnable {
+ private static final Log LOG = LogFactory.getLog(MLTask.class);
+
+ public enum State {
+ RUNNING, SUCCESSFUL, FAILED
+ }
+
+ @Getter
+ private State taskState;
+
+ /**
+ * Name of the algo/algorithm.
+ */
+ @Getter
+ private String algorithm;
+
+ /**
+ * Name of the table containing training data.
+ */
+ @Getter
+ private String trainingTable;
+
+ /**
+ * Name of the table containing test data. Optional, if not provided trainingTable itself is
+ * used for testing
+ */
+ @Getter
+ private String testTable;
+
+ /**
+ * Training table partition spec
+ */
+ @Getter
+ private String partitionSpec;
+
+ /**
+ * Name of the column which is a label for supervised algorithms.
+ */
+ @Getter
+ private String labelColumn;
+
+ /**
+ * Names of columns which are features in the training data.
+ */
+ @Getter
+ private List<String> featureColumns;
+
+ /**
+ * Configuration for the example.
+ */
+ @Getter
+ private HiveConf configuration;
+
+ private LensML ml;
+ private String taskID;
+
+ /**
+ * ml client
+ */
+ @Getter
+ private LensMLClient mlClient;
+
+ /**
+ * Output table name
+ */
+ @Getter
+ private String outputTable;
+
+ /**
+ * Extra params passed to the training algorithm
+ */
+ @Getter
+ private Map<String, String> extraParams;
+
+ @Getter
+ private String modelID;
+
+ @Getter
+ private String reportID;
+
+ /**
+ * Use ExampleTask.Builder to create an instance
+ */
+ private MLTask() {
+ // Use builder to construct the example
+ extraParams = new HashMap<String, String>();
+ taskID = UUID.randomUUID().toString();
+ }
+
+ /**
+ * Builder to create an example task
+ */
+ public static class Builder {
+ private MLTask task;
+
+ public Builder() {
+ task = new MLTask();
+ }
+
+ public Builder trainingTable(String trainingTable) {
+ task.trainingTable = trainingTable;
+ return this;
+ }
+
+ public Builder testTable(String testTable) {
+ task.testTable = testTable;
+ return this;
+ }
+
+ public Builder algorithm(String algorithm) {
+ task.algorithm = algorithm;
+ return this;
+ }
+
+ public Builder labelColumn(String labelColumn) {
+ task.labelColumn = labelColumn;
+ return this;
+ }
+
+ public Builder client(LensMLClient client) {
+ task.mlClient = client;
+ return this;
+ }
+
+ public Builder addFeatureColumn(String featureColumn) {
+ if (task.featureColumns == null) {
+ task.featureColumns = new ArrayList<String>();
+ }
+ task.featureColumns.add(featureColumn);
+ return this;
+ }
+
+ public Builder hiveConf(HiveConf hiveConf) {
+ task.configuration = hiveConf;
+ return this;
+ }
+
+
+
+ public Builder extraParam(String param, String value) {
+ task.extraParams.put(param, value);
+ return this;
+ }
+
+ public Builder partitionSpec(String partitionSpec) {
+ task.partitionSpec = partitionSpec;
+ return this;
+ }
+
+ public Builder outputTable(String outputTable) {
+ task.outputTable = outputTable;
+ return this;
+ }
+
+ public MLTask build() {
+ MLTask builtTask = task;
+ task = null;
+ return builtTask;
+ }
+
+ }
+
+ @Override
+ public void run() {
+ taskState = State.RUNNING;
+ LOG.info("Starting " + taskID);
+ try {
+ runTask();
+ taskState = State.SUCCESSFUL;
+ LOG.info("Complete " + taskID);
+ } catch (Exception e) {
+ taskState = State.FAILED;
+ LOG.info("Error running task " + taskID, e);
+ }
+ }
+
+ /**
+ * Train an ML model, with specified algorithm and input data. Do model evaluation using the evaluation data and print
+ * evaluation result
+ *
+ * @throws Exception
+ */
+ private void runTask() throws Exception {
+ if (mlClient != null) {
+ // Connect to a remote Lens server
+ ml = mlClient;
+ LOG.info("Working in client mode. Lens session handle " + mlClient.getSessionHandle().getPublicId());
+ } else {
+ // In server mode session handle has to be passed by the user as a request parameter
+ ml = MLUtils.getMLService();
+ LOG.info("Working in Lens server");
+ }
+
+ String[] algoArgs = buildTrainingArgs();
+ LOG.info("Starting task " + taskID + " algo args: " + Arrays.toString(algoArgs));
+
+ modelID = ml.train(trainingTable, algorithm, algoArgs);
+ printModelMetadata(taskID, modelID);
+
+ LOG.info("Starting test " + taskID);
+ testTable = (testTable != null) ? testTable : trainingTable;
+ MLTestReport testReport = ml.testModel(mlClient.getSessionHandle(), testTable, algorithm, modelID, outputTable);
+ reportID = testReport.getReportID();
+ printTestReport(taskID, testReport);
+ saveTask();
+ }
+
+ // Save task metadata to DB
+ private void saveTask() {
+ LOG.info("Saving task details to DB");
+ }
+
+ private void printTestReport(String exampleID, MLTestReport testReport) {
+ StringBuilder builder = new StringBuilder("Example: ").append(exampleID);
+ builder.append("\n\t");
+ builder.append("EvaluationReport: ").append(testReport.toString());
+ System.out.println(builder.toString());
+ }
+
+ private String[] buildTrainingArgs() {
+ List<String> argList = new ArrayList<String>();
+ argList.add("label");
+ argList.add(labelColumn);
+
+ // Add all the features
+ for (String featureCol : featureColumns) {
+ argList.add("feature");
+ argList.add(featureCol);
+ }
+
+ // Add extra params
+ for (String param : extraParams.keySet()) {
+ argList.add(param);
+ argList.add(extraParams.get(param));
+ }
+
+ return argList.toArray(new String[argList.size()]);
+ }
+
+ // Get the model instance and print its metadat to stdout
+ private void printModelMetadata(String exampleID, String modelID) throws Exception {
+ StringBuilder builder = new StringBuilder("Example: ").append(exampleID);
+ builder.append("\n\t");
+ builder.append("Model: ");
+ builder.append(ml.getModel(algorithm, modelID).toString());
+ System.out.println(builder.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLUtils.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLUtils.java
index ad75917..9c96d9b 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLUtils.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLUtils.java
@@ -18,28 +18,21 @@
*/
package org.apache.lens.ml.impl;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-
-import org.apache.lens.ml.algo.api.TrainedModel;
-import org.apache.lens.ml.api.Model;
+import org.apache.lens.ml.algo.api.Algorithm;
+import org.apache.lens.ml.algo.api.MLAlgo;
import org.apache.lens.ml.server.MLService;
import org.apache.lens.ml.server.MLServiceImpl;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.ServiceProvider;
import org.apache.lens.server.api.ServiceProviderFactory;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
public final class MLUtils {
- private static final HiveConf HIVE_CONF;
+ private MLUtils() {
+ }
- private static final Log LOG = LogFactory.getLog(MLUtils.class);
+ private static final HiveConf HIVE_CONF;
static {
HIVE_CONF = new HiveConf();
@@ -48,7 +41,12 @@ public final class MLUtils {
HIVE_CONF.addResource("lens-site.xml");
}
- private MLUtils() {
+ public static String getAlgoName(Class<? extends MLAlgo> algoClass) {
+ Algorithm annotation = algoClass.getAnnotation(Algorithm.class);
+ if (annotation != null) {
+ return annotation.name();
+ }
+ throw new IllegalArgumentException("Algo should be decorated with annotation - " + Algorithm.class.getName());
}
public static MLServiceImpl getMLService() throws Exception {
@@ -61,34 +59,4 @@ public final class MLUtils {
ServiceProviderFactory spf = spfClass.newInstance();
return spf.getServiceProvider();
}
-
- public static Path persistModel(TrainedModel trainedModel, Model model, String modelInstanceId) throws IOException {
- Path algoDir = getAlgoDir(model.getAlgoSpec().getAlgo().getName());
- FileSystem fs = algoDir.getFileSystem(HIVE_CONF);
-
- if (!fs.exists(algoDir)) {
- fs.mkdirs(algoDir);
- }
-
- Path modelSavePath = new Path(algoDir, new Path(model.getId(), modelInstanceId));
- ObjectOutputStream outputStream = null;
-
- try {
- outputStream = new ObjectOutputStream(fs.create(modelSavePath, false));
- outputStream.writeObject(trainedModel);
- outputStream.flush();
- } catch (IOException io) {
- LOG.error("Error saving model " + modelInstanceId + " reason: " + io.getMessage());
- throw io;
- } finally {
- IOUtils.closeQuietly(outputStream);
- }
- return modelSavePath;
- }
-
- public static Path getAlgoDir(String algoName) throws IOException {
- String modelSaveBaseDir = HIVE_CONF.get(ModelLoader.MODEL_PATH_BASE_DIR, ModelLoader.MODEL_PATH_BASE_DIR_DEFAULT);
- return new Path(new Path(modelSaveBaseDir), algoName);
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
index dc23807..c0e7953 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
@@ -20,11 +20,13 @@ package org.apache.lens.ml.impl;
import java.io.IOException;
import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import org.apache.lens.ml.algo.api.TrainedModel;
+import org.apache.lens.ml.algo.api.MLModel;
+import org.apache.lens.ml.api.MLTestReport;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
@@ -37,46 +39,38 @@ import org.apache.hadoop.hive.conf.HiveConf;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
-public class ModelLoader {
- /**
- * The Constant MODEL_PATH_BASE_DIR.
- */
+/**
+ * Load ML models from a FS location.
+ */
+public final class ModelLoader {
+ private ModelLoader() {
+ }
+
+ /** The Constant MODEL_PATH_BASE_DIR. */
public static final String MODEL_PATH_BASE_DIR = "lens.ml.model.basedir";
- /**
- * The Constant MODEL_PATH_BASE_DIR_DEFAULT.
- */
+ /** The Constant MODEL_PATH_BASE_DIR_DEFAULT. */
public static final String MODEL_PATH_BASE_DIR_DEFAULT = "file:///tmp";
- /**
- * The Constant LOG.
- */
+
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(ModelLoader.class);
- /**
- * The Constant TEST_REPORT_BASE_DIR.
- */
+
+ /** The Constant TEST_REPORT_BASE_DIR. */
public static final String TEST_REPORT_BASE_DIR = "lens.ml.test.basedir";
- /**
- * The Constant TEST_REPORT_BASE_DIR_DEFAULT.
- */
+
+ /** The Constant TEST_REPORT_BASE_DIR_DEFAULT. */
public static final String TEST_REPORT_BASE_DIR_DEFAULT = "file:///tmp/ml_reports";
- /**
- * The Constant MODEL_CACHE_SIZE.
- */
- public static final long MODEL_CACHE_SIZE = 10;
// Model cache settings
- /**
- * The Constant MODEL_CACHE_TIMEOUT.
- */
+ /** The Constant MODEL_CACHE_SIZE. */
+ public static final long MODEL_CACHE_SIZE = 10;
+
+ /** The Constant MODEL_CACHE_TIMEOUT. */
public static final long MODEL_CACHE_TIMEOUT = 3600000L; // one hour
- /**
- * The model cache.
- */
- private static Cache<Path, TrainedModel> modelCache = CacheBuilder.newBuilder().maximumSize(MODEL_CACHE_SIZE)
- .expireAfterAccess(MODEL_CACHE_TIMEOUT, TimeUnit.MILLISECONDS).build();
- private ModelLoader() {
- }
+ /** The model cache. */
+ private static Cache<Path, MLModel> modelCache = CacheBuilder.newBuilder().maximumSize(MODEL_CACHE_SIZE)
+ .expireAfterAccess(MODEL_CACHE_TIMEOUT, TimeUnit.MILLISECONDS).build();
/**
* Gets the model location.
@@ -86,28 +80,29 @@ public class ModelLoader {
* @param modelID the model id
* @return the model location
*/
- public static Path getModelLocation(Configuration conf, String algorithm, String modelID, String modelInstanceId) {
+ public static Path getModelLocation(Configuration conf, String algorithm, String modelID) {
String modelDataBaseDir = conf.get(MODEL_PATH_BASE_DIR, MODEL_PATH_BASE_DIR_DEFAULT);
// Model location format - <modelDataBaseDir>/<algorithm>/modelID
- return new Path(new Path(new Path(new Path(modelDataBaseDir), algorithm), modelID), modelInstanceId);
+ return new Path(new Path(new Path(modelDataBaseDir), algorithm), modelID);
}
/**
- * @param conf
- * @param algorithm
- * @param modelID
- * @return
- * @throws IOException
+ * Load model.
+ *
+ * @param conf the conf
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @return the ML model
+ * @throws IOException Signals that an I/O exception has occurred.
*/
- public static TrainedModel loadModel(Configuration conf, String algorithm, final String modelID,
- String modelInstanceId) throws IOException {
- final Path modelPath = getModelLocation(conf, algorithm, modelID, modelInstanceId);
+ public static MLModel loadModel(Configuration conf, String algorithm, String modelID) throws IOException {
+ final Path modelPath = getModelLocation(conf, algorithm, modelID);
LOG.info("Loading model for algorithm: " + algorithm + " modelID: " + modelID + " At path: "
+ modelPath.toUri().toString());
try {
- return modelCache.get(modelPath, new Callable<TrainedModel>() {
+ return modelCache.get(modelPath, new Callable<MLModel>() {
@Override
- public TrainedModel call() throws Exception {
+ public MLModel call() throws Exception {
FileSystem fs = modelPath.getFileSystem(new HiveConf());
if (!fs.exists(modelPath)) {
throw new IOException("Model path not found " + modelPath.toString());
@@ -116,8 +111,8 @@ public class ModelLoader {
ObjectInputStream ois = null;
try {
ois = new ObjectInputStream(fs.open(modelPath));
- TrainedModel model = (TrainedModel) ois.readObject();
- LOG.info("Loaded model " + modelID + " from location " + modelPath);
+ MLModel model = (MLModel) ois.readObject();
+ LOG.info("Loaded model " + model.getId() + " from location " + modelPath);
return model;
} catch (ClassNotFoundException e) {
throw new IOException(e);
@@ -139,6 +134,86 @@ public class ModelLoader {
}
/**
+ * Gets the test report path.
+ *
+ * @param conf the conf
+ * @param algorithm the algorithm
+ * @param report the report
+ * @return the test report path
+ */
+ public static Path getTestReportPath(Configuration conf, String algorithm, String report) {
+ String testReportDir = conf.get(TEST_REPORT_BASE_DIR, TEST_REPORT_BASE_DIR_DEFAULT);
+ return new Path(new Path(testReportDir, algorithm), report);
+ }
+
+ /**
+ * Save test report.
+ *
+ * @param conf the conf
+ * @param report the report
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public static void saveTestReport(Configuration conf, MLTestReport report) throws IOException {
+ Path reportDir = new Path(conf.get(TEST_REPORT_BASE_DIR, TEST_REPORT_BASE_DIR_DEFAULT));
+ FileSystem fs = reportDir.getFileSystem(conf);
+
+ if (!fs.exists(reportDir)) {
+ LOG.info("Creating test report dir " + reportDir.toUri().toString());
+ fs.mkdirs(reportDir);
+ }
+
+ Path algoDir = new Path(reportDir, report.getAlgorithm());
+
+ if (!fs.exists(algoDir)) {
+ LOG.info("Creating algorithm report dir " + algoDir.toUri().toString());
+ fs.mkdirs(algoDir);
+ }
+
+ ObjectOutputStream reportOutputStream = null;
+ Path reportSaveLocation;
+ try {
+ reportSaveLocation = new Path(algoDir, report.getReportID());
+ reportOutputStream = new ObjectOutputStream(fs.create(reportSaveLocation));
+ reportOutputStream.writeObject(report);
+ reportOutputStream.flush();
+ } catch (IOException ioexc) {
+ LOG.error("Error saving test report " + report.getReportID(), ioexc);
+ throw ioexc;
+ } finally {
+ IOUtils.closeQuietly(reportOutputStream);
+ }
+ LOG.info("Saved report " + report.getReportID() + " at location " + reportSaveLocation.toUri());
+ }
+
+ /**
+ * Load report.
+ *
+ * @param conf the conf
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @return the ML test report
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public static MLTestReport loadReport(Configuration conf, String algorithm, String reportID) throws IOException {
+ Path reportLocation = getTestReportPath(conf, algorithm, reportID);
+ FileSystem fs = reportLocation.getFileSystem(conf);
+ ObjectInputStream reportStream = null;
+ MLTestReport report = null;
+
+ try {
+ reportStream = new ObjectInputStream(fs.open(reportLocation));
+ report = (MLTestReport) reportStream.readObject();
+ } catch (IOException ioex) {
+ LOG.error("Error reading report " + reportLocation, ioex);
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ } finally {
+ IOUtils.closeQuietly(reportStream);
+ }
+ return report;
+ }
+
+ /**
* Delete model.
*
* @param conf the conf
@@ -146,10 +221,22 @@ public class ModelLoader {
* @param modelID the model id
* @throws IOException Signals that an I/O exception has occurred.
*/
- public static void deleteModel(HiveConf conf, String algorithm, String modelID, String modelInstanceId)
- throws IOException {
- Path modelLocation = getModelLocation(conf, algorithm, modelID, modelInstanceId);
+ public static void deleteModel(HiveConf conf, String algorithm, String modelID) throws IOException {
+ Path modelLocation = getModelLocation(conf, algorithm, modelID);
FileSystem fs = modelLocation.getFileSystem(conf);
fs.delete(modelLocation, false);
}
+
+ /**
+ * Delete test report.
+ *
+ * @param conf the conf
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public static void deleteTestReport(HiveConf conf, String algorithm, String reportID) throws IOException {
+ Path reportPath = getTestReportPath(conf, algorithm, reportID);
+ reportPath.getFileSystem(conf).delete(reportPath, false);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/QueryRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/QueryRunner.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/QueryRunner.java
index 16f5414..2eeee29 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/QueryRunner.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/QueryRunner.java
@@ -30,13 +30,10 @@ import lombok.Setter;
*/
public abstract class QueryRunner {
- /**
- * The session handle.
- */
+ /** The session handle. */
protected final LensSessionHandle sessionHandle;
- @Getter
- @Setter
+ @Getter @Setter
protected String queryName;
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
index 36ccb68..34b2a3f 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import org.apache.lens.ml.api.Feature;
-
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,65 +38,40 @@ import lombok.Getter;
*/
public class TableTestingSpec {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(TableTestingSpec.class);
- /**
- * The db.
- */
+ /** The db. */
private String db;
- /**
- * The table containing input data.
- */
+ /** The table containing input data. */
private String inputTable;
// TODO use partition condition
- /**
- * The partition filter.
- */
+ /** The partition filter. */
private String partitionFilter;
- /**
- * The feature columns.
- */
- private List<Feature> featureColumns;
+ /** The feature columns. */
+ private List<String> featureColumns;
- /**
- * The label column.
- */
- private Feature labelColumn;
+ /** The label column. */
+ private String labelColumn;
- /**
- * The output column.
- */
+ /** The output column. */
private String outputColumn;
- /**
- * The output table.
- */
+ /** The output table. */
private String outputTable;
- /**
- * The conf.
- */
+ /** The conf. */
private transient HiveConf conf;
- /**
- * The algorithm.
- */
+ /** The algorithm. */
private String algorithm;
- /**
- * The model id.
- */
+ /** The model id. */
private String modelID;
- /*The modelInstanceId*/
- private String modelInstanceId;
-
@Getter
private boolean outputTableExists;
@@ -108,128 +81,11 @@ public class TableTestingSpec {
private HashMap<String, FieldSchema> columnNameToFieldSchema;
/**
- * New builder.
- *
- * @return the table testing spec builder
- */
- public static TableTestingSpecBuilder newBuilder() {
- return new TableTestingSpecBuilder();
- }
-
- /**
- * Validate.
- *
- * @return true, if successful
- */
- public boolean validate() {
- List<FieldSchema> columns;
- try {
- Hive metastoreClient = Hive.get(conf);
- Table tbl = (db == null) ? metastoreClient.getTable(inputTable) : metastoreClient.getTable(db, inputTable);
- columns = tbl.getAllCols();
- columnNameToFieldSchema = new HashMap<String, FieldSchema>();
-
- for (FieldSchema fieldSchema : columns) {
- columnNameToFieldSchema.put(fieldSchema.getName(), fieldSchema);
- }
-
- // Check if output table exists
- Table outTbl = metastoreClient.getTable(db == null ? "default" : db, outputTable, false);
- outputTableExists = (outTbl != null);
- } catch (HiveException exc) {
- LOG.error("Error getting table info " + toString(), exc);
- return false;
- }
-
- // Check if labeled column and feature columns are contained in the table
- List<String> testTableColumns = new ArrayList<String>(columns.size());
- for (FieldSchema column : columns) {
- testTableColumns.add(column.getName());
- }
-
- List<String> inputColumnNames = new ArrayList();
- for (Feature feature : featureColumns) {
- inputColumnNames.add(feature.getDataColumn());
- }
-
- if (!testTableColumns.containsAll(inputColumnNames)) {
- LOG.info("Invalid feature columns: " + inputColumnNames + ". Actual columns in table:" + testTableColumns);
- return false;
- }
-
- if (!testTableColumns.contains(labelColumn.getDataColumn())) {
- LOG.info(
- "Invalid label column: " + labelColumn.getDataColumn() + ". Actual columns in table:" + testTableColumns);
- return false;
- }
-
- if (StringUtils.isBlank(outputColumn)) {
- LOG.info("Output column is required");
- return false;
- }
-
- if (StringUtils.isBlank(outputTable)) {
- LOG.info("Output table is required");
- return false;
- }
- return true;
- }
-
- public String getTestQuery() {
- if (!validate()) {
- return null;
- }
-
- // We always insert a dynamic partition
- StringBuilder q = new StringBuilder("INSERT OVERWRITE TABLE " + outputTable + " PARTITION (part_testid='" + testID
- + "') SELECT ");
- List<String> featureNameList = new ArrayList();
- List<String> featureMapBuilder = new ArrayList();
- for (Feature feature : featureColumns) {
- featureNameList.add(feature.getDataColumn());
- featureMapBuilder.add("'" + feature.getDataColumn() + "'");
- featureMapBuilder.add(feature.getDataColumn());
- }
- String featureCols = StringUtils.join(featureNameList, ",");
- String featureMapString = StringUtils.join(featureMapBuilder, ",");
- q.append(featureCols).append(",").append(labelColumn.getDataColumn()).append(", ").append("predict(").append("'")
- .append(algorithm)
- .append("', ").append("'").append(modelID).append("', ").append("'").append(modelInstanceId).append("', ")
- .append(featureMapString).append(") ").append(outputColumn)
- .append(" FROM ").append(inputTable);
-
- return q.toString();
- }
-
- public String getCreateOutputTableQuery() {
- StringBuilder createTableQuery = new StringBuilder("CREATE TABLE IF NOT EXISTS ").append(outputTable).append("(");
- // Output table contains feature columns, label column, output column
- List<String> outputTableColumns = new ArrayList<String>();
- for (Feature featureCol : featureColumns) {
- outputTableColumns.add(featureCol.getDataColumn() + " "
- + columnNameToFieldSchema.get(featureCol.getDataColumn()).getType());
- }
-
- outputTableColumns.add(labelColumn.getDataColumn() + " "
- + columnNameToFieldSchema.get(labelColumn.getDataColumn()).getType());
- outputTableColumns.add(outputColumn + " string");
-
- createTableQuery.append(StringUtils.join(outputTableColumns, ", "));
-
- // Append partition column
- createTableQuery.append(") PARTITIONED BY (part_testid string)");
-
- return createTableQuery.toString();
- }
-
- /**
* The Class TableTestingSpecBuilder.
*/
public static class TableTestingSpecBuilder {
- /**
- * The spec.
- */
+ /** The spec. */
private final TableTestingSpec spec;
/**
@@ -278,7 +134,7 @@ public class TableTestingSpec {
* @param featureColumns the feature columns
* @return the table testing spec builder
*/
- public TableTestingSpecBuilder featureColumns(List<Feature> featureColumns) {
+ public TableTestingSpecBuilder featureColumns(List<String> featureColumns) {
spec.featureColumns = featureColumns;
return this;
}
@@ -289,7 +145,7 @@ public class TableTestingSpec {
* @param labelColumn the label column
* @return the table testing spec builder
*/
- public TableTestingSpecBuilder lableColumn(Feature labelColumn) {
+ public TableTestingSpecBuilder lableColumn(String labelColumn) {
spec.labelColumn = labelColumn;
return this;
}
@@ -349,11 +205,6 @@ public class TableTestingSpec {
return this;
}
- public TableTestingSpecBuilder modelInstanceID(String modelInstanceId) {
- spec.modelInstanceId = modelInstanceId;
- return this;
- }
-
/**
* Builds the.
*
@@ -374,4 +225,101 @@ public class TableTestingSpec {
return this;
}
}
+
+ /**
+ * New builder.
+ *
+ * @return the table testing spec builder
+ */
+ public static TableTestingSpecBuilder newBuilder() {
+ return new TableTestingSpecBuilder();
+ }
+
+ /**
+ * Validate.
+ *
+ * @return true, if successful
+ */
+ public boolean validate() {
+ List<FieldSchema> columns;
+ try {
+ Hive metastoreClient = Hive.get(conf);
+ Table tbl = (db == null) ? metastoreClient.getTable(inputTable) : metastoreClient.getTable(db, inputTable);
+ columns = tbl.getAllCols();
+ columnNameToFieldSchema = new HashMap<String, FieldSchema>();
+
+ for (FieldSchema fieldSchema : columns) {
+ columnNameToFieldSchema.put(fieldSchema.getName(), fieldSchema);
+ }
+
+ // Check if output table exists
+ Table outTbl = metastoreClient.getTable(db == null ? "default" : db, outputTable, false);
+ outputTableExists = (outTbl != null);
+ } catch (HiveException exc) {
+ LOG.error("Error getting table info " + toString(), exc);
+ return false;
+ }
+
+ // Check if labeled column and feature columns are contained in the table
+ List<String> testTableColumns = new ArrayList<String>(columns.size());
+ for (FieldSchema column : columns) {
+ testTableColumns.add(column.getName());
+ }
+
+ if (!testTableColumns.containsAll(featureColumns)) {
+ LOG.info("Invalid feature columns: " + featureColumns + ". Actual columns in table:" + testTableColumns);
+ return false;
+ }
+
+ if (!testTableColumns.contains(labelColumn)) {
+ LOG.info("Invalid label column: " + labelColumn + ". Actual columns in table:" + testTableColumns);
+ return false;
+ }
+
+ if (StringUtils.isBlank(outputColumn)) {
+ LOG.info("Output column is required");
+ return false;
+ }
+
+ if (StringUtils.isBlank(outputTable)) {
+ LOG.info("Output table is required");
+ return false;
+ }
+ return true;
+ }
+
+ public String getTestQuery() {
+ if (!validate()) {
+ return null;
+ }
+
+ // We always insert a dynamic partition
+ StringBuilder q = new StringBuilder("INSERT OVERWRITE TABLE " + outputTable + " PARTITION (part_testid='" + testID
+ + "') SELECT ");
+ String featureCols = StringUtils.join(featureColumns, ",");
+ q.append(featureCols).append(",").append(labelColumn).append(", ").append("predict(").append("'").append(algorithm)
+ .append("', ").append("'").append(modelID).append("', ").append(featureCols).append(") ").append(outputColumn)
+ .append(" FROM ").append(inputTable);
+
+ return q.toString();
+ }
+
+ public String getCreateOutputTableQuery() {
+ StringBuilder createTableQuery = new StringBuilder("CREATE TABLE IF NOT EXISTS ").append(outputTable).append("(");
+ // Output table contains feature columns, label column, output column
+ List<String> outputTableColumns = new ArrayList<String>();
+ for (String featureCol : featureColumns) {
+ outputTableColumns.add(featureCol + " " + columnNameToFieldSchema.get(featureCol).getType());
+ }
+
+ outputTableColumns.add(labelColumn + " " + columnNameToFieldSchema.get(labelColumn).getType());
+ outputTableColumns.add(outputColumn + " string");
+
+ createTableQuery.append(StringUtils.join(outputTableColumns, ", "));
+
+ // Append partition column
+ createTableQuery.append(") PARTITIONED BY (part_testid string)");
+
+ return createTableQuery.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLApp.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLApp.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLApp.java
index 641ec5c..e6e3c02 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLApp.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLApp.java
@@ -29,8 +29,14 @@ import org.glassfish.jersey.media.multipart.MultiPartFeature;
@ApplicationPath("/ml")
public class MLApp extends Application {
+
private final Set<Class<?>> classes;
+ /**
+ * Pass additional classes when running in test mode
+ *
+ * @param additionalClasses
+ */
public MLApp(Class<?>... additionalClasses) {
classes = new HashSet<Class<?>>();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
index 4ec9e96..de9c711 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
@@ -21,13 +21,22 @@ package org.apache.lens.ml.server;
import java.util.List;
import java.util.Map;
+import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.ml.api.*;
+import org.apache.lens.api.query.LensQuery;
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.ml.algo.api.MLAlgo;
+import org.apache.lens.ml.algo.api.MLModel;
+import org.apache.lens.ml.api.MLTestReport;
import org.apache.lens.ml.impl.LensMLImpl;
+import org.apache.lens.ml.impl.ModelLoader;
+import org.apache.lens.ml.impl.QueryRunner;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.ServiceProvider;
import org.apache.lens.server.api.ServiceProviderFactory;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,24 +48,16 @@ import org.apache.hive.service.CompositeService;
*/
public class MLServiceImpl extends CompositeService implements MLService {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(LensMLImpl.class);
- /**
- * The ml.
- */
+ /** The ml. */
private LensMLImpl ml;
- /**
- * The service provider.
- */
+ /** The service provider. */
private ServiceProvider serviceProvider;
- /**
- * The service provider factory.
- */
+ /** The service provider factory. */
private ServiceProviderFactory serviceProviderFactory;
/**
@@ -76,97 +77,48 @@ public class MLServiceImpl extends CompositeService implements MLService {
}
@Override
- public List<Algo> getAlgos() {
- return ml.getAlgos();
- }
-
- @Override
- public Algo getAlgo(String name) throws LensException {
- return ml.getAlgo(name);
- }
-
- @Override
- public String createDataSet(String name, String dataTable, String dataBase) throws LensException {
- return ml.createDataSet(name, dataTable, dataBase);
- }
-
- @Override
- public String createDataSetFromQuery(String name, String query) {
- return ml.createDataSetFromQuery(name, query);
- }
-
- @Override
- public DataSet getDataSet(String name) throws LensException {
- return ml.getDataSet(name);
- }
-
- @Override
- public String createModel(String name, String algo, Map<String, String> algoParams, List<Feature> features,
- Feature label, LensSessionHandle lensSessionHandle) throws LensException {
- return ml.createModel(name, algo, algoParams, features, label, lensSessionHandle);
- }
-
- @Override
- public boolean cancelModelInstance(String modelInstanceId) {
- return ml.cancelModelInstance(modelInstanceId);
- }
-
- @Override
- public boolean cancelEvaluation(String evalId) {
- return ml.cancelEvaluation(evalId);
- }
-
- @Override
- public boolean cancelPrediction(String predicitonId) {
- return ml.cancelPrediction(predicitonId);
- }
-
- @Override
- public Model getModel(String modelId) throws LensException {
- return ml.getModel(modelId);
- }
-
- @Override
- public String trainModel(String modelId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- return ml.trainModel(modelId, dataSetName, lensSessionHandle);
- }
-
- @Override
- public ModelInstance getModelInstance(String modelInstanceId) throws LensException {
- return ml.getModelInstance(modelInstanceId);
- }
-
- @Override
- public List<ModelInstance> getAllModelInstances(String modelId) {
- return ml.getAllModelInstances(modelId);
- }
-
- @Override
- public String evaluate(String modelInstanceId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- return ml.evaluate(modelInstanceId, dataSetName, lensSessionHandle);
+ public List<String> getAlgorithms() {
+ return ml.getAlgorithms();
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getAlgoForName(java.lang.String)
+ */
@Override
- public Evaluation getEvaluation(String evalId) throws LensException {
- return ml.getEvaluation(evalId);
+ public MLAlgo getAlgoForName(String algorithm) throws LensException {
+ return ml.getAlgoForName(algorithm);
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#train(java.lang.String, java.lang.String, java.lang.String[])
+ */
@Override
- public String predict(String modelInstanceId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- return ml.predict(modelInstanceId, dataSetName, lensSessionHandle);
+ public String train(String table, String algorithm, String[] args) throws LensException {
+ return ml.train(table, algorithm, args);
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getModels(java.lang.String)
+ */
@Override
- public Prediction getPrediction(String predictionId) throws LensException {
- return ml.getPrediction(predictionId);
+ public List<String> getModels(String algorithm) throws LensException {
+ return ml.getModels(algorithm);
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getModel(java.lang.String, java.lang.String)
+ */
@Override
- public String predict(String modelInstanceId, Map<String, String> featureVector) throws LensException {
- return ml.predict(modelInstanceId, featureVector);
+ public MLModel getModel(String algorithm, String modelId) throws LensException {
+ return ml.getModel(algorithm, modelId);
}
private ServiceProvider getServiceProvider() {
@@ -193,6 +145,11 @@ public class MLServiceImpl extends CompositeService implements MLService {
}
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.hive.service.CompositeService#init(org.apache.hadoop.hive.conf.HiveConf)
+ */
@Override
public synchronized void init(HiveConf hiveConf) {
ml = new LensMLImpl(hiveConf);
@@ -203,10 +160,10 @@ public class MLServiceImpl extends CompositeService implements MLService {
}
/*
- * (non-Javadoc)
- *
- * @see org.apache.hive.service.CompositeService#start()
- */
+ * (non-Javadoc)
+ *
+ * @see org.apache.hive.service.CompositeService#start()
+ */
@Override
public synchronized void start() {
ml.start();
@@ -225,4 +182,145 @@ public class MLServiceImpl extends CompositeService implements MLService {
super.stop();
LOG.info("Stopped ML service");
}
+
+ /**
+ * Clear models.
+ */
+ public void clearModels() {
+ ModelLoader.clearCache();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getModelPath(java.lang.String, java.lang.String)
+ */
+ @Override
+ public String getModelPath(String algorithm, String modelID) {
+ return ml.getModelPath(algorithm, modelID);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#testModel(org.apache.lens.api.LensSessionHandle, java.lang.String, java.lang.String,
+ * java.lang.String)
+ */
+ @Override
+ public MLTestReport testModel(LensSessionHandle sessionHandle, String table, String algorithm, String modelID,
+ String outputTable) throws LensException {
+ return ml.testModel(sessionHandle, table, algorithm, modelID, new DirectQueryRunner(sessionHandle), outputTable);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getTestReports(java.lang.String)
+ */
+ @Override
+ public List<String> getTestReports(String algorithm) throws LensException {
+ return ml.getTestReports(algorithm);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getTestReport(java.lang.String, java.lang.String)
+ */
+ @Override
+ public MLTestReport getTestReport(String algorithm, String reportID) throws LensException {
+ return ml.getTestReport(algorithm, reportID);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#predict(java.lang.String, java.lang.String, java.lang.Object[])
+ */
+ @Override
+ public Object predict(String algorithm, String modelID, Object[] features) throws LensException {
+ return ml.predict(algorithm, modelID, features);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#deleteModel(java.lang.String, java.lang.String)
+ */
+ @Override
+ public void deleteModel(String algorithm, String modelID) throws LensException {
+ ml.deleteModel(algorithm, modelID);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#deleteTestReport(java.lang.String, java.lang.String)
+ */
+ @Override
+ public void deleteTestReport(String algorithm, String reportID) throws LensException {
+ ml.deleteTestReport(algorithm, reportID);
+ }
+
+ /**
+ * Run the test model query directly in the current lens server process.
+ */
+ private class DirectQueryRunner extends QueryRunner {
+
+ /**
+ * Instantiates a new direct query runner.
+ *
+ * @param sessionHandle the session handle
+ */
+ public DirectQueryRunner(LensSessionHandle sessionHandle) {
+ super(sessionHandle);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.TestQueryRunner#runQuery(java.lang.String)
+ */
+ @Override
+ public QueryHandle runQuery(String testQuery) throws LensException {
+ // Run the query in query executions service
+ QueryExecutionService queryService = (QueryExecutionService) getServiceProvider().getService("query");
+
+ LensConf queryConf = new LensConf();
+ queryConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, false + "");
+ queryConf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, false + "");
+
+ QueryHandle testQueryHandle = queryService.executeAsync(sessionHandle, testQuery, queryConf, queryName);
+
+ // Wait for test query to complete
+ LensQuery query = queryService.getQuery(sessionHandle, testQueryHandle);
+ LOG.info("Submitted query " + testQueryHandle.getHandleId());
+ while (!query.getStatus().finished()) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ throw new LensException(e);
+ }
+
+ query = queryService.getQuery(sessionHandle, testQueryHandle);
+ }
+
+ if (query.getStatus().getStatus() != QueryStatus.Status.SUCCESSFUL) {
+ throw new LensException("Failed to run test query: " + testQueryHandle.getHandleId() + " reason= "
+ + query.getStatus().getErrorMessage());
+ }
+
+ return testQueryHandle;
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.LensML#getAlgoParamDescription(java.lang.String)
+ */
+ @Override
+ public Map<String, String> getAlgoParamDescription(String algorithm) {
+ return ml.getAlgoParamDescription(algorithm);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
index b9a49cb..77a6c40 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
@@ -18,23 +18,35 @@
*/
package org.apache.lens.ml.server;
+import static org.apache.commons.lang.StringUtils.isBlank;
+
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import javax.ws.rs.*;
-import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.*;
+import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
-import org.apache.lens.api.result.LensAPIResult;
-import org.apache.lens.ml.api.Algo;
+import org.apache.lens.ml.algo.api.MLModel;
+import org.apache.lens.ml.api.MLTestReport;
+import org.apache.lens.ml.api.ModelMetadata;
+import org.apache.lens.ml.api.TestReport;
+import org.apache.lens.ml.impl.ModelLoader;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.ServiceProvider;
import org.apache.lens.server.api.ServiceProviderFactory;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.glassfish.jersey.media.multipart.FormDataParam;
+
/**
* Machine Learning service.
*/
@@ -42,15 +54,24 @@ import org.apache.hadoop.hive.conf.HiveConf;
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
public class MLServiceResource {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(MLServiceResource.class);
+
+ /** The ml service. */
+ MLService mlService;
+
+ /** The service provider. */
+ ServiceProvider serviceProvider;
+
+ /** The service provider factory. */
+ ServiceProviderFactory serviceProviderFactory;
+
+ private static final HiveConf HIVE_CONF;
+
/**
* Message indicating if ML service is up
*/
public static final String ML_UP_MESSAGE = "ML service is up";
- private static final HiveConf HIVE_CONF;
static {
HIVE_CONF = new HiveConf();
@@ -60,19 +81,6 @@ public class MLServiceResource {
}
/**
- * The ml service.
- */
- MLService mlService;
- /**
- * The service provider.
- */
- ServiceProvider serviceProvider;
- /**
- * The service provider factory.
- */
- ServiceProviderFactory serviceProviderFactory;
-
- /**
* Instantiates a new ML service resource.
*/
public MLServiceResource() {
@@ -121,57 +129,286 @@ public class MLServiceResource {
}
/**
- * Get the list of algos available.
+ * Get a list of algos available
*
* @return
*/
@GET
@Path("algos")
- public List<Algo> getAlgos() {
- List<Algo> algos = getMlService().getAlgos();
- return algos;
+ public StringList getAlgoNames() {
+ List<String> algos = getMlService().getAlgorithms();
+ StringList result = new StringList(algos);
+ return result;
}
+ /**
+ * Gets the human readable param description of an algorithm
+ *
+ * @param algorithm the algorithm
+ * @return the param description
+ */
@GET
- @Path("algonames")
- public StringList getAlgoNames() {
- List<Algo> algos = getMlService().getAlgos();
- ArrayList<String> stringArrayList = new ArrayList();
- for (Algo algo : algos) {
- stringArrayList.add(algo.getName());
+ @Path("algos/{algorithm}")
+ public StringList getParamDescription(@PathParam("algorithm") String algorithm) {
+ Map<String, String> paramDesc = getMlService().getAlgoParamDescription(algorithm);
+ if (paramDesc == null) {
+ throw new NotFoundException("Param description not found for " + algorithm);
}
- StringList result = new StringList(stringArrayList);
- return result;
+ List<String> descriptions = new ArrayList<String>();
+ for (String key : paramDesc.keySet()) {
+ descriptions.add(key + " : " + paramDesc.get(key));
+ }
+ return new StringList(descriptions);
}
/**
+ * Get model ID list for a given algorithm.
*
- * @param dataSetName
- * @param dataTableName
- * @param dataBaseName
- * @return
+ * @param algorithm algorithm name
+ * @return the models for algo
+ * @throws LensException the lens exception
+ */
+ @GET
+ @Path("models/{algorithm}")
+ public StringList getModelsForAlgo(@PathParam("algorithm") String algorithm) throws LensException {
+ List<String> models = getMlService().getModels(algorithm);
+ if (models == null || models.isEmpty()) {
+ throw new NotFoundException("No models found for algorithm " + algorithm);
+ }
+ return new StringList(models);
+ }
+
+ /**
+ * Get metadata of the model given algorithm and model ID.
+ *
+ * @param algorithm algorithm name
+ * @param modelID model ID
+ * @return model metadata
+ * @throws LensException the lens exception
+ */
+ @GET
+ @Path("models/{algorithm}/{modelID}")
+ public ModelMetadata getModelMetadata(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID)
+ throws LensException {
+ MLModel model = getMlService().getModel(algorithm, modelID);
+ if (model == null) {
+ throw new NotFoundException("Model not found " + modelID + ", algo=" + algorithm);
+ }
+
+ ModelMetadata meta = new ModelMetadata(model.getId(), model.getTable(), model.getAlgoName(), StringUtils.join(
+ model.getParams(), ' '), model.getCreatedAt().toString(), getMlService().getModelPath(algorithm, modelID),
+ model.getLabelColumn(), StringUtils.join(model.getFeatureColumns(), ","));
+ return meta;
+ }
+
+ /**
+ * Delete a model given model ID and algorithm name.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @return confirmation text
+ * @throws LensException the lens exception
+ */
+ @DELETE
+ @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
+ @Path("models/{algorithm}/{modelID}")
+ public String deleteModel(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID)
+ throws LensException {
+ getMlService().deleteModel(algorithm, modelID);
+ return "DELETED model=" + modelID + " algorithm=" + algorithm;
+ }
+
+ /**
+ * Train a model given an algorithm name and algorithm parameters
+ * <p>
+ * Following parameters are mandatory and must be passed as part of the form
+ * </p>
+ * <ol>
+ * <li>table - input Hive table to load training data from</li>
+ * <li>label - name of the labelled column</li>
+ * <li>feature - one entry per feature column. At least one feature column is required</li>
+ * </ol>
+ * <p></p>
+ *
+ * @param algorithm algorithm name
+ * @param form form data
+ * @return if model is successfully trained, the model ID will be returned
+ * @throws LensException the lens exception
*/
@POST
- @Path("dataset")
- public LensAPIResult<String> createDataSet(@QueryParam("dataSetName") String dataSetName,
- @QueryParam("dataTableName") String dataTableName,
- @QueryParam("dataBaseName")
- String dataBaseName) {
- String dataSetId = "";
- try {
- getMlService().createDataSet(dataSetName, dataTableName, dataBaseName);
- return LensAPIResult.composedOf(null, null, null);
- } catch (Exception e) {
- LOG.error("Error creating dataSet " + dataSetName, e);
- return LensAPIResult.composedOf(null, "1.2", dataSetId);
+ @Consumes(MediaType.APPLICATION_FORM_URLENCODED)
+ @Path("{algorithm}/train")
+ public String train(@PathParam("algorithm") String algorithm, MultivaluedMap<String, String> form)
+ throws LensException {
+
+ // Check if algo is valid
+ if (getMlService().getAlgoForName(algorithm) == null) {
+ throw new NotFoundException("Algo for algo: " + algorithm + " not found");
+ }
+
+ if (isBlank(form.getFirst("table"))) {
+ throw new BadRequestException("table parameter is rquired");
+ }
+
+ String table = form.getFirst("table");
+
+ if (isBlank(form.getFirst("label"))) {
+ throw new BadRequestException("label parameter is required");
+ }
+
+ // Check features
+ List<String> featureNames = form.get("feature");
+ if (featureNames.size() < 1) {
+ throw new BadRequestException("At least one feature is required");
+ }
+
+ List<String> algoArgs = new ArrayList<String>();
+ Set<Map.Entry<String, List<String>>> paramSet = form.entrySet();
+
+ for (Map.Entry<String, List<String>> e : paramSet) {
+ String p = e.getKey();
+ List<String> values = e.getValue();
+ if ("algorithm".equals(p) || "table".equals(p)) {
+ continue;
+ } else if ("feature".equals(p)) {
+ for (String feature : values) {
+ algoArgs.add("feature");
+ algoArgs.add(feature);
+ }
+ } else if ("label".equals(p)) {
+ algoArgs.add("label");
+ algoArgs.add(values.get(0));
+ } else {
+ algoArgs.add(p);
+ algoArgs.add(values.get(0));
+ }
+ }
+ LOG.info("Training table " + table + " with algo " + algorithm + " params=" + algoArgs.toString());
+ String modelId = getMlService().train(table, algorithm, algoArgs.toArray(new String[]{}));
+ LOG.info("Done training " + table + " modelid = " + modelId);
+ return modelId;
+ }
+
+ /**
+ * Clear model cache (for admin use).
+ *
+ * @return OK if the cache was cleared
+ */
+ @DELETE
+ @Path("clearModelCache")
+ @Produces(MediaType.TEXT_PLAIN)
+ public Response clearModelCache() {
+ ModelLoader.clearCache();
+ LOG.info("Cleared model cache");
+ return Response.ok("Cleared cache", MediaType.TEXT_PLAIN_TYPE).build();
+ }
+
+ /**
+ * Run a test on a model for an algorithm.
+ *
+ * @param algorithm algorithm name
+ * @param modelID model ID
+ * @param table Hive table to run test on
+ * @param session Lens session ID. This session ID will be used to run the test query
+ * @return Test report ID
+ * @throws LensException the lens exception
+ */
+ @POST
+ @Path("test/{table}/{algorithm}/{modelID}")
+ @Consumes(MediaType.MULTIPART_FORM_DATA)
+ public String test(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID,
+ @PathParam("table") String table, @FormDataParam("sessionid") LensSessionHandle session,
+ @FormDataParam("outputTable") String outputTable) throws LensException {
+ MLTestReport testReport = getMlService().testModel(session, table, algorithm, modelID, outputTable);
+ return testReport.getReportID();
+ }
+
+ /**
+ * Get list of reports for a given algorithm.
+ *
+ * @param algoritm the algoritm
+ * @return the reports for algorithm
+ * @throws LensException the lens exception
+ */
+ @GET
+ @Path("reports/{algorithm}")
+ public StringList getReportsForAlgorithm(@PathParam("algorithm") String algoritm) throws LensException {
+ List<String> reports = getMlService().getTestReports(algoritm);
+ if (reports == null || reports.isEmpty()) {
+ throw new NotFoundException("No test reports found for " + algoritm);
}
+ return new StringList(reports);
+ }
+
+ /**
+ * Get a single test report given the algorithm name and report id.
+ *
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @return the test report
+ * @throws LensException the lens exception
+ */
+ @GET
+ @Path("reports/{algorithm}/{reportID}")
+ public TestReport getTestReport(@PathParam("algorithm") String algorithm, @PathParam("reportID") String reportID)
+ throws LensException {
+ MLTestReport report = getMlService().getTestReport(algorithm, reportID);
+
+ if (report == null) {
+ throw new NotFoundException("Test report: " + reportID + " not found for algorithm " + algorithm);
+ }
+
+ TestReport result = new TestReport(report.getTestTable(), report.getOutputTable(), report.getOutputColumn(),
+ report.getLabelColumn(), StringUtils.join(report.getFeatureColumns(), ","), report.getAlgorithm(),
+ report.getModelID(), report.getReportID(), report.getLensQueryID());
+ return result;
}
- //@POST
- //@Path("evaluation")
- //public Evaluation evaluation(){
- // return new Evaluation("modelInstance1", "dataSet1","id", new Date(), new Date(), Status.CANCELLED);
- //}
+ /**
+ * DELETE a report given the algorithm name and report ID.
+ *
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @return the string
+ * @throws LensException the lens exception
+ */
+ @DELETE
+ @Path("reports/{algorithm}/{reportID}")
+ @Consumes({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
+ public String deleteTestReport(@PathParam("algorithm") String algorithm, @PathParam("reportID") String reportID)
+ throws LensException {
+ getMlService().deleteTestReport(algorithm, reportID);
+ return "DELETED report=" + reportID + " algorithm=" + algorithm;
+ }
+ /**
+ * Predict.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @param uriInfo the uri info
+ * @return the string
+ * @throws LensException the lens exception
+ */
+ @GET
+ @Path("/predict/{algorithm}/{modelID}")
+ @Produces({MediaType.APPLICATION_ATOM_XML, MediaType.APPLICATION_JSON})
+ public String predict(@PathParam("algorithm") String algorithm, @PathParam("modelID") String modelID,
+ @Context UriInfo uriInfo) throws LensException {
+ // Load the model instance
+ MLModel<?> model = getMlService().getModel(algorithm, modelID);
+
+ // Get input feature names
+ MultivaluedMap<String, String> params = uriInfo.getQueryParameters();
+ String[] features = new String[model.getFeatureColumns().size()];
+ // Assuming that feature name parameters are same
+ int i = 0;
+ for (String feature : model.getFeatureColumns()) {
+ features[i++] = params.getFirst(feature);
+ }
+
+ // TODO needs a 'prediction formatter'
+ return getMlService().predict(algorithm, modelID, features).toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
index 9d7ee78..fc4164f 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
@@ -78,17 +78,25 @@ import org.apache.spark.rdd.RDD;
*/
public class LensRDDClient {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(LensRDDClient.class);
// Default input format for table created from Lens result set
- /**
- * The Constant HIVE_CONF.
- */
+ /** The Constant INPUT_FORMAT. */
+ private static final String INPUT_FORMAT = TextInputFormat.class.getName();
+ // Default output format
+ /** The Constant OUTPUT_FORMAT. */
+ private static final String OUTPUT_FORMAT = TextOutputFormat.class.getName();
+ // Name of partition column and its value. There is always exactly one partition in the table created from
+ // Result set.
+ /** The Constant TEMP_TABLE_PART_COL. */
+ private static final String TEMP_TABLE_PART_COL = "dummy_partition_column";
+
+ /** The Constant TEMP_TABLE_PART_VAL. */
+ private static final String TEMP_TABLE_PART_VAL = "placeholder_value";
+
+ /** The Constant HIVE_CONF. */
protected static final HiveConf HIVE_CONF = new HiveConf();
- // Default output format
static {
HIVE_CONF.setVar(HiveConf.ConfVars.METASTOREURIS, "");
HIVE_CONF.set("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=./metastore_db;create=true");
@@ -96,32 +104,11 @@ public class LensRDDClient {
HIVE_CONF.setBoolean("hive.metastore.local", true);
HIVE_CONF.set("hive.metastore.warehouse.dir", "file://${user.dir}/warehouse");
}
- // Name of partition column and its value. There is always exactly one partition in the table created from
- // Result set.
- /**
- * The Constant INPUT_FORMAT.
- */
- private static final String INPUT_FORMAT = TextInputFormat.class.getName();
- /**
- * The Constant OUTPUT_FORMAT.
- */
- private static final String OUTPUT_FORMAT = TextOutputFormat.class.getName();
- /**
- * The Constant TEMP_TABLE_PART_COL.
- */
- private static final String TEMP_TABLE_PART_COL = "dummy_partition_column";
- /**
- * The Constant TEMP_TABLE_PART_VAL.
- */
- private static final String TEMP_TABLE_PART_VAL = "placeholder_value";
- /**
- * The spark context.
- */
+
+ /** The spark context. */
private final JavaSparkContext sparkContext; // Spark context
- /**
- * The lens client.
- */
+ /** The lens client. */
private LensClient lensClient; // Lens client instance. Initialized lazily.
/**
@@ -331,19 +318,13 @@ public class LensRDDClient {
*/
public static class LensRDDResult implements Serializable {
- /**
- * The result rdd.
- */
+ /** The result rdd. */
private transient RDD<List<Object>> resultRDD;
- /**
- * The lens query.
- */
+ /** The lens query. */
private QueryHandle lensQuery;
- /**
- * The temp table name.
- */
+ /** The temp table name. */
private String tempTableName;
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
index 712c278..9fe1ea0 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
@@ -40,11 +40,11 @@ import org.apache.hadoop.mapred.TextInputFormat;
* The Class ExampleUtils.
*/
public final class ExampleUtils {
- private static final Log LOG = LogFactory.getLog(ExampleUtils.class);
-
private ExampleUtils() {
}
+ private static final Log LOG = LogFactory.getLog(ExampleUtils.class);
+
/**
* Creates the example table.
*
@@ -57,8 +57,7 @@ public final class ExampleUtils {
* @throws HiveException the hive exception
*/
public static void createTable(HiveConf conf, String database, String tableName, String sampleDataFile,
- String labelColumn, Map<String, String> tableParams, String... features)
- throws HiveException {
+ String labelColumn, Map<String, String> tableParams, String... features) throws HiveException {
Path dataFilePath = new Path(sampleDataFile);
Path partDir = dataFilePath.getParent();
@@ -66,16 +65,16 @@ public final class ExampleUtils {
// Create table
List<FieldSchema> columns = new ArrayList<FieldSchema>();
- for (String feature : features) {
- columns.add(new FieldSchema(feature, "double", "Feature " + feature));
- }
-
// Label is optional. Not used for unsupervised models.
// If present, label will be the first column, followed by features
if (labelColumn != null) {
columns.add(new FieldSchema(labelColumn, "double", "Labelled Column"));
}
+ for (String feature : features) {
+ columns.add(new FieldSchema(feature, "double", "Feature " + feature));
+ }
+
Table tbl = Hive.get(conf).newTable(database + "." + tableName);
tbl.setTableType(TableType.MANAGED_TABLE);
tbl.getTTable().getSd().setCols(columns);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
index 2fe846b..51344ce 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLResource.java
@@ -18,9 +18,51 @@
*/
package org.apache.lens.ml;
-public class TestMLResource {
- /*
- private static final Log LOG = LogFactory.getLog(TestMLResource.class);
+import java.io.File;
+import java.net.URI;
+import java.util.*;
+
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.UriBuilder;
+
+import org.apache.lens.client.LensClient;
+import org.apache.lens.client.LensClientConfig;
+import org.apache.lens.client.LensMLClient;
+import org.apache.lens.ml.algo.spark.dt.DecisionTreeAlgo;
+import org.apache.lens.ml.algo.spark.lr.LogisticRegressionAlgo;
+import org.apache.lens.ml.algo.spark.nb.NaiveBayesAlgo;
+import org.apache.lens.ml.algo.spark.svm.SVMAlgo;
+import org.apache.lens.ml.impl.MLTask;
+import org.apache.lens.ml.impl.MLUtils;
+import org.apache.lens.ml.server.MLApp;
+import org.apache.lens.ml.server.MLServiceResource;
+import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.query.QueryServiceResource;
+import org.apache.lens.server.session.SessionResource;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import org.testng.Assert;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Test
+public class TestMLResource extends LensJerseyTest {
+
private static final String TEST_DB = "default";
private WebTarget mlTarget;
@@ -28,7 +70,7 @@ public class TestMLResource {
@Override
protected int getTestPort() {
- return 10003;
+ return 10002;
}
@Override
@@ -56,7 +98,7 @@ public class TestMLResource {
LensClientConfig lensClientConfig = new LensClientConfig();
lensClientConfig.setLensDatabase(TEST_DB);
lensClientConfig.set(LensConfConstants.SERVER_BASE_URL,
- "http://localhost:" + getTestPort() + "/lensapi");
+ "http://localhost:" + getTestPort() + "/lensapi");
LensClient client = new LensClient(lensClientConfig);
mlClient = new LensMLClient(client);
}
@@ -70,7 +112,7 @@ public class TestMLResource {
hive.dropDatabase(TEST_DB);
} catch (Exception exc) {
// Ignore drop db exception
- ////LOG.error(exc.getMessage());
+ log.error("Exception while dropping database.", exc);
}
mlClient.close();
}
@@ -88,7 +130,120 @@ public class TestMLResource {
@Test
public void testGetAlgos() throws Exception {
- mlClient.test();
+ List<String> algoNames = mlClient.getAlgorithms();
+ Assert.assertNotNull(algoNames);
+
+ Assert.assertTrue(
+ algoNames.contains(MLUtils.getAlgoName(NaiveBayesAlgo.class)),
+ MLUtils.getAlgoName(NaiveBayesAlgo.class));
+
+ Assert.assertTrue(algoNames.contains(MLUtils.getAlgoName(SVMAlgo.class)),
+ MLUtils.getAlgoName(SVMAlgo.class));
+
+ Assert.assertTrue(
+ algoNames.contains(MLUtils.getAlgoName(LogisticRegressionAlgo.class)),
+ MLUtils.getAlgoName(LogisticRegressionAlgo.class));
+
+ Assert.assertTrue(
+ algoNames.contains(MLUtils.getAlgoName(DecisionTreeAlgo.class)),
+ MLUtils.getAlgoName(DecisionTreeAlgo.class));
}
-*/
+
+ @Test
+ public void testGetAlgoParams() throws Exception {
+ Map<String, String> params = mlClient.getAlgoParamDescription(MLUtils
+ .getAlgoName(DecisionTreeAlgo.class));
+ Assert.assertNotNull(params);
+ Assert.assertFalse(params.isEmpty());
+
+ for (String key : params.keySet()) {
+ log.info("## Param " + key + " help = " + params.get(key));
+ }
+ }
+
+ @Test
+ public void trainAndEval() throws Exception {
+ log.info("Starting train & eval");
+ final String algoName = MLUtils.getAlgoName(NaiveBayesAlgo.class);
+ HiveConf conf = new HiveConf();
+ String tableName = "naivebayes_training_table";
+ String sampleDataFilePath = "data/naive_bayes/naive_bayes_train.data";
+
+ File sampleDataFile = new File(sampleDataFilePath);
+ URI sampleDataFileURI = sampleDataFile.toURI();
+
+ String labelColumn = "label";
+ String[] features = { "feature_1", "feature_2", "feature_3" };
+ String outputTable = "naivebayes_eval_table";
+
+ log.info("Creating training table from file "
+ + sampleDataFileURI.toString());
+
+ Map<String, String> tableParams = new HashMap<String, String>();
+ try {
+ ExampleUtils.createTable(conf, TEST_DB, tableName,
+ sampleDataFileURI.toString(), labelColumn, tableParams, features);
+ } catch (HiveException exc) {
+ log.error("Hive exception encountered.", exc);
+ }
+ MLTask.Builder taskBuilder = new MLTask.Builder();
+
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn)
+ .outputTable(outputTable).client(mlClient).trainingTable(tableName);
+
+ // Add features
+ taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2")
+ .addFeatureColumn("feature_3");
+
+ MLTask task = taskBuilder.build();
+
+ log.info("Created task " + task.toString());
+ task.run();
+ Assert.assertEquals(task.getTaskState(), MLTask.State.SUCCESSFUL);
+
+ String firstModelID = task.getModelID();
+ String firstReportID = task.getReportID();
+ Assert.assertNotNull(firstReportID);
+ Assert.assertNotNull(firstModelID);
+
+ taskBuilder = new MLTask.Builder();
+ taskBuilder.algorithm(algoName).hiveConf(conf).labelColumn(labelColumn)
+ .outputTable(outputTable).client(mlClient).trainingTable(tableName);
+
+ taskBuilder.addFeatureColumn("feature_1").addFeatureColumn("feature_2")
+ .addFeatureColumn("feature_3");
+
+ MLTask anotherTask = taskBuilder.build();
+
+ log.info("Created second task " + anotherTask.toString());
+ anotherTask.run();
+
+ String secondModelID = anotherTask.getModelID();
+ String secondReportID = anotherTask.getReportID();
+ Assert.assertNotNull(secondModelID);
+ Assert.assertNotNull(secondReportID);
+
+ Hive metastoreClient = Hive.get(conf);
+ Table outputHiveTable = metastoreClient.getTable(outputTable);
+ List<Partition> partitions = metastoreClient.getPartitions(outputHiveTable);
+
+ Assert.assertNotNull(partitions);
+
+ int i = 0;
+ Set<String> partReports = new HashSet<String>();
+ for (Partition part : partitions) {
+ log.info("@@PART#" + i + " " + part.getSpec().toString());
+ partReports.add(part.getSpec().get("part_testid"));
+ }
+
+ // Verify partitions created for each run
+ Assert.assertTrue(partReports.contains(firstReportID), firstReportID
+ + " first partition not there");
+ Assert.assertTrue(partReports.contains(secondReportID), secondReportID
+ + " second partition not there");
+
+ log.info("Completed task run");
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
index e52c31b..7f6413b 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
@@ -18,9 +18,39 @@
*/
package org.apache.lens.ml;
-//@Test
-public class TestMLRunner {
- /*
+import java.net.URI;
+
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.UriBuilder;
+
+import org.apache.lens.client.LensClient;
+import org.apache.lens.client.LensClientConfig;
+import org.apache.lens.client.LensMLClient;
+import org.apache.lens.ml.impl.MLRunner;
+import org.apache.lens.ml.impl.MLTask;
+import org.apache.lens.ml.server.MLApp;
+import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.metastore.MetastoreResource;
+import org.apache.lens.server.query.QueryServiceResource;
+import org.apache.lens.server.session.SessionResource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import org.testng.Assert;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+
+@Test
+public class TestMLRunner extends LensJerseyTest {
private static final Log LOG = LogFactory.getLog(TestMLRunner.class);
private static final String TEST_DB = TestMLRunner.class.getSimpleName();
@@ -105,5 +135,4 @@ public class TestMLRunner {
Assert.assertNotNull(modelID);
Assert.assertNotNull(reportID);
}
- */
}
[31/50] [abbrv] incubator-lens git commit: LENS-668 : Use Slf4j in
lens-server module
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
index 71d29b4..da3bfd1 100644
--- a/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/ui/MetastoreUIResource.java
@@ -30,13 +30,12 @@ import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.metastore.CubeMetastoreService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
+import lombok.extern.slf4j.Slf4j;
+
/**
* metastore UI resource api
* <p></p>
@@ -44,10 +43,9 @@ import org.json.JSONObject;
*/
@Path("metastoreapi")
+@Slf4j
public class MetastoreUIResource {
- public static final Log LOG = LogFactory.getLog(MetastoreUIResource.class);
-
public CubeMetastoreService getSvc() {
return (CubeMetastoreService) LensServices.get().getService("metastore");
}
@@ -96,7 +94,7 @@ public class MetastoreUIResource {
try {
tableList.put(new JSONObject().put("name", cube).put("type", "cube"));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -113,7 +111,7 @@ public class MetastoreUIResource {
try {
tableList.put(new JSONObject().put("name", dimTable).put("type", "dimtable"));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -124,7 +122,7 @@ public class MetastoreUIResource {
tableList.put(new JSONObject().put("name", nativeTable).put("type", "native"));
}
} catch (Exception e) {
- LOG.error(e);
+ log.error("Error while getting all native tables", e);
}
return tableList.toString();
@@ -160,7 +158,7 @@ public class MetastoreUIResource {
try {
attribList.put(new JSONObject().put("name", measure.getName()).put("type", measure.getType()));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -169,7 +167,7 @@ public class MetastoreUIResource {
try {
attribList.put(new JSONObject().put("name", dim.getName()).put("type", dim.getType()));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -185,7 +183,7 @@ public class MetastoreUIResource {
try {
attribList.put(new JSONObject().put("name", col.getName()).put("type", col.getType()));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -196,7 +194,7 @@ public class MetastoreUIResource {
attribList.put(new JSONObject().put("name", expr.getName()).put("type", "expression")
.put("expression", expr.getExprSpec()));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -223,7 +221,7 @@ public class MetastoreUIResource {
tableList = new JSONArray(getAllTables(publicId));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
for (int item = 0; item < tableList.length(); item++) {
String name = null, type = null;
@@ -231,7 +229,7 @@ public class MetastoreUIResource {
name = tableList.getJSONObject(item).getString("name");
type = tableList.getJSONObject(item).getString("type");
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
if (type.equals("cube")) {
JSONArray cubeAttribList = null;
@@ -240,7 +238,7 @@ public class MetastoreUIResource {
cubeAttribList = new JSONArray(getDescription(publicId, "cube", name));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
for (int col = 0; col < cubeAttribList.length(); col++) {
String colname = null, coltype = null;
@@ -249,13 +247,13 @@ public class MetastoreUIResource {
coltype = cubeAttribList.getJSONObject(col).getString("type");
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
if (colname.contains(keyword)) {
try {
cubeSearchResultList.put(new JSONObject().put("name", colname).put("type", coltype));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -264,14 +262,14 @@ public class MetastoreUIResource {
searchResultList.put(new JSONObject().put("name", name).put("type", type).put("columns",
cubeSearchResultList));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
} else if (name.contains(keyword)) {
try {
searchResultList.put(new JSONObject().put("name", name).put("type", type).put("columns",
cubeSearchResultList));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
} else if (type.equals("dimtable")) {
@@ -281,7 +279,7 @@ public class MetastoreUIResource {
dimAttribList = new JSONArray(getDescription(publicId, "dimtable", name));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
for (int col = 0; col < dimAttribList.length(); col++) {
String colname = null, coltype = null;
@@ -290,13 +288,13 @@ public class MetastoreUIResource {
coltype = dimAttribList.getJSONObject(col).getString("type");
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
if (colname.contains(keyword)) {
try {
dimSearchResultList.put(new JSONObject().put("name", colname).put("type", coltype));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
@@ -305,14 +303,14 @@ public class MetastoreUIResource {
searchResultList.put(new JSONObject().put("name", name).put("type", type).put("columns",
dimSearchResultList));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
} else if (name.contains(keyword)) {
try {
searchResultList.put(new JSONObject().put("name", name).put("type", type).put("columns",
dimSearchResultList));
} catch (JSONException j) {
- LOG.error(j);
+ log.error("Error while parsing json", j);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
index 391cf08..ed2552c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/ui/QueryServiceUIResource.java
@@ -35,20 +35,18 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.glassfish.jersey.media.multipart.FormDataParam;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class QueryServiceUIResource.
*/
@Path("/queryuiapi")
+@Slf4j
public class QueryServiceUIResource {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(QueryServiceUIResource.class);
-
/** The query server. */
private QueryExecutionService queryServer;
@@ -82,7 +80,7 @@ public class QueryServiceUIResource {
* Instantiates a new query service ui resource.
*/
public QueryServiceUIResource() {
- LOG.info("Query UI Service");
+ log.info("Query UI Service");
queryServer = (QueryExecutionService) LensServices.get().getService("query");
}
@@ -194,7 +192,7 @@ public class QueryServiceUIResource {
LensSessionHandle sessionHandle = SessionUIResource.getOpenSession(publicId);
checkSessionHandle(sessionHandle);
List<Object> rows = new ArrayList<Object>();
- LOG.info("FetchResultSet for queryHandle:" + queryHandle);
+ log.info("FetchResultSet for queryHandle:{}", queryHandle);
try {
QueryResultSetMetadata resultSetMetadata = queryServer.getResultSetMetadata(sessionHandle,
getQueryHandle(queryHandle));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
index cd1ba36..01db262 100644
--- a/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/ui/SessionUIResource.java
@@ -33,22 +33,19 @@ import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import org.glassfish.jersey.media.multipart.FormDataParam;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Session resource api
* <p></p>
* This provides api for all things in session.
*/
@Path("/uisession")
+@Slf4j
public class SessionUIResource {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(SessionUIResource.class);
-
/** The open sessions. */
private static HashMap<UUID, LensSessionHandle> openSessions
= new HashMap<UUID, LensSessionHandle>();
@@ -137,7 +134,7 @@ public class SessionUIResource {
@Path("{publicId}")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN})
public APIResult closeSession(@PathParam("publicId") UUID publicId) {
- LOG.info("Closing session with id: " + publicId);
+ log.info("Closing session with id: {}", publicId);
LensSessionHandle sessionHandle = getOpenSession(publicId);
checkSessionHandle(sessionHandle);
openSessions.remove(publicId);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/main/java/org/apache/lens/server/ui/StaticFileResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/ui/StaticFileResource.java b/lens-server/src/main/java/org/apache/lens/server/ui/StaticFileResource.java
index 8563951..7e07589 100644
--- a/lens-server/src/main/java/org/apache/lens/server/ui/StaticFileResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/ui/StaticFileResource.java
@@ -31,8 +31,6 @@ import javax.ws.rs.core.Response;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import com.google.common.cache.CacheBuilder;
@@ -46,9 +44,6 @@ import com.google.common.io.Files;
@Path("/")
public class StaticFileResource {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(StaticFileResource.class);
-
// Cache for file content, bound by both size and time
/** The Constant CONTENT_CACHE. */
private static final LoadingCache<String, String> CONTENT_CACHE = CacheBuilder.newBuilder().maximumSize(100)
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
index 44aa0aa..2beb3b9 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
@@ -33,8 +33,6 @@ import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.Service;
import org.apache.hive.service.Service.STATE;
@@ -43,13 +41,14 @@ import org.glassfish.jersey.test.JerseyTest;
import org.testng.annotations.AfterSuite;
import org.testng.annotations.BeforeSuite;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Extend this class for unit testing Lens Jersey resources
*/
+@Slf4j
public abstract class LensJerseyTest extends JerseyTest {
- public static final Log LOG = LogFactory.getLog(LensJerseyTest.class);
-
private int port = -1;
private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
@@ -63,11 +62,11 @@ public abstract class LensJerseyTest extends JerseyTest {
}
public void setUp() throws Exception {
- LOG.info("setUp in class: " + this.getClass().getCanonicalName());
+ log.info("setUp in class: {}", this.getClass().getCanonicalName());
super.setUp();
}
public void tearDown() throws Exception {
- LOG.info("tearDown in class: " + this.getClass().getCanonicalName());
+ log.info("tearDown in class: {}", this.getClass().getCanonicalName());
super.tearDown();
}
protected int getTestPort() {
@@ -79,14 +78,14 @@ public abstract class LensJerseyTest extends JerseyTest {
socket = new ServerSocket(0);
setPort(socket.getLocalPort());
} catch (IOException e) {
- LOG.info("Exception occured while creating socket. Use a default port number " + port);
+ log.info("Exception occured while creating socket. Use a default port number {}", port);
} finally {
try {
if (socket != null) {
socket.close();
}
} catch (IOException e) {
- LOG.info("Exception occured while closing the socket ", e);
+ log.info("Exception occured while closing the socket", e);
}
}
return port;
@@ -112,7 +111,7 @@ public abstract class LensJerseyTest extends JerseyTest {
*/
@BeforeSuite
public void startAll() throws Exception {
- LOG.info("Before suite");
+ log.info("Before suite");
System.setProperty("lens.log.dir", "target/");
TestRemoteHiveDriver.createHS2Service();
System.out.println("Remote hive server started!");
@@ -142,7 +141,7 @@ public abstract class LensJerseyTest extends JerseyTest {
*/
@AfterSuite
public void stopAll() throws Exception {
- LOG.info("After suite");
+ log.info("After suite");
verifyMetrics();
LensServices.get().stop();
System.out.println("Lens services stopped!");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/TestResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestResource.java b/lens-server/src/test/java/org/apache/lens/server/TestResource.java
index da07a6c..bfc360b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestResource.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestResource.java
@@ -23,18 +23,12 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
/**
* The Class TestResource.
*/
@Path("/test")
public class TestResource {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(TestResource.class);
-
/**
* API to check if resource is up and running
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index a84fcd3..697e867 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -39,8 +39,6 @@ import org.apache.lens.server.query.TestQueryService;
import org.apache.lens.server.session.HiveSessionService;
import org.apache.lens.server.session.LensSessionImpl;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hive.service.Service;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
@@ -51,15 +49,15 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestServerRestart.
*/
@Test(alwaysRun = true, groups = "restart-test", dependsOnGroups = "unit-test")
+@Slf4j
public class TestServerRestart extends LensAllApplicationJerseyTest {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(TestServerRestart.class);
-
/** The data file. */
private File dataFile;
@@ -120,7 +118,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
*/
@Test
public void testQueryService() throws InterruptedException, IOException, LensException {
- LOG.info("Server restart test");
+ log.info("Server restart test");
QueryExecutionServiceImpl queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
LensSessionHandle lensSessionId = queryService.openSession("foo", "bar", new HashMap<String, String>());
@@ -130,7 +128,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
// Create a test table
LensTestUtil.createTable("test_server_restart", target(), lensSessionId);
LensTestUtil.loadData("test_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(), lensSessionId);
- LOG.info("Loaded data");
+ log.info("Loaded data");
// test post execute op
final WebTarget target = target().path("queryapi/queries");
@@ -144,9 +142,9 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
// Kill the query submitter thread to make sure some queries stay in accepted queue
try {
queryService.pauseQuerySubmitter();
- LOG.info("Stopped query submitter");
+ log.info("Stopped query submitter");
} catch (Exception exc) {
- LOG.error("Could not kill query submitter", exc);
+ log.error("Could not kill query submitter", exc);
}
killed = true;
}
@@ -166,25 +164,25 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
QueryStatus stat = ctx.getStatus();
- LOG.info(i + " submitted query " + handle + " state: " + ctx.getStatus().getStatus());
+ log.info("{} submitted query {} state: {}", i, handle, ctx.getStatus().getStatus());
launchedQueries.add(handle);
}
// Restart the server
- LOG.info("Restarting lens server!");
+ log.info("Restarting lens server!");
restartLensServer();
- LOG.info("Restarted lens server!");
+ log.info("Restarted lens server!");
queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
// All queries should complete after server restart
for (QueryHandle handle : launchedQueries) {
- LOG.info("Polling query " + handle);
+ log.info("Polling query {}", handle);
try {
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
QueryStatus stat = ctx.getStatus();
while (!stat.finished()) {
- LOG.info("Polling query " + handle + " Status:" + stat);
+ log.info("Polling query {} Status:{}", handle, stat);
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
stat = ctx.getStatus();
Thread.sleep(1000);
@@ -196,13 +194,13 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
List<String> rows = TestQueryService.readResultSet(resultset, handle, true);
Assert.assertEquals(rows.size(), 1);
Assert.assertEquals(rows.get(0), "" + NROWS);
- LOG.info("Completed " + handle);
+ log.info("Completed {}", handle);
} catch (Exception exc) {
- LOG.error("Failed query " + handle, exc);
+ log.error("Failed query {}", handle, exc);
Assert.fail(exc.getMessage());
}
}
- LOG.info("End server restart test");
+ log.info("End server restart test");
LensTestUtil.dropTable("test_server_restart", target(), lensSessionId);
queryService.closeSession(lensSessionId);
}
@@ -227,15 +225,15 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
HiveSessionService sessionService = (HiveSessionService) LensServices.get().getService(SessionService.NAME);
sessionService.addResource(lensSessionId, "FILE", dataFile.toURI().toString());
queryService.getSession(lensSessionId).addResource("FILE", dataFile.toURI().toString());
- LOG.info("@@ Added resource " + dataFile.toURI());
+ log.info("@@ Added resource {}", dataFile.toURI());
// Create a test table
LensTestUtil.createTable("test_hive_server_restart", target(), lensSessionId);
LensTestUtil.loadData("test_hive_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(),
lensSessionId);
- LOG.info("Loaded data");
+ log.info("Loaded data");
- LOG.info("Hive Server restart test");
+ log.info("Hive Server restart test");
// test post execute op
final WebTarget target = target().path("queryapi/queries");
@@ -260,24 +258,24 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
for (int i = 0; i < sessionResources.size(); i++) {
restoreCounts[i] = sessionResources.get(i).getRestoreCount();
}
- LOG.info("@@ Current counts " + Arrays.toString(restoreCounts));
+ log.info("@@ Current counts {}", Arrays.toString(restoreCounts));
// Restart hive server
TestRemoteHiveDriver.stopHS2Service();
// Wait for server to stop
while (TestRemoteHiveDriver.getServerState() != Service.STATE.STOPPED) {
- LOG.info("Waiting for HS2 to stop. Current state " + TestRemoteHiveDriver.getServerState());
+ log.info("Waiting for HS2 to stop. Current state {}", TestRemoteHiveDriver.getServerState());
Thread.sleep(1000);
}
TestRemoteHiveDriver.createHS2Service();
// Wait for server to come up
while (Service.STATE.STARTED != TestRemoteHiveDriver.getServerState()) {
- LOG.info("Waiting for HS2 to start " + TestRemoteHiveDriver.getServerState());
+ log.info("Waiting for HS2 to start {}", TestRemoteHiveDriver.getServerState());
Thread.sleep(1000);
}
Thread.sleep(10000);
- LOG.info("Server restarted");
+ log.info("Server restarted");
// Check params to be set
verifyParamOnRestart(lensSessionId);
@@ -288,14 +286,14 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
.get(LensQuery.class);
QueryStatus stat = ctx.getStatus();
while (!stat.finished()) {
- LOG.info("Polling query " + handle + " Status:" + stat);
+ log.info("Polling query {} Status:{}", handle, stat);
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
stat = ctx.getStatus();
Thread.sleep(1000);
}
Assert.assertTrue(stat.finished());
- LOG.info("Previous query status: " + stat.getStatusMessage());
+ log.info("Previous query status: {}", stat.getStatusMessage());
for (int i = 0; i < 5; i++) {
// Submit another query, again no exception expected
@@ -315,12 +313,12 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
stat = ctx.getStatus();
while (!stat.finished()) {
- LOG.info("Post restart polling query " + handle + " Status:" + stat);
+ log.info("Post restart polling query {} Status:{}", handle, stat);
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
stat = ctx.getStatus();
Thread.sleep(1000);
}
- LOG.info("@@ " + i + " Final status for " + handle + " " + stat.getStatus());
+ log.info("@@ {} Final status for {}: {}", i, handle, stat.getStatus());
}
// Now we can expect that session resources have been added back exactly once
@@ -328,12 +326,12 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
LensSessionImpl.ResourceEntry resourceEntry = sessionResources.get(i);
Assert.assertEquals(resourceEntry.getRestoreCount(), 1 + restoreCounts[i], "Restore test failed for "
+ resourceEntry + " pre count=" + restoreCounts[i] + " post count=" + resourceEntry.getRestoreCount());
- LOG.info("@@ Latest count " + resourceEntry + " = " + resourceEntry.getRestoreCount());
+ log.info("@@ Latest count {}={}", resourceEntry, resourceEntry.getRestoreCount());
}
// Assert.assertEquals(stat.getStatus(), QueryStatus.Status.SUCCESSFUL,
// "Expected to be successful " + handle);
- LOG.info("End hive server restart test");
+ log.info("End hive server restart test");
LensTestUtil.dropTable("test_hive_server_restart", target(), lensSessionId);
queryService.closeSession(lensSessionId);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
index 3fb123e..8d3ee60 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestStartupOnMetastoreDown.java
@@ -21,16 +21,15 @@ package org.apache.lens.server;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.testng.Assert;
-public class TestStartupOnMetastoreDown {
- private static final Log LOG = LogFactory.getLog(TestStartupOnMetastoreDown.class);
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
+public class TestStartupOnMetastoreDown {
private final LogSegregationContext logSegregationContext = new MappedDiagnosticLogSegregationContext();
// @Test
@@ -51,7 +50,7 @@ public class TestStartupOnMetastoreDown {
try {
services.stop();
} catch (Exception exc) {
- LOG.error("Error stopping services", exc);
+ log.error("Error stopping services", exc);
Assert.fail("services.stop() got unexpected exception " + exc);
}
Hive.closeCurrent();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/TestStaticFileResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestStaticFileResource.java b/lens-server/src/test/java/org/apache/lens/server/TestStaticFileResource.java
index cf3b579..f8c5bf2 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestStaticFileResource.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestStaticFileResource.java
@@ -30,9 +30,6 @@ import javax.ws.rs.core.UriBuilder;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.ui.UIApp;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import org.glassfish.jersey.filter.LoggingFilter;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
@@ -44,8 +41,6 @@ import org.testng.annotations.Test;
@Test(groups = "unit-test")
public class TestStaticFileResource extends LensJerseyTest {
- public static final Log LOG = LogFactory.getLog(TestStaticFileResource.class);
-
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 4eb52b9..22dfd55 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.log4j.BasicConfigurator;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.*;
@@ -73,7 +72,6 @@ public class TestMetastoreService extends LensJerseyTest {
@BeforeTest
public void setUp() throws Exception {
super.setUp();
- BasicConfigurator.configure();
cubeObjectFactory = new ObjectFactory();
metastoreService = (CubeMetastoreServiceImpl) LensServices.get().getService("metastore");
lensSessionId = metastoreService.openSession("foo", "bar", new HashMap<String, String>());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java b/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
index 118970d..1747d03 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metrics/TestResourceMethodMetrics.java
@@ -49,8 +49,6 @@ import org.apache.lens.server.common.TestResourceFile;
import org.apache.lens.server.metastore.CubeMetastoreServiceImpl;
import org.apache.lens.server.query.TestQueryService;
-import org.apache.log4j.BasicConfigurator;
-
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -61,7 +59,10 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
@Test(groups = "unit-test")
+@Slf4j
public class TestResourceMethodMetrics extends LensAllApplicationJerseyTest {
private CubeMetastoreServiceImpl metastoreService;
private MetricsServiceImpl metricsSvc;
@@ -77,7 +78,6 @@ public class TestResourceMethodMetrics extends LensAllApplicationJerseyTest {
@BeforeTest
public void setUp() throws Exception {
super.setUp();
- BasicConfigurator.configure();
metricsSvc = (MetricsServiceImpl) LensServices.get().getService(MetricsService.NAME);
metastoreService = (CubeMetastoreServiceImpl) LensServices.get().getService(CubeMetastoreServiceImpl.NAME);
lensSessionId = metastoreService.openSession("foo", "bar", new HashMap<String, String>());
@@ -123,16 +123,16 @@ public class TestResourceMethodMetrics extends LensAllApplicationJerseyTest {
disableResourceMethodMetering();
metricsSvc.setEnableResourceMethodMetering(true);
Assert.assertEquals(methodMetricsMap.size(), 0);
- LOG.info("database operations");
+ log.info("database operations");
databaseOperations();
Assert.assertEquals(methodMetricsMap.size(), 3);
- LOG.info("create table");
+ log.info("create table");
createTable(TestQueryService.TEST_TABLE);
Assert.assertEquals(methodMetricsMap.size(), 5);
- LOG.info("load data");
+ log.info("load data");
loadData(TestQueryService.TEST_TABLE, TestResourceFile.TEST_DATA2_FILE.getValue());
Assert.assertEquals(methodMetricsMap.size(), 5);
- LOG.info("execute async");
+ log.info("execute async");
executeAsync();
verifyValues();
makeClientError();
@@ -149,7 +149,7 @@ public class TestResourceMethodMetrics extends LensAllApplicationJerseyTest {
fail("Should get 404");
} catch (NotAllowedException e) {
// expected
- LOG.error(e);
+ log.error("Not found excepiton:", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
index a77378b..b8012be 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
@@ -43,20 +43,18 @@ import org.apache.lens.server.api.session.SessionRestored;
import org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger;
import org.apache.lens.server.stats.event.query.QueryExecutionStatistics;
-import org.apache.log4j.Logger;
-
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestEventService.
*/
@Test(groups = "unit-test")
+@Slf4j
public class TestEventService {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(TestEventService.class);
-
/** The service. */
EventServiceImpl service;
@@ -110,7 +108,7 @@ public class TestEventService {
public void process(LensEvent event) {
processed = true;
latch.countDown();
- LOG.info("LensEvent:" + event.getEventId());
+ log.info("LensEvent: {}", event.getEventId());
}
}
@@ -137,7 +135,7 @@ public class TestEventService {
public void onEvent(QueryFailed change) throws LensException {
processed = true;
latch.countDown();
- LOG.info("Query Failed event: " + change);
+ log.info("Query Failed event: {}", change);
}
}
@@ -164,7 +162,7 @@ public class TestEventService {
public void onEvent(QueryEnded change) throws LensException {
processed = true;
latch.countDown();
- LOG.info("Query ended event: " + change);
+ log.info("Query ended event: {}", change);
}
}
@@ -185,7 +183,7 @@ public class TestEventService {
public void onEvent(QueuePositionChange change) throws LensException {
processed = true;
latch.countDown();
- LOG.info("Query position changed: " + change);
+ log.info("Query position changed: {}", change);
}
}
@@ -206,7 +204,7 @@ public class TestEventService {
public void onEvent(SessionOpened event) throws LensException {
processed = true;
latch.countDown();
- LOG.info("Session opened: " + event);
+ log.info("Session opened: {}", event);
}
}
@@ -227,7 +225,7 @@ public class TestEventService {
public void onEvent(SessionClosed event) throws LensException {
processed = true;
latch.countDown();
- LOG.info("Session closed: " + event);
+ log.info("Session closed: {}", event);
}
}
@@ -248,7 +246,7 @@ public class TestEventService {
public void onEvent(SessionExpired event) throws LensException {
processed = true;
latch.countDown();
- LOG.info("Session expired: " + event);
+ log.info("Session expired: {}", event);
}
}
@@ -269,7 +267,7 @@ public class TestEventService {
public void onEvent(SessionRestored event) throws LensException {
processed = true;
latch.countDown();
- LOG.info("Session restored: " + event);
+ log.info("Session restored: {}", event);
}
}
@@ -284,7 +282,7 @@ public class TestEventService {
LensServices.get().start();
service = LensServices.get().getService(LensEventService.NAME);
assertNotNull(service);
- LOG.info("Service started " + service);
+ log.info("Service started {}", service);
}
/**
@@ -369,9 +367,9 @@ public class TestEventService {
try {
latch = new CountDownLatch(3);
- LOG.info("Sending session opened event: " + sessionOpenedEvent);
+ log.info("Sending session opened event: {}", sessionOpenedEvent);
service.notifyEvent(sessionOpenedEvent);
- LOG.info("Sending session restored event: " + sessionRestored);
+ log.info("Sending session restored event: {}", sessionRestored);
service.notifyEvent(sessionRestored);
latch.await(5, TimeUnit.SECONDS);
assertTrue(genericEventListener.processed);
@@ -387,16 +385,16 @@ public class TestEventService {
};
latch = new CountDownLatch(2);
- LOG.info("Sending generic event " + genEvent.getEventId());
+ log.info("Sending generic event {}", genEvent.getEventId());
service.notifyEvent(genEvent);
latch.await(5, TimeUnit.SECONDS);
assertTrue(genericEventListener.processed);
resetSessionListeners();
latch = new CountDownLatch(3);
- LOG.info("Sending session closed event " + sessionClosedEvent);
+ log.info("Sending session closed event {}", sessionClosedEvent);
service.notifyEvent(sessionClosedEvent);
- LOG.info("Sending session expired event " + sessionExpired);
+ log.info("Sending session expired event {}", sessionExpired);
service.notifyEvent(sessionExpired);
latch.await(5, TimeUnit.SECONDS);
assertTrue(sessionClosedListener.processed);
@@ -425,7 +423,7 @@ public class TestEventService {
try {
latch = new CountDownLatch(3);
- LOG.info("Sending event: " + failed);
+ log.info("Sending event: {}", failed);
service.notifyEvent(failed);
latch.await(5, TimeUnit.SECONDS);
assertTrue(genericEventListener.processed);
@@ -435,7 +433,7 @@ public class TestEventService {
resetListeners();
latch = new CountDownLatch(2);
- LOG.info("Sending event : " + success);
+ log.info("Sending event : {}", success);
service.notifyEvent(success);
latch.await(5, TimeUnit.SECONDS);
assertTrue(genericEventListener.processed);
@@ -445,7 +443,7 @@ public class TestEventService {
resetListeners();
latch = new CountDownLatch(2);
- LOG.info("Sending event: " + positionChange);
+ log.info("Sending event: {}", positionChange);
service.notifyEvent(positionChange);
latch.await(5, TimeUnit.SECONDS);
assertTrue(genericEventListener.processed);
@@ -462,7 +460,7 @@ public class TestEventService {
};
latch = new CountDownLatch(1);
- LOG.info("Sending generic event " + genEvent.getEventId());
+ log.info("Sending generic event {}", genEvent.getEventId());
service.notifyEvent(genEvent);
latch.await(5, TimeUnit.SECONDS);
assertTrue(genericEventListener.processed);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
index f3f2cac..bdd49f5 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
@@ -44,10 +44,13 @@ import org.codehaus.jackson.map.ObjectMapper;
import org.testng.Assert;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestLensDAO.
*/
@Test(groups = "unit-test")
+@Slf4j
public class TestLensDAO extends LensJerseyTest {
/**
@@ -85,12 +88,12 @@ public class TestLensDAO extends LensJerseyTest {
String jsonMetadata = MAPPER.writeValueAsString(jdbcRsMeta);
- LOG.info("@@@JSON " + jsonMetadata);
+ log.info("@@@JSON {}" + jsonMetadata);
finishedLensQuery.setMetadata(MAPPER.writeValueAsString(jdbcRsMeta));
finishedLensQuery.setMetadataClass(JDBCResultSet.JDBCResultSetMetadata.class.getName());
} catch (SQLException ex) {
- LOG.error("Error creating result set ", ex);
+ log.error("Error creating result set ", ex);
} finally {
if (stmt != null) {
stmt.close();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
index 0b152bf..f1461bc 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
@@ -42,8 +42,6 @@ import org.apache.lens.server.LensTestUtil;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.common.TestResourceFile;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.glassfish.jersey.client.ClientConfig;
@@ -64,8 +62,6 @@ import org.testng.annotations.Test;
@Test(groups = "unit-test")
public class TestQueryEndEmailNotifier extends LensJerseyTest {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(TestQueryEndEmailNotifier.class);
private static final int NUM_ITERS = 30;
/** The query service. */
QueryExecutionServiceImpl queryService;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
index ea81e90..c17b28b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
@@ -57,10 +57,13 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class TestResultFormatting.
*/
@Test(groups = "unit-test")
+@Slf4j
public class TestResultFormatting extends LensJerseyTest {
/** The query service. */
@@ -235,9 +238,9 @@ public class TestResultFormatting extends LensJerseyTest {
if (qctx == null) {
// This shouldn't occur. It is appearing when query gets purged. So adding extra logs
// for debugging in the future.
- LOG.info("successful query's QueryContext is null");
- LOG.info("query handle: " + handle);
- LOG.info("allQueries: " + queryService.allQueries);
+ log.info("successful query's QueryContext is null");
+ log.info("query handle: {}", handle);
+ log.info("allQueries: {}", queryService.allQueries);
// not doing formatter validation if qctx is null
} else if (!isDir) {
// isDir is true if the formatter is skipped due to result being the max size allowed
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
index 9a79e65..464533d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
+++ b/lens-server/src/test/java/org/apache/lens/server/session/TestSessionClassLoaders.java
@@ -29,8 +29,6 @@ import org.apache.lens.server.LensTestUtil;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.user.UserConfigLoaderFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hive.service.cli.CLIService;
@@ -40,8 +38,10 @@ import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class TestSessionClassLoaders {
- private static final Log LOG = LogFactory.getLog(TestSessionClassLoaders.class);
private final HiveConf conf = LensServerConf.createHiveConf();
private HiveSessionService sessionService;
@@ -119,7 +119,7 @@ public class TestSessionClassLoaders {
sessionService.release(sessionHandle);
}
- LOG.info("@@ Teting DB2");
+ log.info("@@ Teting DB2");
// Loading test class should pass for DB1
session.setCurrentDatabase(DB1);
@@ -136,7 +136,7 @@ public class TestSessionClassLoaders {
clz = session.getCubeMetastoreClient().getConf().getClassByName(TestDatabaseResourceService.TEST_CLASS);
Assert.assertNotNull(clz);
} catch (ClassNotFoundException cnf) {
- LOG.error(cnf.getMessage(), cnf);
+ log.error(cnf.getMessage(), cnf);
Assert.fail("Should not have thrown class not found exception: " + cnf.getMessage());
} finally {
sessionService.release(sessionHandle);
@@ -166,7 +166,7 @@ public class TestSessionClassLoaders {
boolean loadedSessionClass = false;
boolean loadedDBClass = false;
try {
- LOG.info("@@@ TEST 1");
+ log.info("@@@ TEST 1");
sessionService.acquire(sessionHandle);
ClassLoader dbClassLoader = session.getClassLoader("default");
@@ -181,7 +181,7 @@ public class TestSessionClassLoaders {
Class clz = Class.forName("ClassLoaderTestClass", true, Thread.currentThread().getContextClassLoader());
loadedDBClass = true;
} catch (ClassNotFoundException cnf) {
- LOG.error(cnf.getMessage(), cnf);
+ log.error(cnf.getMessage(), cnf);
Assert.assertTrue(loadedSessionClass);
Assert.assertFalse(loadedDBClass);
} finally {
@@ -192,7 +192,7 @@ public class TestSessionClassLoaders {
loadedSessionClass = false;
loadedDBClass = false;
try {
- LOG.info("@@@ TEST 1 - cube client");
+ log.info("@@@ TEST 1 - cube client");
sessionService.acquire(sessionHandle);
// testClass2 should be loaded since test2.jar is added to the session
@@ -204,14 +204,14 @@ public class TestSessionClassLoaders {
Class clz = session.getCubeMetastoreClient().getConf().getClassByName("ClassLoaderTestClass");
loadedDBClass = true;
} catch (ClassNotFoundException cnf) {
- LOG.error(cnf.getMessage(), cnf);
+ log.error(cnf.getMessage(), cnf);
Assert.assertTrue(loadedSessionClass);
Assert.assertFalse(loadedDBClass);
} finally {
sessionService.release(sessionHandle);
}
- LOG.info("@@@ TEST 2");
+ log.info("@@@ TEST 2");
session.setCurrentDatabase(DB1);
loadedSessionClass = false;
loadedDBClass = false;
@@ -230,7 +230,7 @@ public class TestSessionClassLoaders {
Assert.assertTrue(loadedSessionClass);
Assert.assertTrue(loadedDBClass);
- LOG.info("@@@ TEST 2 - cube client");
+ log.info("@@@ TEST 2 - cube client");
loadedSessionClass = false;
loadedDBClass = false;
try {
@@ -248,7 +248,7 @@ public class TestSessionClassLoaders {
Assert.assertTrue(loadedDBClass);
// Switch back to default DB, again the test2.jar should be available, test.jar should not be available
- LOG.info("@@@ TEST 3");
+ log.info("@@@ TEST 3");
session.setCurrentDatabase("default");
loadedSessionClass = false;
loadedDBClass = false;
@@ -267,7 +267,7 @@ public class TestSessionClassLoaders {
sessionService.release(sessionHandle);
}
- LOG.info("@@@ TEST 3 -- cube client");
+ log.info("@@@ TEST 3 -- cube client");
session.setCurrentDatabase("default");
loadedSessionClass = false;
loadedDBClass = false;
@@ -282,7 +282,7 @@ public class TestSessionClassLoaders {
Class clz = session.getCubeMetastoreClient().getConf().getClassByName("ClassLoaderTestClass");
loadedDBClass = true;
} catch (ClassNotFoundException cnf) {
- LOG.error(cnf.getMessage(), cnf);
+ log.error(cnf.getMessage(), cnf);
Assert.assertTrue(loadedSessionClass);
Assert.assertFalse(loadedDBClass);
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44ec2afd/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
index a122d1e..1656f4e 100644
--- a/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
+++ b/lens-server/src/test/java/org/apache/lens/server/user/TestUserConfigLoader.java
@@ -29,8 +29,6 @@ import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.dbcp.BasicDataSource;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.hsqldb.server.Server;
@@ -49,7 +47,6 @@ import liquibase.resource.FileSystemResourceAccessor;
*/
@Test(groups = "unit-test")
public class TestUserConfigLoader {
- public static final Log LOG = LogFactory.getLog(TestUserConfigLoader.class);
/** The conf. */
private HiveConf conf;
[11/50] [abbrv] incubator-lens git commit: LENS-24 : Add support for
User Query Rewriter chain before Cube Rewrites
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index b4e63bc..c5bf70c 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -125,94 +125,98 @@ Lens server configuration
*--+--+---+--+
|49|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.|
*--+--+---+--+
-|50|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
+|50|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ |
+| | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ |
+| | | |Or define shortcuts for certain frequently used queries :) |
*--+--+---+--+
-|51|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
+|51|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
*--+--+---+--+
-|52|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
+|52|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
*--+--+---+--+
-|53|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
+|53|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
*--+--+---+--+
-|54|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
+|54|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
*--+--+---+--+
-|55|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
+|55|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
*--+--+---+--+
-|56|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
+|56|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
*--+--+---+--+
-|57|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
+|57|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
*--+--+---+--+
-|58|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
+|58|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
*--+--+---+--+
-|59|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
+|59|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
*--+--+---+--+
-|60|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
+|60|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
*--+--+---+--+
-|61|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
+|61|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
*--+--+---+--+
-|62|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
+|62|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|63|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|63|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
*--+--+---+--+
-|64|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|64|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|65|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|65|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|66|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
+|66|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|67|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|67|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
*--+--+---+--+
-|68|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|68|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|69|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|69|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|70|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|70|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|71|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|71|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|72|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
+|72|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|73|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
+|73|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
*--+--+---+--+
-|74|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|74|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
*--+--+---+--+
-|75|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
+|75|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|76|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|76|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|77|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|77|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|78|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|78|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|79|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|79|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|80|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|80|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|81|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|81|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|82|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|82|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|83|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|83|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|84|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|84|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|85|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|85|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|86|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|86|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|87|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|87|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|88|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|88|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|89|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|89|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|90|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|90|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|91|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|91|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
-|92|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|92|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|93|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|93|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|94|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|94|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+*--+--+---+--+
+|95|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values
[09/50] [abbrv] incubator-lens git commit: LENS-387: Propagate error
for failing in QueryHandleWithResultSet in EXECUTE_WITH_TIMEOUT queries
Posted by sh...@apache.org.
LENS-387: Propagate error for failing in QueryHandleWithResultSet in EXECUTE_WITH_TIMEOUT queries
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/7f4d793f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/7f4d793f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/7f4d793f
Branch: refs/heads/LENS-581
Commit: 7f4d793fb12333817b57fb57c1f9ffa272d6435f
Parents: f1387c3
Author: Tushar Ghosh <tu...@inmobi.com>
Authored: Mon Jul 13 15:10:03 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Jul 13 15:10:03 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/api/query/QueryHandleWithResultSet.java | 7 +++++++
.../apache/lens/server/query/QueryExecutionServiceImpl.java | 4 ++++
.../java/org/apache/lens/server/query/TestQueryService.java | 5 +----
3 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7f4d793f/lens-api/src/main/java/org/apache/lens/api/query/QueryHandleWithResultSet.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandleWithResultSet.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandleWithResultSet.java
index 4fd5081..a5da867 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryHandleWithResultSet.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryHandleWithResultSet.java
@@ -54,6 +54,13 @@ public class QueryHandleWithResultSet extends QuerySubmitResult {
private QueryResult result;
/**
+ * The status.
+ */
+ @Getter
+ @Setter
+ private QueryStatus status;
+
+ /**
* Instantiates a new query handle with result set.
*
* @param handle the handle
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7f4d793f/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 8a32202..a778a1f 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1749,6 +1749,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
QueryCompletionListener listener = new QueryCompletionListenerImpl(handle);
if (getQueryContext(sessionHandle, handle).getSelectedDriver() == null) {
+ result.setStatus(getQueryContext(sessionHandle, handle).getStatus());
return result;
}
synchronized (ctx) {
@@ -1770,6 +1771,9 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
result.setResult(getResultset(handle).toQueryResult());
}
}
+
+ result.setStatus(getQueryContext(sessionHandle, handle).getStatus());
+
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7f4d793f/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 76d3dee..59066f9 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -1230,10 +1230,7 @@ public class TestQueryService extends LensJerseyTest {
new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
Assert.assertNotNull(result.getQueryHandle());
Assert.assertNull(result.getResult());
-
- LensQuery ctx = target.path(result.getQueryHandle().toString()).queryParam("sessionid", lensSessionId).request()
- .get(LensQuery.class);
- Assert.assertEquals(ctx.getStatus().getStatus(), Status.FAILED);
+ Assert.assertEquals(result.getStatus().getStatus(), Status.FAILED);
}
/**
[27/50] [abbrv] incubator-lens git commit: LENS-679 : Use slf4j in
api, regression, ml and client modules
Posted by sh...@apache.org.
LENS-679 : Use slf4j in api, regression, ml and client modules
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/90415a10
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/90415a10
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/90415a10
Branch: refs/heads/LENS-581
Commit: 90415a10db1284943e2b042fb2f81464beb8207d
Parents: 5e70bee
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Wed Jul 22 12:33:03 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed Jul 22 12:33:03 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/client/LensClient.java | 47 +++++-----
.../lens/client/LensClientSingletonWrapper.java | 9 +-
.../org/apache/lens/client/LensConnection.java | 16 ++--
.../apache/lens/client/LensMetadataClient.java | 96 ++++++++++----------
.../apache/lens/client/LensMLJerseyClient.java | 10 +-
.../lens/ml/algo/spark/BaseSparkAlgo.java | 14 ++-
.../ml/algo/spark/ColumnFeatureFunction.java | 10 +-
.../apache/lens/ml/algo/spark/HiveTableRDD.java | 4 -
.../lens/ml/algo/spark/SparkMLDriver.java | 28 +++---
.../lens/ml/algo/spark/TableTrainingSpec.java | 20 ++--
.../java/org/apache/lens/ml/impl/HiveMLUDF.java | 12 +--
.../java/org/apache/lens/ml/impl/MLRunner.java | 15 ++-
.../java/org/apache/lens/ml/impl/MLTask.java | 21 ++---
.../org/apache/lens/ml/impl/ModelLoader.java | 24 +++--
.../apache/lens/ml/impl/TableTestingSpec.java | 17 ++--
.../apache/lens/ml/server/MLServiceImpl.java | 16 ++--
.../lens/ml/server/MLServiceResource.java | 14 ++-
.../java/org/apache/lens/rdd/LensRDDClient.java | 19 ++--
.../java/org/apache/lens/ml/ExampleUtils.java | 11 +--
.../java/org/apache/lens/ml/TestMLRunner.java | 3 -
lens-regression/pom.xml | 4 -
.../core/helpers/LensServerHelper.java | 10 +-
.../core/helpers/MetastoreHelper.java | 12 +--
.../regression/core/helpers/QueryHelper.java | 60 ++++++------
.../core/helpers/ServiceManagerHelper.java | 10 +-
.../regression/core/helpers/SessionHelper.java | 24 ++---
.../apache/lens/regression/util/AssertUtil.java | 7 --
.../org/apache/lens/regression/util/Util.java | 20 ++--
.../lens/regression/sanity/ITSmokeTest.java | 23 +++--
.../server/api/driver/MinQueryCostSelector.java | 2 -
.../api/metrics/MethodMetricsFactory.java | 9 +-
.../server/api/query/AbstractQueryContext.java | 15 +--
.../api/query/DriverSelectorQueryContext.java | 12 +--
33 files changed, 269 insertions(+), 345 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index d7df1ba..9ff2124 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -38,12 +38,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.google.common.collect.Maps;
+
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class LensClient {
- private static final Log LOG = LogFactory.getLog(LensClient.class);
-
public static final String CLILOGGER = "cliLogger";
private static final String DEFAULT_PASSWORD = "";
private final LensClientConfig conf;
@@ -93,10 +94,10 @@ public class LensClient {
}
public LensAPIResult<QueryHandle> executeQueryAsynch(String sql, String queryName) throws LensAPIException {
- LOG.debug("Executing query " + sql);
+ log.debug("Executing query {}", sql);
LensAPIResult<QueryHandle> lensAPIResult = statement.execute(sql, false, queryName);
LensQuery query = statement.getQuery();
- LOG.debug("Adding query to statementMap " + query.getQueryHandle());
+ log.debug("Adding query to statementMap {}", query.getQueryHandle());
statementMap.put(query.getQueryHandle(), statement);
return lensAPIResult;
}
@@ -130,7 +131,7 @@ public class LensClient {
}
public LensClientResultSetWithStats getResults(String sql, String queryName) throws LensAPIException {
- LOG.debug("Executing query " + sql);
+ log.debug("Executing query {}", sql);
statement.execute(sql, true, queryName);
return getResultsFromStatement(statement);
}
@@ -213,7 +214,7 @@ public class LensClient {
public QueryResult getResults(QueryHandle query) {
QueryStatus status = getLensStatement(query).getStatus();
if (!status.isResultSetAvailable()) {
- LOG.debug("Current status of the query is " + status);
+ log.debug("Current status of the query is {}", status);
throw new IllegalStateException("Resultset for the query "
+ query + " is not available, its current status is " + status);
}
@@ -226,76 +227,76 @@ public class LensClient {
private void connectToLensServer() {
- LOG.debug("Connecting to lens server " + new LensConnectionParams(conf));
+ log.debug("Connecting to lens server {}", new LensConnectionParams(conf));
connection = new LensConnection(new LensConnectionParams(conf));
connection.open(password);
- LOG.debug("Successfully connected to server " + connection);
+ log.debug("Successfully connected to server {}", connection);
}
public List<String> getAllDatabases() {
- LOG.debug("Getting all database");
+ log.debug("Getting all database");
return mc.getAlldatabases();
}
public List<String> getAllNativeTables() {
- LOG.debug("Getting all native tables");
+ log.debug("Getting all native tables");
return mc.getAllNativeTables();
}
public List<String> getAllFactTables() {
- LOG.debug("Getting all fact table");
+ log.debug("Getting all fact table");
return mc.getAllFactTables();
}
public List<String> getAllFactTables(String cubeName) {
- LOG.debug("Getting all fact table");
+ log.debug("Getting all fact table");
return mc.getAllFactTables(cubeName);
}
public List<String> getAllDimensionTables() {
- LOG.debug("Getting all dimension table");
+ log.debug("Getting all dimension table");
return mc.getAllDimensionTables();
}
public List<String> getAllDimensionTables(String dimensionName) {
- LOG.debug("Getting all dimension table");
+ log.debug("Getting all dimension table");
return mc.getAllDimensionTables(dimensionName);
}
public List<String> getAllCubes() {
- LOG.debug("Getting all cubes in database");
+ log.debug("Getting all cubes in database");
return mc.getAllCubes();
}
public List<String> getAllDimensions() {
- LOG.debug("Getting all dimensions in database");
+ log.debug("Getting all dimensions in database");
return mc.getAllDimensions();
}
public String getCurrentDatabae() {
- LOG.debug("Getting current database");
+ log.debug("Getting current database");
return mc.getCurrentDatabase();
}
public boolean setDatabase(String database) {
- LOG.debug("Set the database to " + database);
+ log.debug("Set the database to {}", database);
APIResult result = mc.setDatabase(database);
return result.getStatus() == APIResult.Status.SUCCEEDED;
}
public APIResult dropDatabase(String database, boolean cascade) {
- LOG.debug("Dropping database " + database + ", cascade: " + cascade);
+ log.debug("Dropping database {}, cascade: {}", database, cascade);
APIResult result = mc.dropDatabase(database, cascade);
- LOG.debug("Return status of dropping " + database + " result " + result);
+ log.debug("Return status of dropping {} result {}", database, result);
return result;
}
public APIResult createDatabase(String database, boolean ignoreIfExists) {
- LOG.debug("Creating database " + database + " ignore " + ignoreIfExists);
+ log.debug("Creating database {} ignore {}", database, ignoreIfExists);
APIResult result = mc.createDatabase(database, ignoreIfExists);
- LOG.debug("Create database result " + result);
+ log.debug("Create database result {}", result);
return result;
}
@@ -312,7 +313,7 @@ public class LensClient {
}
public APIResult closeConnection() {
- LOG.debug("Closing lens connection: " + new LensConnectionParams(conf));
+ log.debug("Closing lens connection: {}", new LensConnectionParams(conf));
return this.connection.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
index 14a0690..4629559 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClientSingletonWrapper.java
@@ -20,12 +20,12 @@ package org.apache.lens.client;
import org.apache.lens.client.exceptions.LensClientServerConnectionException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import lombok.extern.slf4j.Slf4j;
/**
* The Enum LensClientSingletonWrapper.
*/
+@Slf4j
public class LensClientSingletonWrapper {
/** The instance. */
@@ -37,9 +37,6 @@ public class LensClientSingletonWrapper {
return InstanceHolder.INSTANCE;
}
- /** The log. */
- private static final Log LOG = LogFactory.getLog(LensClientSingletonWrapper.class);
-
/** The client. */
private LensClient client;
@@ -58,7 +55,7 @@ public class LensClientSingletonWrapper {
* @param e the e
*/
public void explainFailedAttempt(LensClientServerConnectionException e) {
- LOG.error("failed login attempt", e);
+ log.error("failed login attempt", e);
switch (e.getErrorCode()) {
case 401:
System.console().printf("username/password combination incorrect.\n");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
index 61e16f7..10819ea 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java
@@ -35,24 +35,20 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
import org.apache.lens.client.exceptions.LensClientServerConnectionException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
/**
* Top level client connection class which is used to connect to a lens server.
*/
+@Slf4j
public class LensConnection {
- /** The Constant LOG. */
- private static final Log LOG = LogFactory.getLog(LensConnection.class);
-
/** The params. */
private final LensConnectionParams params;
@@ -148,7 +144,7 @@ public class LensConnection {
final LensSessionHandle handle = response.readEntity(LensSessionHandle.class);
if (handle != null) {
sessionHandle = handle;
- LOG.debug("Created a new session " + sessionHandle.getPublicId());
+ log.debug("Created a new session {}", sessionHandle.getPublicId());
} else {
throw new IllegalStateException("Unable to connect to lens " + "server with following paramters" + params);
}
@@ -158,7 +154,7 @@ public class LensConnection {
}
}
- LOG.debug("Successfully switched to database " + params.getDbName());
+ log.debug("Successfully switched to database {}", params.getDbName());
open.set(true);
return sessionHandle;
@@ -189,7 +185,7 @@ public class LensConnection {
if (result.getStatus() != APIResult.Status.SUCCEEDED) {
throw new IllegalStateException("Unable to close lens connection " + "with params " + params);
}
- LOG.debug("Lens connection closed.");
+ log.debug("Lens connection closed.");
return result;
}
@@ -258,7 +254,7 @@ public class LensConnection {
MediaType.APPLICATION_XML_TYPE));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("key").build(), key));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("value").build(), value));
- LOG.debug("Setting connection params " + key + "=" + value);
+ log.debug("Setting connection params {}={}", key, value);
APIResult result = target.path("params").request()
.put(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
return result;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index dba13ca..4bec6ad 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -42,18 +42,16 @@ import org.apache.lens.api.DateTime;
import org.apache.lens.api.StringList;
import org.apache.lens.api.metastore.*;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import org.glassfish.jersey.media.multipart.*;
import org.xml.sax.SAXException;
import com.google.common.base.Joiner;
import com.google.common.io.Files;
-public class LensMetadataClient {
+import lombok.extern.slf4j.Slf4j;
- private static final Log LOG = LogFactory.getLog(LensMetadataClient.class);
+@Slf4j
+public class LensMetadataClient {
private final LensConnection connection;
private final LensConnectionParams params;
@@ -66,13 +64,13 @@ public class LensMetadataClient {
ClassLoader classLoader = LensMetadataClient.class.getClassLoader();
SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
URI uri = classLoader.getResource("cube-0.1.xsd").toURI();
- LOG.info("URI for cube schema: " + uri.toString());
+ log.info("URI for cube schema: {}", uri.toString());
Schema schema = sf.newSchema(uri.toURL());
JAXBContext jaxbContext = JAXBContext.newInstance(ObjectFactory.class);
JAXB_UNMARSHALLER = jaxbContext.createUnmarshaller();
JAXB_UNMARSHALLER.setSchema(schema);
} catch (JAXBException | URISyntaxException | MalformedURLException | SAXException e) {
- LOG.error("Could not initialize JAXBContext. ", e);
+ log.error("Could not initialize JAXBContext. ", e);
throw new RuntimeException("Could not initialize JAXBContext. ", e);
}
}
@@ -207,10 +205,10 @@ public class LensMetadataClient {
try {
return createCube(this.<XCube>readFromXML(cubeSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -228,10 +226,10 @@ public class LensMetadataClient {
try {
return updateCube(cubeName, this.<XCube>readFromXML(cubeSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -301,10 +299,10 @@ public class LensMetadataClient {
try {
return createDimension(this.<XDimension>readFromXML(dimSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -322,10 +320,10 @@ public class LensMetadataClient {
try {
return updateDimension(dimName, this.<XDimension>readFromXML(dimSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -370,10 +368,10 @@ public class LensMetadataClient {
try {
return createNewStorage(this.<XStorage>readFromXML(storage));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -400,10 +398,10 @@ public class LensMetadataClient {
try {
return updateStorage(storageName, this.<XStorage>readFromXML(storage));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -492,10 +490,10 @@ public class LensMetadataClient {
try {
return createFactTable(this.<XFactTable>readFromXML(factSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -514,10 +512,10 @@ public class LensMetadataClient {
}
return Joiner.on("\n").skipNulls().join(content);
} catch (IOException e) {
- LOG.error("IO Error reading content ", e);
+ log.error("IO Error reading content ", e);
throw new IllegalStateException(e);
} catch (URISyntaxException e) {
- LOG.error("URI Error reading content ", e);
+ log.error("URI Error reading content ", e);
throw new IllegalStateException(e);
}
}
@@ -536,10 +534,10 @@ public class LensMetadataClient {
try {
return updateFactTable(factName, this.<XFactTable>readFromXML(table));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -589,10 +587,10 @@ public class LensMetadataClient {
try {
return addStorageToFactTable(factname, this.<XStorageTableElement>readFromXML(storageSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -706,10 +704,10 @@ public class LensMetadataClient {
try {
return createDimensionTable(this.<XDimensionTable>readFromXML(tableXml));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -731,10 +729,10 @@ public class LensMetadataClient {
dimensionTable.setTableName(dimTblName);
return updateDimensionTable(dimensionTable);
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
@@ -787,10 +785,10 @@ public class LensMetadataClient {
try {
return addStorageToDimTable(dimTblName, this.<XStorageTableElement>readFromXML(table));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -888,10 +886,10 @@ public class LensMetadataClient {
try {
return addPartitionToDimensionTable(dimTblName, storage, (XPartition) readFromXML(partitionSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -912,10 +910,10 @@ public class LensMetadataClient {
try {
return addPartitionsToDimensionTable(dimTblName, storage, (XPartitionList) readFromXML(partitionsSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -936,10 +934,10 @@ public class LensMetadataClient {
try {
return addPartitionToFactTable(fact, storage, (XPartition) readFromXML(partitionSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -960,10 +958,10 @@ public class LensMetadataClient {
try {
return addPartitionsToFactTable(fact, storage, (XPartitionList) readFromXML(partitionsSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -983,10 +981,10 @@ public class LensMetadataClient {
try {
return updatePartitionOfDimensionTable(dimTblName, storage, (XPartition) readFromXML(partitionSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -1007,10 +1005,10 @@ public class LensMetadataClient {
try {
return updatePartitionsOfDimensionTable(dimTblName, storage, (XPartitionList) readFromXML(partitionsSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -1031,10 +1029,10 @@ public class LensMetadataClient {
try {
return updatePartitionOfFactTable(fact, storage, (XPartition) readFromXML(partitionSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
@@ -1055,10 +1053,10 @@ public class LensMetadataClient {
try {
return updatePartitionsOfFactTable(fact, storage, (XPartitionList) readFromXML(partitionsSpec));
} catch (JAXBException e) {
- LOG.info("Unmarshalling error:", e);
+ log.error("Unmarshalling error:", e);
return new APIResult(Status.FAILED, "Unmarshalling failed");
} catch (IOException e) {
- LOG.info("File error:", e);
+ log.error("File error:", e);
return new APIResult(Status.FAILED, "File not found");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
index c68dd12..875e692 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
@@ -34,8 +34,6 @@ import org.apache.lens.api.StringList;
import org.apache.lens.ml.api.ModelMetadata;
import org.apache.lens.ml.api.TestReport;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
@@ -43,6 +41,8 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import lombok.extern.slf4j.Slf4j;
+
/*
* Client code to invoke server side ML API
*/
@@ -50,6 +50,7 @@ import org.glassfish.jersey.media.multipart.MultiPartFeature;
/**
* The Class LensMLJerseyClient.
*/
+@Slf4j
public class LensMLJerseyClient {
/** The Constant LENS_ML_RESOURCE_PATH. */
public static final String LENS_ML_RESOURCE_PATH = "lens.ml.resource.path";
@@ -57,9 +58,6 @@ public class LensMLJerseyClient {
/** The Constant DEFAULT_ML_RESOURCE_PATH. */
public static final String DEFAULT_ML_RESOURCE_PATH = "ml";
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensMLJerseyClient.class);
-
/** The connection. */
private final LensConnection connection;
@@ -257,7 +255,7 @@ public class LensMLJerseyClient {
try {
connection.close();
} catch (Exception exc) {
- LOG.error("Error closing connection", exc);
+ log.error("Error closing connection", exc);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
index 86cab38..3936693 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
@@ -28,21 +28,19 @@ import org.apache.lens.ml.algo.api.MLAlgo;
import org.apache.lens.ml.algo.api.MLModel;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.rdd.RDD;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class BaseSparkAlgo.
*/
+@Slf4j
public abstract class BaseSparkAlgo implements MLAlgo {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(BaseSparkAlgo.class);
-
/** The name. */
private final String name;
@@ -126,7 +124,7 @@ public abstract class BaseSparkAlgo implements MLAlgo {
}
TableTrainingSpec spec = builder.build();
- LOG.info("Training " + " with " + features.size() + " features");
+ log.info("Training with {} features", features.size());
spec.createRDDs(sparkContext);
@@ -208,7 +206,7 @@ public abstract class BaseSparkAlgo implements MLAlgo {
try {
return Double.parseDouble(params.get(param));
} catch (NumberFormatException nfe) {
- LOG.warn("Couldn't parse param value: " + param + " as double.");
+ log.warn("Couldn't parse param value: {} as double.", param);
}
}
return defaultVal;
@@ -226,7 +224,7 @@ public abstract class BaseSparkAlgo implements MLAlgo {
try {
return Integer.parseInt(params.get(param));
} catch (NumberFormatException nfe) {
- LOG.warn("Couldn't parse param value: " + param + " as integer.");
+ log.warn("Couldn't parse param value: {} as integer.", param);
}
}
return defaultVal;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
index d75efc0..900792e 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
@@ -20,22 +20,22 @@ package org.apache.lens.ml.algo.spark;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hive.hcatalog.data.HCatRecord;
-import org.apache.log4j.Logger;
import org.apache.spark.mllib.linalg.Vectors;
import org.apache.spark.mllib.regression.LabeledPoint;
import com.google.common.base.Preconditions;
+
+import lombok.extern.slf4j.Slf4j;
+
import scala.Tuple2;
/**
* A feature function that directly maps an HCatRecord to a feature vector. Each column becomes a feature in the vector,
* with the value of the feature obtained using the value mapper for that column
*/
+@Slf4j
public class ColumnFeatureFunction extends FeatureFunction {
- /** The Constant LOG. */
- public static final Logger LOG = Logger.getLogger(ColumnFeatureFunction.class);
-
/** The feature value mappers. */
private final FeatureValueMapper[] featureValueMappers;
@@ -85,7 +85,7 @@ public class ColumnFeatureFunction extends FeatureFunction {
HCatRecord record = tuple._2();
if (record == null) {
- LOG.info("@@@ Null record");
+ log.info("@@@ Null record");
return defaultLabeledPoint;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
index 4960e1e..fd5651e 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
@@ -20,8 +20,6 @@ package org.apache.lens.ml.algo.spark;
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hive.hcatalog.data.HCatRecord;
@@ -36,8 +34,6 @@ public final class HiveTableRDD {
private HiveTableRDD() {
}
- public static final Log LOG = LogFactory.getLog(HiveTableRDD.class);
-
/**
* Creates the hive table rdd.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
index 21ed87d..9ac62ce 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
@@ -34,19 +34,17 @@ import org.apache.lens.ml.algo.spark.svm.SVMAlgo;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class SparkMLDriver.
*/
+@Slf4j
public class SparkMLDriver implements MLDriver {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(SparkMLDriver.class);
-
/** The owns spark context. */
private boolean ownsSparkContext = true;
@@ -120,7 +118,7 @@ public class SparkMLDriver implements MLDriver {
((BaseSparkAlgo) algo).setSparkContext(sparkContext);
}
} catch (LensException exc) {
- LOG.error("Error creating algo object", exc);
+ log.error("Error creating algo object", exc);
}
return algo;
}
@@ -178,8 +176,8 @@ public class SparkMLDriver implements MLDriver {
sparkConf.set("spark.yarn.jar", yarnJars);
}
- LOG.info("Spark home is set to " + sparkConf.get("spark.home"));
- LOG.info("spark.yarn.jar is set to " + yarnJars);
+ log.info("Spark home is set to {}", sparkConf.get("spark.home"));
+ log.info("spark.yarn.jar is set to {}", yarnJars);
}
sparkConf.setAppName("lens-ml");
@@ -206,7 +204,7 @@ public class SparkMLDriver implements MLDriver {
throw new LensException("HIVE_HOME is not set");
}
- LOG.info("HIVE_HOME at " + hiveLocation);
+ log.info("HIVE_HOME at {}", hiveLocation);
File hiveLibDir = new File(hiveLocation, "lib");
FilenameFilter jarFileFilter = new FilenameFilter() {
@@ -220,7 +218,7 @@ public class SparkMLDriver implements MLDriver {
// Add hive jars
for (File jarFile : hiveLibDir.listFiles(jarFileFilter)) {
jarFiles.add(jarFile.getAbsolutePath());
- LOG.info("Adding HIVE jar " + jarFile.getAbsolutePath());
+ log.info("Adding HIVE jar {}", jarFile.getAbsolutePath());
sparkContext.addJar(jarFile.getAbsolutePath());
}
@@ -228,20 +226,20 @@ public class SparkMLDriver implements MLDriver {
File hcatalogDir = new File(hiveLocation + "/hcatalog/share/hcatalog");
for (File jarFile : hcatalogDir.listFiles(jarFileFilter)) {
jarFiles.add(jarFile.getAbsolutePath());
- LOG.info("Adding HCATALOG jar " + jarFile.getAbsolutePath());
+ log.info("Adding HCATALOG jar {}", jarFile.getAbsolutePath());
sparkContext.addJar(jarFile.getAbsolutePath());
}
// Add the current jar
String[] lensSparkLibJars = JavaSparkContext.jarOfClass(SparkMLDriver.class);
for (String lensSparkJar : lensSparkLibJars) {
- LOG.info("Adding Lens JAR " + lensSparkJar);
+ log.info("Adding Lens JAR {}", lensSparkJar);
sparkContext.addJar(lensSparkJar);
}
}
isStarted = true;
- LOG.info("Created Spark context for app: '" + sparkContext.appName() + "', Spark master: " + sparkContext.master());
+ log.info("Created Spark context for app: '{}', Spark master: {}", sparkContext.appName(), sparkContext.master());
}
/*
@@ -252,14 +250,14 @@ public class SparkMLDriver implements MLDriver {
@Override
public void stop() throws LensException {
if (!isStarted) {
- LOG.warn("Spark driver was not started");
+ log.warn("Spark driver was not started");
return;
}
isStarted = false;
if (ownsSparkContext) {
sparkContext.stop();
}
- LOG.info("Stopped spark context " + this);
+ log.info("Stopped spark context {}", this);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
index 897aacb..5b7c48b 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
@@ -25,8 +25,6 @@ import java.util.List;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hive.hcatalog.data.HCatRecord;
@@ -41,18 +39,18 @@ import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.rdd.RDD;
import com.google.common.base.Preconditions;
+
import lombok.Getter;
import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
/**
* The Class TableTrainingSpec.
*/
@ToString
+@Slf4j
public class TableTrainingSpec implements Serializable {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(TableTrainingSpec.class);
-
/** The training rdd. */
@Getter
private transient RDD<LabeledPoint> trainingRDD;
@@ -324,11 +322,11 @@ public class TableTrainingSpec implements Serializable {
HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
columns = tableSchema.getFields();
} catch (IOException exc) {
- LOG.error("Error getting table info " + toString(), exc);
+ log.error("Error getting table info {}", toString(), exc);
return false;
}
- LOG.info(table + " columns " + columns.toString());
+ log.info("{} columns {}", table, columns.toString());
boolean valid = false;
if (columns != null && !columns.isEmpty()) {
@@ -388,7 +386,7 @@ public class TableTrainingSpec implements Serializable {
throw new LensException("Table spec not valid: " + toString());
}
- LOG.info("Creating RDDs with spec " + toString());
+ log.info("Creating RDDs with spec {}", toString());
// Get the RDD for table
JavaPairRDD<WritableComparable, HCatRecord> tableRDD;
@@ -412,7 +410,7 @@ public class TableTrainingSpec implements Serializable {
if (splitTraining) {
// We have to split the RDD between a training RDD and a testing RDD
- LOG.info("Splitting RDD for table " + database + "." + table + " with split fraction " + trainingFraction);
+ log.info("Splitting RDD for table {}.{} with split fraction {}", database, table, trainingFraction);
JavaRDD<DataSample> sampledRDD = labeledRDD.map(new Function<LabeledPoint, DataSample>() {
@Override
public DataSample call(LabeledPoint v1) throws Exception {
@@ -423,11 +421,11 @@ public class TableTrainingSpec implements Serializable {
trainingRDD = sampledRDD.filter(new TrainingFilter(trainingFraction)).map(new GetLabeledPoint()).rdd();
testingRDD = sampledRDD.filter(new TestingFilter(trainingFraction)).map(new GetLabeledPoint()).rdd();
} else {
- LOG.info("Using same RDD for train and test");
+ log.info("Using same RDD for train and test");
trainingRDD = labeledRDD.rdd();
testingRDD = trainingRDD;
}
- LOG.info("Generated RDDs");
+ log.info("Generated RDDs");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
index 60a4008..2addb20 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/HiveMLUDF.java
@@ -22,8 +22,6 @@ import java.io.IOException;
import org.apache.lens.ml.algo.api.MLModel;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
@@ -38,12 +36,15 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
import org.apache.hadoop.mapred.JobConf;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Generic UDF to laod ML Models saved in HDFS and apply the model on list of columns passed as argument.
*/
@Description(name = "predict",
value = "_FUNC_(algorithm, modelID, features...) - Run prediction algorithm with given "
+ "algorithm name, model ID and input feature columns")
+@Slf4j
public final class HiveMLUDF extends GenericUDF {
private HiveMLUDF() {
}
@@ -51,9 +52,6 @@ public final class HiveMLUDF extends GenericUDF {
/** The Constant UDF_NAME. */
public static final String UDF_NAME = "predict";
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(HiveMLUDF.class);
-
/** The conf. */
private JobConf conf;
@@ -80,7 +78,7 @@ public final class HiveMLUDF extends GenericUDF {
throw new UDFArgumentLengthException("Algo name, model ID and at least one feature should be passed to "
+ UDF_NAME);
}
- LOG.info(UDF_NAME + " initialized");
+ log.info("{} initialized", UDF_NAME);
return PrimitiveObjectInspectorFactory.javaDoubleObjectInspector;
}
@@ -133,6 +131,6 @@ public final class HiveMLUDF extends GenericUDF {
conf = context.getJobConf();
soi = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
doi = LazyPrimitiveObjectInspectorFactory.LAZY_DOUBLE_OBJECT_INSPECTOR;
- LOG.info(UDF_NAME + " configured. Model base dir path: " + conf.get(ModelLoader.MODEL_PATH_BASE_DIR));
+ log.info("{} configured. Model base dir path: {}", UDF_NAME, conf.get(ModelLoader.MODEL_PATH_BASE_DIR));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java
index e5ec272..91840e9 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLRunner.java
@@ -26,8 +26,6 @@ import org.apache.lens.client.LensClient;
import org.apache.lens.client.LensClientConfig;
import org.apache.lens.client.LensMLClient;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
@@ -39,9 +37,10 @@ import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.mapred.TextInputFormat;
-public class MLRunner {
+import lombok.extern.slf4j.Slf4j;
- private static final Log LOG = LogFactory.getLog(MLRunner.class);
+@Slf4j
+public class MLRunner {
private LensMLClient mlClient;
private String algoName;
@@ -90,7 +89,7 @@ public class MLRunner {
}
public MLTask train() throws Exception {
- LOG.info("Starting train & eval");
+ log.info("Starting train & eval");
createTable(trainTable, trainFile);
createTable(testTable, testFile);
@@ -103,7 +102,7 @@ public class MLRunner {
taskBuilder.addFeatureColumn(feature);
}
MLTask task = taskBuilder.build();
- LOG.info("Created task " + task.toString());
+ log.info("Created task {}", task.toString());
task.run();
return task;
}
@@ -141,7 +140,7 @@ public class MLRunner {
Hive.get(conf).dropTable(database, tableName, false, true);
Hive.get(conf).createTable(tbl, true);
- LOG.info("Created table " + tableName);
+ log.info("Created table {}", tableName);
// Add partition for the data file
AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName,
@@ -150,7 +149,7 @@ public class MLRunner {
partSpec.put("dummy_partition_col", "dummy_val");
partitionDesc.addPartition(partSpec, partDir.toUri().toString());
Hive.get(conf).createPartitions(partitionDesc);
- LOG.info(tableName + ": Added partition " + partDir.toUri().toString());
+ log.info("{}: Added partition {}", tableName, partDir.toUri().toString());
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java
index 2867b90..a3695ba 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/MLTask.java
@@ -24,19 +24,18 @@ import org.apache.lens.client.LensMLClient;
import org.apache.lens.ml.api.LensML;
import org.apache.lens.ml.api.MLTestReport;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import lombok.Getter;
import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
/**
* Run a complete cycle of train and test (evaluation) for an ML algorithm
*/
@ToString
+@Slf4j
public class MLTask implements Runnable {
- private static final Log LOG = LogFactory.getLog(MLTask.class);
public enum State {
RUNNING, SUCCESSFUL, FAILED
@@ -200,14 +199,14 @@ public class MLTask implements Runnable {
@Override
public void run() {
taskState = State.RUNNING;
- LOG.info("Starting " + taskID);
+ log.info("Starting {}", taskID);
try {
runTask();
taskState = State.SUCCESSFUL;
- LOG.info("Complete " + taskID);
+ log.info("Complete {}", taskID);
} catch (Exception e) {
taskState = State.FAILED;
- LOG.info("Error running task " + taskID, e);
+ log.info("Error running task {}", taskID, e);
}
}
@@ -221,20 +220,20 @@ public class MLTask implements Runnable {
if (mlClient != null) {
// Connect to a remote Lens server
ml = mlClient;
- LOG.info("Working in client mode. Lens session handle " + mlClient.getSessionHandle().getPublicId());
+ log.info("Working in client mode. Lens session handle {}", mlClient.getSessionHandle().getPublicId());
} else {
// In server mode session handle has to be passed by the user as a request parameter
ml = MLUtils.getMLService();
- LOG.info("Working in Lens server");
+ log.info("Working in Lens server");
}
String[] algoArgs = buildTrainingArgs();
- LOG.info("Starting task " + taskID + " algo args: " + Arrays.toString(algoArgs));
+ log.info("Starting task {} algo args: {} ", taskID, Arrays.toString(algoArgs));
modelID = ml.train(trainingTable, algorithm, algoArgs);
printModelMetadata(taskID, modelID);
- LOG.info("Starting test " + taskID);
+ log.info("Starting test {}", taskID);
testTable = (testTable != null) ? testTable : trainingTable;
MLTestReport testReport = ml.testModel(mlClient.getSessionHandle(), testTable, algorithm, modelID, outputTable);
reportID = testReport.getReportID();
@@ -244,7 +243,7 @@ public class MLTask implements Runnable {
// Save task metadata to DB
private void saveTask() {
- LOG.info("Saving task details to DB");
+ log.info("Saving task details to DB");
}
private void printTestReport(String exampleID, MLTestReport testReport) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
index c0e7953..4fc45d3 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/ModelLoader.java
@@ -29,8 +29,6 @@ import org.apache.lens.ml.algo.api.MLModel;
import org.apache.lens.ml.api.MLTestReport;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -39,9 +37,12 @@ import org.apache.hadoop.hive.conf.HiveConf;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Load ML models from a FS location.
*/
+@Slf4j
public final class ModelLoader {
private ModelLoader() {
}
@@ -52,9 +53,6 @@ public final class ModelLoader {
/** The Constant MODEL_PATH_BASE_DIR_DEFAULT. */
public static final String MODEL_PATH_BASE_DIR_DEFAULT = "file:///tmp";
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(ModelLoader.class);
-
/** The Constant TEST_REPORT_BASE_DIR. */
public static final String TEST_REPORT_BASE_DIR = "lens.ml.test.basedir";
@@ -97,8 +95,8 @@ public final class ModelLoader {
*/
public static MLModel loadModel(Configuration conf, String algorithm, String modelID) throws IOException {
final Path modelPath = getModelLocation(conf, algorithm, modelID);
- LOG.info("Loading model for algorithm: " + algorithm + " modelID: " + modelID + " At path: "
- + modelPath.toUri().toString());
+ log.info("Loading model for algorithm: {} modelID: {} At path: {}", algorithm, modelID,
+ modelPath.toUri().toString());
try {
return modelCache.get(modelPath, new Callable<MLModel>() {
@Override
@@ -112,7 +110,7 @@ public final class ModelLoader {
try {
ois = new ObjectInputStream(fs.open(modelPath));
MLModel model = (MLModel) ois.readObject();
- LOG.info("Loaded model " + model.getId() + " from location " + modelPath);
+ log.info("Loaded model {} from location {}", model.getId(), modelPath);
return model;
} catch (ClassNotFoundException e) {
throw new IOException(e);
@@ -158,14 +156,14 @@ public final class ModelLoader {
FileSystem fs = reportDir.getFileSystem(conf);
if (!fs.exists(reportDir)) {
- LOG.info("Creating test report dir " + reportDir.toUri().toString());
+ log.info("Creating test report dir {}", reportDir.toUri().toString());
fs.mkdirs(reportDir);
}
Path algoDir = new Path(reportDir, report.getAlgorithm());
if (!fs.exists(algoDir)) {
- LOG.info("Creating algorithm report dir " + algoDir.toUri().toString());
+ log.info("Creating algorithm report dir {}", algoDir.toUri().toString());
fs.mkdirs(algoDir);
}
@@ -177,12 +175,12 @@ public final class ModelLoader {
reportOutputStream.writeObject(report);
reportOutputStream.flush();
} catch (IOException ioexc) {
- LOG.error("Error saving test report " + report.getReportID(), ioexc);
+ log.error("Error saving test report {}", report.getReportID(), ioexc);
throw ioexc;
} finally {
IOUtils.closeQuietly(reportOutputStream);
}
- LOG.info("Saved report " + report.getReportID() + " at location " + reportSaveLocation.toUri());
+ log.info("Saved report {} at location {}", report.getReportID(), reportSaveLocation.toUri());
}
/**
@@ -204,7 +202,7 @@ public final class ModelLoader {
reportStream = new ObjectInputStream(fs.open(reportLocation));
report = (MLTestReport) reportStream.readObject();
} catch (IOException ioex) {
- LOG.error("Error reading report " + reportLocation, ioex);
+ log.error("Error reading report {}", reportLocation, ioex);
} catch (ClassNotFoundException e) {
throw new IOException(e);
} finally {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
index 34b2a3f..470c977 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/impl/TableTestingSpec.java
@@ -23,8 +23,6 @@ import java.util.HashMap;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -32,15 +30,14 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
/**
* Table specification for running test on a table.
*/
+@Slf4j
public class TableTestingSpec {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(TableTestingSpec.class);
-
/** The db. */
private String db;
@@ -256,7 +253,7 @@ public class TableTestingSpec {
Table outTbl = metastoreClient.getTable(db == null ? "default" : db, outputTable, false);
outputTableExists = (outTbl != null);
} catch (HiveException exc) {
- LOG.error("Error getting table info " + toString(), exc);
+ log.error("Error getting table info {}", toString(), exc);
return false;
}
@@ -267,22 +264,22 @@ public class TableTestingSpec {
}
if (!testTableColumns.containsAll(featureColumns)) {
- LOG.info("Invalid feature columns: " + featureColumns + ". Actual columns in table:" + testTableColumns);
+ log.info("Invalid feature columns: {}. Actual columns in table:{}", featureColumns, testTableColumns);
return false;
}
if (!testTableColumns.contains(labelColumn)) {
- LOG.info("Invalid label column: " + labelColumn + ". Actual columns in table:" + testTableColumns);
+ log.info("Invalid label column: {}. Actual columns in table:{}", labelColumn, testTableColumns);
return false;
}
if (StringUtils.isBlank(outputColumn)) {
- LOG.info("Output column is required");
+ log.info("Output column is required");
return false;
}
if (StringUtils.isBlank(outputTable)) {
- LOG.info("Output table is required");
+ log.info("Output table is required");
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
index de9c711..ff046c7 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceImpl.java
@@ -38,19 +38,17 @@ import org.apache.lens.server.api.ServiceProviderFactory;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryExecutionService;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.CompositeService;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class MLServiceImpl.
*/
+@Slf4j
public class MLServiceImpl extends CompositeService implements MLService {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensMLImpl.class);
-
/** The ml. */
private LensMLImpl ml;
@@ -156,7 +154,7 @@ public class MLServiceImpl extends CompositeService implements MLService {
ml.init(hiveConf);
super.init(hiveConf);
serviceProviderFactory = getServiceProviderFactory(hiveConf);
- LOG.info("Inited ML service");
+ log.info("Inited ML service");
}
/*
@@ -168,7 +166,7 @@ public class MLServiceImpl extends CompositeService implements MLService {
public synchronized void start() {
ml.start();
super.start();
- LOG.info("Started ML service");
+ log.info("Started ML service");
}
/*
@@ -180,7 +178,7 @@ public class MLServiceImpl extends CompositeService implements MLService {
public synchronized void stop() {
ml.stop();
super.stop();
- LOG.info("Stopped ML service");
+ log.info("Stopped ML service");
}
/**
@@ -294,7 +292,7 @@ public class MLServiceImpl extends CompositeService implements MLService {
// Wait for test query to complete
LensQuery query = queryService.getQuery(sessionHandle, testQueryHandle);
- LOG.info("Submitted query " + testQueryHandle.getHandleId());
+ log.info("Submitted query {}", testQueryHandle.getHandleId());
while (!query.getStatus().finished()) {
try {
Thread.sleep(500);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
index 77a6c40..ac2957d 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/server/MLServiceResource.java
@@ -41,22 +41,20 @@ import org.apache.lens.server.api.ServiceProviderFactory;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.glassfish.jersey.media.multipart.FormDataParam;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Machine Learning service.
*/
@Path("/ml")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
+@Slf4j
public class MLServiceResource {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(MLServiceResource.class);
-
/** The ml service. */
MLService mlService;
@@ -284,9 +282,9 @@ public class MLServiceResource {
algoArgs.add(values.get(0));
}
}
- LOG.info("Training table " + table + " with algo " + algorithm + " params=" + algoArgs.toString());
+ log.info("Training table {} with algo {} params={}", table, algorithm, algoArgs.toString());
String modelId = getMlService().train(table, algorithm, algoArgs.toArray(new String[]{}));
- LOG.info("Done training " + table + " modelid = " + modelId);
+ log.info("Done training {} modelid = {}", table, modelId);
return modelId;
}
@@ -300,7 +298,7 @@ public class MLServiceResource {
@Produces(MediaType.TEXT_PLAIN)
public Response clearModelCache() {
ModelLoader.clearCache();
- LOG.info("Cleared model cache");
+ log.info("Cleared model cache");
return Response.ok("Cleared cache", MediaType.TEXT_PLAIN_TYPE).build();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
index fc4164f..b4f43ec 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/rdd/LensRDDClient.java
@@ -32,8 +32,6 @@ import org.apache.lens.client.exceptions.LensAPIException;
import org.apache.lens.ml.algo.spark.HiveTableRDD;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -50,6 +48,8 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.rdd.RDD;
+import lombok.extern.slf4j.Slf4j;
+
/**
* <p>
* Create RDD from a Lens query. User can poll returned query handle with isReadyForRDD() until the RDD is ready to be
@@ -76,10 +76,9 @@ import org.apache.spark.rdd.RDD;
* JavaRDD<ResultRow> rdd = client.createLensRDD("SELECT msr1 from TEST_CUBE WHERE ...", conf);
* </pre>
*/
+@Slf4j
public class LensRDDClient {
- /** The Constant LOG. */
- public static final Log LOG = LogFactory.getLog(LensRDDClient.class);
// Default input format for table created from Lens result set
/** The Constant INPUT_FORMAT. */
private static final String INPUT_FORMAT = TextInputFormat.class.getName();
@@ -231,7 +230,7 @@ public class LensRDDClient {
try {
rdd = HiveTableRDD.createHiveTableRDD(sparkContext, HIVE_CONF, "default", tempTableName, TEMP_TABLE_PART_COL
+ "='" + TEMP_TABLE_PART_VAL + "'");
- LOG.info("Created RDD " + rdd.name() + " for table " + tempTableName);
+ log.info("Created RDD {} for table {}", rdd.name(), tempTableName);
} catch (IOException e) {
throw new LensException("Error creating RDD for table " + tempTableName, e);
}
@@ -268,7 +267,7 @@ public class LensRDDClient {
tbl.getPartCols().add(new FieldSchema(TEMP_TABLE_PART_COL, "string", "default"));
hiveClient.createTable(tbl);
- LOG.info("Table " + tableName + " created");
+ log.info("Table {} created", tableName);
// Add partition to the table
AddPartitionDesc partitionDesc = new AddPartitionDesc("default", tableName, false);
@@ -276,7 +275,7 @@ public class LensRDDClient {
partSpec.put(TEMP_TABLE_PART_COL, TEMP_TABLE_PART_VAL);
partitionDesc.addPartition(partSpec, dataLocation);
hiveClient.createPartitions(partitionDesc);
- LOG.info("Created partition in " + tableName + " for data in " + dataLocation);
+ log.info("Created partition in {} for data in {}", tableName, dataLocation);
return tableName;
}
@@ -306,7 +305,7 @@ public class LensRDDClient {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting for query", e);
+ log.warn("Interrupted while waiting for query", e);
break;
}
}
@@ -369,7 +368,7 @@ public class LensRDDClient {
JavaPairRDD<WritableComparable, HCatRecord> javaPairRDD = HiveTableRDD.createHiveTableRDD(sparkContext,
HIVE_CONF, "default", tempTableName, TEMP_TABLE_PART_COL + "='" + TEMP_TABLE_PART_VAL + "'");
resultRDD = javaPairRDD.map(new HCatRecordToObjectListMapper()).rdd();
- LOG.info("Created RDD " + resultRDD.name() + " for table " + tempTableName);
+ log.info("Created RDD {} for table {}", resultRDD.name(), tempTableName);
} catch (IOException e) {
throw new LensException("Error creating RDD for table " + tempTableName, e);
}
@@ -391,7 +390,7 @@ public class LensRDDClient {
try {
hiveClient = Hive.get(HIVE_CONF);
hiveClient.dropTable("default." + tempTableName);
- LOG.info("Dropped temp table " + tempTableName);
+ log.info("Dropped temp table {}", tempTableName);
} catch (HiveException e) {
throw new LensException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
index 9fe1ea0..ac3a55e 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/ExampleUtils.java
@@ -23,8 +23,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
@@ -36,15 +34,16 @@ import org.apache.hadoop.hive.ql.plan.AddPartitionDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.mapred.TextInputFormat;
+import lombok.extern.slf4j.Slf4j;
+
/**
* The Class ExampleUtils.
*/
+@Slf4j
public final class ExampleUtils {
private ExampleUtils() {
}
- private static final Log LOG = LogFactory.getLog(ExampleUtils.class);
-
/**
* Creates the example table.
*
@@ -88,7 +87,7 @@ public final class ExampleUtils {
tbl.setPartCols(partCols);
Hive.get(conf).createTable(tbl, false);
- LOG.info("Created table " + tableName);
+ log.info("Created table {}", tableName);
// Add partition for the data file
AddPartitionDesc partitionDesc = new AddPartitionDesc(database, tableName, false);
@@ -96,6 +95,6 @@ public final class ExampleUtils {
partSpec.put("dummy_partition_col", "dummy_val");
partitionDesc.addPartition(partSpec, partDir.toUri().toString());
Hive.get(conf).createPartitions(partitionDesc);
- LOG.info(tableName + ": Added partition " + partDir.toUri().toString());
+ log.info("{}: Added partition {}", tableName, partDir.toUri().toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
index 7f6413b..67e3379 100644
--- a/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
+++ b/lens-ml-lib/src/test/java/org/apache/lens/ml/TestMLRunner.java
@@ -35,8 +35,6 @@ import org.apache.lens.server.metastore.MetastoreResource;
import org.apache.lens.server.query.QueryServiceResource;
import org.apache.lens.server.session.SessionResource;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -51,7 +49,6 @@ import org.testng.annotations.Test;
@Test
public class TestMLRunner extends LensJerseyTest {
- private static final Log LOG = LogFactory.getLog(TestMLRunner.class);
private static final String TEST_DB = TestMLRunner.class.getSimpleName();
private LensMLClient mlClient;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/pom.xml
----------------------------------------------------------------------
diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml
index 66ddbb0..cfbdc0a 100644
--- a/lens-regression/pom.xml
+++ b/lens-regression/pom.xml
@@ -44,10 +44,6 @@
<version>${project.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </dependency>
- <dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java
index ad67d7d..8b41dba 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/LensServerHelper.java
@@ -30,15 +30,13 @@ import org.apache.lens.regression.util.AssertUtil;
import org.apache.lens.regression.util.Util;
import org.apache.lens.server.api.error.LensException;
-import org.apache.log4j.Logger;
-
import com.jcraft.jsch.JSchException;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class LensServerHelper extends ServiceManagerHelper {
- private static Logger logger = Logger.getLogger(LensServerHelper.class);
-
private WebTarget servLens = ServiceManagerHelper.getServerLens();
private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
@@ -60,9 +58,9 @@ public class LensServerHelper extends ServiceManagerHelper {
Response response = this.exec("get", "", servLens, null, null);
while (response == null && counter < 40) {
Thread.sleep(5000);
- logger.info("Waiting for Lens server to come up ");
+ log.info("Waiting for Lens server to come up ");
response = this.exec("get", "", servLens, null, null);
- logger.info(response);
+ log.info("Response:{}", response);
counter++;
}
AssertUtil.assertSucceededResponse(response);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java
index 7fd3c47..aff0e8e 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/MetastoreHelper.java
@@ -32,15 +32,11 @@ import org.apache.lens.regression.core.type.MapBuilder;
import org.apache.lens.regression.util.AssertUtil;
import org.apache.lens.server.api.error.LensException;
-import org.apache.log4j.Logger;
-
-
-
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class MetastoreHelper extends ServiceManagerHelper {
- private static Logger logger = Logger.getLogger(MetastoreHelper.class);
-
private WebTarget servLens = ServiceManagerHelper.getServerLens();
private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
@@ -66,11 +62,11 @@ public class MetastoreHelper extends ServiceManagerHelper {
response = this.exec("get", MetastoreURL.METASTORE_DATABASES_CURRENT_URL, servLens, null, query);
String responseString = response.readEntity(String.class);
AssertUtil.assertSucceededResponse(response);
- logger.info(responseString.trim());
+ log.info("Response: {}", responseString.trim());
if (!responseString.trim().equals(currentDBName)) {
throw new LensException("Could not set database");
}
- logger.info("Set Current database to " + currentDBName);
+ log.info("Set Current database to {}", currentDBName);
}
public void setCurrentDatabase(String currentDBName) throws JAXBException, LensException {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
index 5d58ef2..70cad28 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/QueryHelper.java
@@ -40,16 +40,14 @@ import org.apache.lens.regression.util.AssertUtil;
import org.apache.lens.regression.util.Util;
import org.apache.lens.server.api.error.LensException;
-import org.apache.log4j.Logger;
-
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
+import lombok.extern.slf4j.Slf4j;
-
+@Slf4j
public class QueryHelper extends ServiceManagerHelper {
- private static Logger logger = Logger.getLogger(QueryHelper.class);
private WebTarget servLens = ServiceManagerHelper.getServerLens();
private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
@@ -83,13 +81,13 @@ public class QueryHelper extends ServiceManagerHelper {
MediaType.APPLICATION_XML, formData.getForm());
AssertUtil.assertSucceededResponse(response);
String queryHandleString = response.readEntity(String.class);
- logger.info(queryHandleString);
+ log.info("QueryHandle String:{}", queryHandleString);
LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryHandleString, LensAPIResult.class);
QueryHandle queryHandle = (QueryHandle) successResponse.getData();
if (queryHandle == null) {
throw new LensException("Query Execute Failed");
}
- logger.info("Query Handle : " + queryHandle);
+ log.info("Query Handle : {}", queryHandle);
return queryHandle;
}
@@ -138,13 +136,13 @@ public class QueryHelper extends ServiceManagerHelper {
MediaType.APPLICATION_XML, formData.getForm());
AssertUtil.assertSucceededResponse(response);
String queryHandleString = response.readEntity(String.class);
- logger.info(queryHandleString);
+ log.info("QueryHandle String:{}", queryHandleString);
LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryHandleString, LensAPIResult.class);
QueryHandleWithResultSet queryHandleWithResultSet = (QueryHandleWithResultSet) successResponse.getData();
if (queryHandleWithResultSet==null) {
throw new LensException("Query Execute Failed");
}
- logger.info("Query Handle with ResultSet : " + queryHandleWithResultSet);
+ log.info("Query Handle with ResultSet : {}", queryHandleWithResultSet);
return queryHandleWithResultSet;
}
@@ -200,7 +198,7 @@ public class QueryHelper extends ServiceManagerHelper {
MediaType.APPLICATION_XML, formData.getForm());
AssertUtil.assertSucceededResponse(response);
String queryHandleString = response.readEntity(String.class);
- logger.info(queryHandleString);
+ log.info("QueryHandle String:{}", queryHandleString);
LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryHandleString, LensAPIResult.class);
QueryHandle queryHandle = (QueryHandle) successResponse.getData();
return queryHandle;
@@ -228,7 +226,7 @@ public class QueryHelper extends ServiceManagerHelper {
MediaType.APPLICATION_XML, formData.getForm());
AssertUtil.assertSucceededResponse(response);
String queryPlanString = response.readEntity(String.class);
- logger.info(queryPlanString);
+ log.info("QueryPlan String:{}", queryPlanString);
LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryPlanString, LensAPIResult.class);
QueryPlan queryPlan = (QueryPlan) successResponse.getData();
return queryPlan;
@@ -265,7 +263,7 @@ public class QueryHelper extends ServiceManagerHelper {
MediaType.APPLICATION_XML, formData.getForm());
AssertUtil.assertSucceededResponse(response);
String queryCostString = response.readEntity(String.class);
- logger.info(queryCostString);
+ log.info("QueryCost String:{}", queryCostString);
LensAPIResult successResponse = (LensAPIResult) Util.getObject(queryCostString, LensAPIResult.class);
QueryCostTO queryCostTO = (QueryCostTO) successResponse.getData();
if (queryCostTO == null) {
@@ -306,7 +304,7 @@ public class QueryHelper extends ServiceManagerHelper {
.exec("post", "/queryapi/preparedqueries", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
MediaType.APPLICATION_XML, formData.getForm());
String queryPlanString = response.readEntity(String.class);
- logger.info(queryPlanString);
+ log.info("QueryPlan String:{}", queryPlanString);
QueryPlan queryPlan = (QueryPlan) Util.getObject(queryPlanString, QueryPlan.class);
return queryPlan;
}
@@ -341,9 +339,9 @@ public class QueryHelper extends ServiceManagerHelper {
Response response = this
.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/resultset", servLens, null, query);
AssertUtil.assertSucceededResponse(response);
- logger.info(response);
+ log.info("Response {}", response);
String queryResultString = response.readEntity(String.class);
- logger.info(queryResultString);
+ log.info("QueryResult String:{}", queryResultString);
QueryResult queryResult = (QueryResult) Util.getObject(queryResultString, QueryResult.class);
return queryResult;
}
@@ -370,9 +368,9 @@ public class QueryHelper extends ServiceManagerHelper {
Response response = this
.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString() + "/httpresultset", servLens, null, null);
AssertUtil.assertSucceededResponse(response);
- logger.info(response);
+ log.info("Response {}", response);
String queryResultString = response.readEntity(String.class);
- logger.info(queryResultString);
+ log.info("QueryResult String:{}", queryResultString);
QueryResult queryResult = (QueryResult) Util.getObject(queryResultString, QueryResult.class);
return queryResult;
}
@@ -397,7 +395,7 @@ public class QueryHelper extends ServiceManagerHelper {
MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML, formData.getForm());
String queryHandleString = response.readEntity(String.class);
AssertUtil.assertSucceededResponse(response);
- logger.info(queryHandleString);
+ log.info("QueryHandle String:{}", queryHandleString);
QueryHandle handle = (QueryHandle) Util.getObject(queryHandleString, QueryHandle.class);
return handle;
}
@@ -438,7 +436,7 @@ public class QueryHelper extends ServiceManagerHelper {
MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML, formData.getForm());
String queryHandleString = response.readEntity(String.class);
AssertUtil.assertSucceededResponse(response);
- logger.info(queryHandleString);
+ log.info("QueryHandle String:{}", queryHandleString);
QueryHandleWithResultSet handle = (QueryHandleWithResultSet) Util
.getObject(queryHandleString, QueryHandleWithResultSet.class);
return handle;
@@ -483,7 +481,7 @@ public class QueryHelper extends ServiceManagerHelper {
.exec("post", "/queryapi/preparedqueries", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE,
MediaType.APPLICATION_XML, formData.getForm());
String queryHandleString = response.readEntity(String.class);
- logger.info(queryHandleString);
+ log.info("QueryHandle String:{}", queryHandleString);
AssertUtil.assertSucceededResponse(response);
QueryPrepareHandle queryHandle = (QueryPrepareHandle) Util.getObject(queryHandleString, QueryPrepareHandle.class);
return queryHandle;
@@ -517,7 +515,7 @@ public class QueryHelper extends ServiceManagerHelper {
MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
Response response = this
.exec("delete", QueryURL.PREPAREDQUERY_URL + "/" + queryPreparedHandle.toString(), servLens, null, query);
- logger.info("Response : " + response);
+ log.info("Response : {}" + response);
AssertUtil.assertSucceededResponse(response);
}
@@ -552,9 +550,9 @@ public class QueryHelper extends ServiceManagerHelper {
queryList.put("toDate", toDate);
}
Response response = this.sendQuery("get", QueryURL.PREPAREDQUERY_URL, queryList);
- logger.info("Response : " + response);
+ log.info("Response : {}" + response);
String responseString = response.readEntity(String.class);
- logger.info(responseString);
+ log.info("Response String:{}", responseString);
PrepareQueryHandles result = (PrepareQueryHandles) Util.getObject(responseString, PrepareQueryHandles.class);
List<QueryPrepareHandle> list = result.getQueryHandles();
return list;
@@ -606,7 +604,7 @@ public class QueryHelper extends ServiceManagerHelper {
}
Response response = this.exec("delete", QueryURL.PREPAREDQUERY_URL, servLens, null, query);
- logger.info("Response : " + response);
+ log.info("Response : {}", response);
AssertUtil.assertSucceededResponse(response);
}
@@ -677,7 +675,7 @@ public class QueryHelper extends ServiceManagerHelper {
queryList.put("toDate", toDate);
}
Response response = this.sendQuery("get", QueryURL.QUERY_URL, queryList);
- logger.info("Response : " + response);
+ log.info("Response : {}", response);
String responseString = response.readEntity(String.class);
QueryHandles result = (QueryHandles) Util.getObject(responseString, QueryHandles.class);
List<QueryHandle> list = result.getQueryHandles();
@@ -723,12 +721,12 @@ public class QueryHelper extends ServiceManagerHelper {
String responseString = response.readEntity(String.class);
LensQuery lensQuery = (LensQuery) Util.getObject(responseString, LensQuery.class);
while (!lensQuery.getStatus().finished()) {
+ log.info("Waiting...");
Thread.sleep(1000);
- logger.info("Waiting...");
response = this.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString(), servLens, null, query);
lensQuery = (LensQuery) Util.getObject(response.readEntity(String.class), LensQuery.class);
}
- logger.info(lensQuery.getStatus().getStatusMessage());
+ log.info("QueryStatus message:{}", lensQuery.getStatus().getStatusMessage());
return lensQuery;
}
@@ -749,7 +747,7 @@ public class QueryHelper extends ServiceManagerHelper {
JAXBException, InterruptedException, InstantiationException, IllegalAccessException, LensException {
QueryStatus queryStatus = getQueryStatus(sessionHandleString, queryHandle);
while (queryStatus.getStatus() == QueryStatus.Status.QUEUED) {
- logger.info("Waiting for Query to be in Running Phase");
+ log.info("Waiting for Query to be in Running Phase");
Thread.sleep(1000);
queryStatus = getQueryStatus(sessionHandleString, queryHandle);
}
@@ -773,11 +771,11 @@ public class QueryHelper extends ServiceManagerHelper {
JAXBException, InstantiationException, IllegalAccessException, LensException {
MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
Response response = this.exec("get", QueryURL.QUERY_URL + "/" + queryHandle.toString(), servLens, null, query);
- logger.info("Response : " + response);
+ log.info("Response : {}", response);
AssertUtil.assertSucceededResponse(response);
LensQuery lensQuery = (LensQuery) Util.getObject(response.readEntity(String.class), LensQuery.class);
QueryStatus qStatus = lensQuery.getStatus();
- logger.info("Query Status : " + qStatus);
+ log.info("Query Status : {}", qStatus);
return qStatus;
}
@@ -797,7 +795,7 @@ public class QueryHelper extends ServiceManagerHelper {
JAXBException, LensException {
MapBuilder query = new MapBuilder("sessionid", sessionHandleString);
Response response = this.exec("delete", QueryURL.QUERY_URL + "/" + queryHandle.toString(), servLens, null, query);
- logger.info("Response : " + response);
+ log.info("Response : {}", response);
AssertUtil.assertSucceededResponse(response);
}
@@ -841,7 +839,7 @@ public class QueryHelper extends ServiceManagerHelper {
}
Response response = this.exec("delete", QueryURL.QUERY_URL, servLens, null, query);
- logger.info("Response : " + response);
+ log.info("Response : {}", response);
AssertUtil.assertSucceededResponse(response);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java
index 2aecbfd..b14b00b 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/ServiceManagerHelper.java
@@ -40,12 +40,12 @@ import org.apache.lens.regression.util.AssertUtil;
import org.apache.lens.regression.util.Util;
import org.apache.lens.server.api.error.LensException;
-import org.apache.log4j.Logger;
-
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public abstract class ServiceManagerHelper {
private static final String LENS_BASE_URL = "lens.baseurl";
@@ -69,8 +69,6 @@ public abstract class ServiceManagerHelper {
protected String serverHdfsUrl;
protected String currentDB;
- private Logger logger = Logger.getLogger(ServiceManagerHelper.class);
-
public ServiceManagerHelper(String envFileName) {
Properties prop = Util.getPropertiesObj(envFileName);
this.baseUrl = prop.getProperty(LENS_BASE_URL);
@@ -153,7 +151,7 @@ public abstract class ServiceManagerHelper {
MediaType.MULTIPART_FORM_DATA_TYPE, MediaType.APPLICATION_XML, formData.getForm());
AssertUtil.assertSucceededResponse(response);
sessionHandleString = response.readEntity(String.class);
- logger.info("Session Handle String" + sessionHandleString);
+ log.info("Session Handle String:{}", sessionHandleString);
return sessionHandleString;
}
@@ -174,7 +172,7 @@ public abstract class ServiceManagerHelper {
if (result.getMessage() == null) {
throw new LensException("Status message is null");
}
- logger.info("Closed Session : " + sessionHandleString);
+ log.info("Closed Session : {}", sessionHandleString);
}
public <T> Response exec(String functionName, String path, WebTarget service, FormDataMultiPart headers,
[40/50] [abbrv] incubator-lens git commit: LENS-689 : Add doc for
code module structure
Posted by sh...@apache.org.
LENS-689 : Add doc for code module structure
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/211ef3ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/211ef3ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/211ef3ba
Branch: refs/heads/LENS-581
Commit: 211ef3ba99d40fea51dfe19f4828d77d3b08defe
Parents: 75001a1
Author: Yash Sharma <ya...@gmail.com>
Authored: Mon Jul 27 16:48:11 2015 +0530
Committer: arshad-matin <ar...@apache.org>
Committed: Mon Jul 27 16:48:11 2015 +0530
----------------------------------------------------------------------
src/site/apt/developer/codestructure.apt | 55 +++++++++++++++++++++++++++
src/site/site.xml | 51 +++++++++++++------------
2 files changed, 81 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/211ef3ba/src/site/apt/developer/codestructure.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/developer/codestructure.apt b/src/site/apt/developer/codestructure.apt
new file mode 100644
index 0000000..871189b
--- /dev/null
+++ b/src/site/apt/developer/codestructure.apt
@@ -0,0 +1,55 @@
+~~
+~~ Licensed to the Apache Software Foundation (ASF) under one
+~~ or more contributor license agreements. See the NOTICE file
+~~ distributed with this work for additional information
+~~ regarding copyright ownership. The ASF licenses this file
+~~ to you under the Apache License, Version 2.0 (the
+~~ "License"); you may not use this file except in compliance
+~~ with the License. You may obtain a copy of the License at
+~~
+~~ http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing,
+~~ software distributed under the License is distributed on an
+~~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+~~ KIND, either express or implied. See the License for the
+~~ specific language governing permissions and limitations
+~~ under the License.
+~~
+
+Lens Code Structure
+
+ Here is some detail on how the Apache Lens modules are structured -
+
+ * <<checkstyle>> - Has checkstyle rules imposed on the project.
+
+ * <<lens-api>> - Lens API. Has API primarily for the clients.
+
+ * <<lens-server-api>> - Lens API mainly for server extensions. This contains API for Service, Driver, LensEvent, QueryRewriter etc.
+
+ * <<lens-cube>> - Cube data model and query rewriting from cube QL to HQL done here.
+
+ * <<lens-storage-db>> - Storage definition for DBStorage and DBStorageHandler.
+
+ * <<lens-query-lib>> - The Query lib containing some Serde or OutputFormatter implementations.
+
+ * <<lens-driver-hive>> - Driver implementation for Hive as execution engine.
+
+ * <<lens-driver-jdbc>> - Driver implementation for JDBC as execution engine.
+
+ * <<lens-server>> - The server module - containing the service implementations, REST resources and HttpServer itself.
+
+ * <<lens-client>> - Java client built on top of REST interface.
+
+ * <<lens-cli>> - CLI built using spring shell and lens-client.
+
+ * <<lens-examples>> - Examples containing example schema and queries.
+
+ * <<lens-dist>> - Packaging for binary distribution of lens
+
+ * <<lens-ml-lib>> - Wrapper over Spark ML lib where input to a model is output of a query.
+
+ * <<lens-ml-dist>> - Packaging for ml-lib - which has server, client, spark libraries and ml-lib.
+
+ * <<lens-regression>> - The regression test suite.
+
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/211ef3ba/src/site/site.xml
----------------------------------------------------------------------
diff --git a/src/site/site.xml b/src/site/site.xml
index 9cf7f71..bd7a80b 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -22,21 +22,21 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/DECORATION/1.3.0 http://maven.apache.org/xsd/decoration-1.3.0.xsd">
- <publishDate position="right"/>
- <version position="right"/>
- <bannerLeft>
- <name>Lens Analytics Platform</name>
- <src>/images/apache-lens-logo.png</src>
- <title>Lens Analytics Platform</title>
- <href>http://lens.incubator.apache.org</href>
- <height>54px</height>
+ <publishDate position="right"/>
+ <version position="right"/>
+ <bannerLeft>
+ <name>Lens Analytics Platform</name>
+ <src>/images/apache-lens-logo.png</src>
+ <title>Lens Analytics Platform</title>
+ <href>http://lens.incubator.apache.org</href>
+ <height>54px</height>
</bannerLeft>
- <bannerRight>
- <name>Apache Incubator</name>
- <src>/images/apache-incubator-logo.png</src>
- <href>http://incubator.apache.org</href>
- </bannerRight>
+ <bannerRight>
+ <name>Apache Incubator</name>
+ <src>/images/apache-incubator-logo.png</src>
+ <href>http://incubator.apache.org</href>
+ </bannerRight>
<body>
<menu name="Lens" >
@@ -47,37 +47,38 @@
<item name="20 Minute Video demo" target="_new"
href="http://cwiki.apache.org/confluence/display/LENS/2015/07/13/20+Minute+video+demo+of+Apache+Lens+through+examples" />
</menu>
- <menu name="Releases" >
+ <menu name="Releases" >
<item name="Download" href="./releases/download.html" />
<item name="Compatibility" href="./releases/compatibility.html" />
<item name="All Releases" href="./releases/release-history.html" />
- </menu>
- <menu name="User Documentation">
+ </menu>
+ <menu name="User Documentation">
<item name="Lens User Guide" href="./user/index.html" />
<item name="OLAP data cube" href="./user/olap-cube.html" />
<item name="REST API" href="./rest.html" />
<item name="Lens CLI" href="./user/cli.html" />
- <item name="Java Client Library" href="./user/java-client.html"/>
- <item name="JDBC Client" href="./user/jdbc-client.html"/>
+ <item name="Java Client Library" href="./user/java-client.html"/>
+ <item name="JDBC Client" href="./user/jdbc-client.html"/>
<item name="Lens ML" href="./user/ml.html" />
- <item name="FAQ" href="./user/faq.html"/>
- </menu>
- <menu name="Developer Documentation">
+ <item name="FAQ" href="./user/faq.html"/>
+ </menu>
+ <menu name="Developer Documentation">
<item name="Contributor Guide" href="./developer/contribute.html" />
<item name="Committer Guide" href="./developer/commit.html" />
<item name="Lens Design" href="./developer/design.html" />
+ <item name="Lens Code Structure" href="./developer/codestructure.html" />
<item name="Driver Developer Guide" href="./developer/driver.html" />
</menu>
- <menu name="Admin Documentation">
+ <menu name="Admin Documentation">
<item name="Admin Overview" href="./admin/index.html"/>
<item name="Deployment" href="./admin/deployment.html"/>
<item name="Configuration" href="./admin/config-server.html"/>
- <item name="Admin CLI" href="./admin/cli.html" />
+ <item name="Admin CLI" href="./admin/cli.html" />
</menu>
<menu ref="reports"/>
<breadcrumbs>
- <item name="Apache" href="http://www.apache.org" />
- <item name="Lens" href="http://lens.incubator.apache.org" />
+ <item name="Apache" href="http://www.apache.org" />
+ <item name="Lens" href="http://lens.incubator.apache.org" />
</breadcrumbs>
</body>
<skin>
[37/50] [abbrv] incubator-lens git commit: LENS-682: Fixing
Checkstyle issue
Posted by sh...@apache.org.
LENS-682: Fixing Checkstyle issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ad8f4a88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ad8f4a88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ad8f4a88
Branch: refs/heads/LENS-581
Commit: ad8f4a88bee24c008f1cb5e73a049123ec26bcdb
Parents: c611bd7
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri Jul 24 16:44:47 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Fri Jul 24 16:44:47 2015 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/lens/client/LensMetadataClient.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ad8f4a88/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
index 31666a0..169170c 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java
@@ -36,7 +36,6 @@ import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
import org.apache.lens.api.APIResult;
-import org.apache.lens.api.APIResult.Status;
import org.apache.lens.api.DateTime;
import org.apache.lens.api.StringList;
import org.apache.lens.api.metastore.*;
[10/50] [abbrv] incubator-lens git commit: Reverse merge
current-release-line
Posted by sh...@apache.org.
Reverse merge current-release-line
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/9cfb4fef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/9cfb4fef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/9cfb4fef
Branch: refs/heads/LENS-581
Commit: 9cfb4fef431830c3fb1ce7496c2543c58b9252ad
Parents: 7f4d793 68166c9
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Thu Jul 16 15:23:39 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Jul 16 15:23:39 2015 +0530
----------------------------------------------------------------------
checkstyle/pom.xml | 3 +--
lens-api/pom.xml | 3 +--
lens-cli/pom.xml | 3 +--
lens-client/pom.xml | 3 +--
lens-cube/pom.xml | 3 +--
lens-dist/pom.xml | 3 +--
lens-driver-hive/pom.xml | 3 +--
lens-driver-jdbc/pom.xml | 3 +--
lens-examples/pom.xml | 3 +--
lens-ml-lib/pom.xml | 3 +--
lens-query-lib/pom.xml | 3 +--
lens-regression/pom.xml | 3 +--
lens-server-api/pom.xml | 3 +--
lens-server/pom.xml | 3 +--
lens-storage-db/pom.xml | 3 +--
pom.xml | 3 +--
16 files changed, 16 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/checkstyle/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-api/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-cli/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-client/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-cube/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-dist/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-driver-hive/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-driver-jdbc/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-examples/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-ml-lib/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-query-lib/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-regression/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-server-api/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-server/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/lens-storage-db/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9cfb4fef/pom.xml
----------------------------------------------------------------------
[08/50] [abbrv] incubator-lens git commit: LENS-647 : Fix multi fact
query with expressions
Posted by sh...@apache.org.
LENS-647 : Fix multi fact query with expressions
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/f1387c31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/f1387c31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/f1387c31
Branch: refs/heads/LENS-581
Commit: f1387c31537eb7f73ba2de503e7a17120df1e691
Parents: c8d38f7
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Jul 10 12:19:10 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jul 10 12:19:10 2015 +0530
----------------------------------------------------------------------
.../apache/lens/cube/parse/CandidateFact.java | 10 +-
.../cube/parse/CandidateTablePruneCause.java | 9 +-
.../lens/cube/parse/CubeQueryContext.java | 4 +-
.../lens/cube/parse/ExpressionResolver.java | 65 ++++---
.../lens/cube/parse/TestBaseCubeQueries.java | 171 +++++++++++++------
5 files changed, 174 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f1387c31/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
index 63d2508..86c420b 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateFact.java
@@ -190,10 +190,10 @@ public class CandidateFact implements CandidateTable {
int currentChild = 0;
for (int i = 0; i < cubeql.getSelectAST().getChildCount(); i++) {
ASTNode selectExpr = (ASTNode) this.selectAST.getChild(currentChild);
- Set<String> exprCols = getColsInExpr(cubeCols, selectExpr);
+ Set<String> exprCols = getColsInExpr(cubeql, cubeCols, selectExpr);
if (getColumns().containsAll(exprCols)) {
selectIndices.add(i);
- if (cubeql.getQueriedDimAttrs().containsAll(exprCols)) {
+ if (cubeql.getCube().getDimAttributeNames().containsAll(exprCols)) {
dimFieldIndices.add(i);
}
ASTNode aliasNode = HQLParser.findNodeByPath(selectExpr, Identifier);
@@ -225,7 +225,8 @@ public class CandidateFact implements CandidateTable {
// TODO
}
- private Set<String> getColsInExpr(final Set<String> cubeCols, ASTNode expr) throws SemanticException {
+ private Set<String> getColsInExpr(final CubeQueryContext cubeql, final Set<String> cubeCols,
+ ASTNode expr) throws SemanticException {
final Set<String> cubeColsInExpr = new HashSet<String>();
HQLParser.bft(expr, new ASTNodeVisitor() {
@Override
@@ -244,9 +245,10 @@ public class CandidateFact implements CandidateTable {
cubeColsInExpr.add(column);
}
} else if (node.getToken().getType() == DOT) {
+ String alias = HQLParser.findNodeByPath(node, TOK_TABLE_OR_COL, Identifier).getText().toLowerCase();
ASTNode colIdent = (ASTNode) node.getChild(1);
String column = colIdent.getText().toLowerCase();
- if (cubeCols.contains(column)) {
+ if (cubeql.getAliasForTableName(cubeql.getCube()).equalsIgnoreCase(alias) && cubeCols.contains(column)) {
cubeColsInExpr.add(column);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f1387c31/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
index 75d5581..9ea43bb 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTablePruneCause.java
@@ -36,10 +36,11 @@ import lombok.NoArgsConstructor;
@JsonWriteNullProperties(false)
@Data
@NoArgsConstructor
-
public class CandidateTablePruneCause {
public enum CandidateTablePruneCode {
+ // other fact set element is removed
+ ELEMENT_IN_SET_PRUNED("Other candidate from measure covering set is pruned"),
FACT_NOT_AVAILABLE_IN_RANGE("No facts available for all of these time ranges: %s") {
@Override
Object[] getFormatPlaceholders(Set<CandidateTablePruneCause> causes) {
@@ -70,6 +71,9 @@ public class CandidateTablePruneCause {
return new String[]{columns.toString()};
}
},
+ // candidate table tries to get denormalized field from dimension and the
+ // referred dimension is invalid.
+ INVALID_DENORM_TABLE("Referred dimension is invalid in one of the candidate tables"),
// column not valid in cube table
COLUMN_NOT_VALID("Column not valid in cube table"),
// column not found in cube table
@@ -92,9 +96,6 @@ public class CandidateTablePruneCause {
}
}
},
- // candidate table tries to get denormalized field from dimension and the
- // referred dimension is invalid.
- INVALID_DENORM_TABLE("Referred dimension is invalid in one of the candidate tables"),
// missing storage tables for cube table
MISSING_STORAGES("Missing storage tables for the cube table"),
// no candidate storges for cube table, storage cause will have why each
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f1387c31/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
index 57c075c..39b13cc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryContext.java
@@ -1196,10 +1196,12 @@ public class CubeQueryContext implements TrackQueriedColumns {
Set<CandidateFact> cfacts = i.next();
if (!candidateFacts.containsAll(cfacts)) {
LOG.info("Not considering fact table set:" + cfacts
- + " as they have non candidate tables and facts missing because of" + pruneCause);
+ + " as they have non candidate tables and facts missing because of " + pruneCause);
i.remove();
}
}
+ // prune candidate facts
+ pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f1387c31/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
index b2997dd..895cb06 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/ExpressionResolver.java
@@ -32,6 +32,7 @@ import org.apache.lens.cube.metadata.CubeInterface;
import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.cube.metadata.ExprColumn;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
+import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.cube.parse.HQLParser.ASTNodeVisitor;
import org.apache.lens.cube.parse.HQLParser.TreeNode;
@@ -204,6 +205,16 @@ class ExpressionResolver implements ContextRewriter {
}
return false;
}
+
+ boolean isEvaluable(CandidateTable cTable) {
+ if (directlyAvailableIn.contains(cTable)) {
+ return true;
+ }
+ if (evaluableExpressions.get(cTable) == null) {
+ return false;
+ }
+ return !evaluableExpressions.get(cTable).isEmpty();
+ }
}
static class ExprSpecContext implements TrackQueriedColumns {
@@ -395,13 +406,7 @@ class ExpressionResolver implements ContextRewriter {
// checks if expr is evaluable
public boolean isEvaluable(String expr, CandidateTable cTable) {
ExpressionContext ec = getExpressionContext(expr, cubeql.getAliasForTableName(cTable.getBaseTable().getName()));
- if (ec.directlyAvailableIn.contains(cTable)) {
- return true;
- }
- if (ec.evaluableExpressions.get(cTable) == null) {
- return false;
- }
- return !ec.evaluableExpressions.get(cTable).isEmpty();
+ return ec.isEvaluable(cTable);
}
/**
@@ -671,24 +676,40 @@ class ExpressionResolver implements ContextRewriter {
cubeql.getExprCtx().pruneExpressions();
// prune candidate facts without any valid expressions
if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) {
- for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext();) {
- CandidateFact cfact = i.next();
- for (Map.Entry<String, Set<ExpressionContext>> ecEntry : exprCtx.allExprsQueried.entrySet()) {
- Set<ExpressionContext> ecSet = ecEntry.getValue();
- for (ExpressionContext ec : ecSet) {
- if (ec.getSrcTable().getName().equals(cfact.getBaseTable().getName())) {
- if (!ec.directlyAvailableIn.contains(cfact)
- && (ec.evaluableExpressions.get(cfact) == null
- || ec.evaluableExpressions.get(cfact).isEmpty())) {
- log.info("Not considering fact table:{} as {} is not evaluable", cfact, ec.exprCol.getName());
- cubeql.addFactPruningMsgs(cfact.fact,
- CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
- i.remove();
+ for (Map.Entry<String, Set<ExpressionContext>> ecEntry : exprCtx.allExprsQueried.entrySet()) {
+ String expr = ecEntry.getKey();
+ Set<ExpressionContext> ecSet = ecEntry.getValue();
+ for (ExpressionContext ec : ecSet) {
+ if (ec.getSrcTable().getName().equals(cubeql.getCube().getName())) {
+ if (cubeql.getQueriedExprsWithMeasures().contains(expr)) {
+ for (Iterator<Set<CandidateFact>> sItr = cubeql.getCandidateFactSets().iterator(); sItr.hasNext();) {
+ Set<CandidateFact> factSet = sItr.next();
+ boolean evaluableInSet = false;
+ for (CandidateFact cfact : factSet) {
+ if (ec.isEvaluable(cfact)) {
+ evaluableInSet = true;
+ }
+ }
+ if (!evaluableInSet) {
+ log.info("Not considering fact table set:{} as {} is not evaluable", factSet, ec.exprCol.getName());
+ sItr.remove();
+ }
+ }
+ } else {
+ for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext();) {
+ CandidateFact cfact = i.next();
+ if (!ec.isEvaluable(cfact)) {
+ log.info("Not considering fact table:{} as {} is not evaluable", cfact, ec.exprCol.getName());
+ cubeql.addFactPruningMsgs(cfact.fact,
+ CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
+ i.remove();
+ }
}
}
}
}
}
+ cubeql.pruneCandidateFactWithCandidateSet(CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE);
}
// prune candidate dims without any valid expressions
if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) {
@@ -699,9 +720,7 @@ class ExpressionResolver implements ContextRewriter {
Set<ExpressionContext> ecSet = ecEntry.getValue();
for (ExpressionContext ec : ecSet) {
if (ec.getSrcTable().getName().equals(cdim.getBaseTable().getName())) {
- if (!ec.directlyAvailableIn.contains(cdim)
- && (ec.evaluableExpressions.get(cdim) == null
- || ec.evaluableExpressions.get(cdim).isEmpty())) {
+ if (!ec.isEvaluable(cdim)) {
log.info("Not considering dim table:{} as {} is not evaluable", cdim, ec.exprCol.getName());
cubeql.addDimPruningMsgs(dim, cdim.dimtable,
CandidateTablePruneCause.expressionNotEvaluable(ec.exprCol.getName()));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/f1387c31/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
index 53996c4..f65bd28 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestBaseCubeQueries.java
@@ -33,6 +33,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lens.cube.metadata.UpdatePeriod;
+import org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.time.DateUtils;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.antlr.runtime.CommonToken;
+import org.testng.Assert;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
@@ -83,18 +85,18 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
String columnSetsStr = matcher.group(1);
assertNotEquals(columnSetsStr.indexOf("stateid"), -1);
assertNotEquals(columnSetsStr.indexOf("msr3, msr13"), -1);
- assertEquals(pruneCauses.getDetails(),
- new HashMap<String, List<CandidateTablePruneCause>>() {
- {
- put("testfact3_base,testfact3_raw_base", Arrays.asList(CandidateTablePruneCause.columnNotFound("stateid")));
- put("testfact_deprecated,testfact2_raw_base,testfact2_base",
- Arrays.asList(CandidateTablePruneCause.columnNotFound("msr3", "msr13")));
- }
- }
- );
+ assertEquals(pruneCauses.getDetails().get("testfact3_base,testfact3_raw_base"),
+ Arrays.asList(CandidateTablePruneCause.columnNotFound("stateid")));
+ assertEquals(pruneCauses.getDetails().get("testfact_deprecated,testfact2_raw_base,testfact2_base"),
+ Arrays.asList(CandidateTablePruneCause.columnNotFound("msr3", "msr13")));
+ assertTrue(pruneCauses.getDetails().containsKey("testfact1_base,testfact1_raw_base")
+ || pruneCauses.getDetails().containsKey("testfact1_raw_base,testfact1_base"));
+ String fact1BaseKey = pruneCauses.getDetails().containsKey("testfact1_base,testfact1_raw_base")
+ ? "testfact1_base,testfact1_raw_base" : "testfact1_raw_base,testfact1_base";
+ assertEquals(pruneCauses.getDetails().get(fact1BaseKey),
+ Arrays.asList(new CandidateTablePruneCause(CandidateTablePruneCode.ELEMENT_IN_SET_PRUNED)));
}
-
@Test
public void testCommonDimensions() throws Exception {
String hqlQuery = rewrite("select dim1, SUM(msr1) from basecube" + " where " + TWO_DAYS_RANGE, conf);
@@ -132,8 +134,15 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
}
@Test
- public void testMultipleFacts() throws Exception {
- String hqlQuery = rewrite("select roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ public void testMultiFactQueryWithNoDimensionsSelected() throws Exception {
+ CubeQueryContext ctx = rewriteCtx("select roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ Set<String> candidateFacts = new HashSet<String>();
+ for (CandidateFact cfact : ctx.getCandidateFacts()) {
+ candidateFacts.add(cfact.getName().toLowerCase());
+ }
+ Assert.assertTrue(candidateFacts.contains("testfact1_base"));
+ Assert.assertTrue(candidateFacts.contains("testfact2_base"));
+ String hqlQuery = ctx.toHQL();
String expected1 =
getExpectedQuery(cubeName, "select sum(basecube.msr12) msr12 FROM ", null,
null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
@@ -147,17 +156,20 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
|| lower.startsWith("select mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12 from "), hqlQuery);
assertTrue(lower.contains("mq1 full outer join") && lower.endsWith("mq2"), hqlQuery);
assertFalse(lower.contains("<=>"), hqlQuery);
+ }
- hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ @Test
+ public void testMultiFactQueryWithSingleCommonDimension() throws Exception {
+ String hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ String expected1 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, sum(basecube.msr12) msr12 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, round(sum(basecube.msr2)/1000) roundedmsr2 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
TestCubeRewriter.compareContains(expected2, hqlQuery);
- lower = hqlQuery.toLowerCase();
+ String lower = hqlQuery.toLowerCase();
assertTrue(
lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.roundedmsr2 roundedmsr2, mq1.msr12 msr12 from ")
|| lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12"
@@ -165,18 +177,44 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"),
hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryWithSingleCommonDimensionWithLightestFactFirst() throws Exception {
+ Configuration tConf = new Configuration(conf);
+ tConf.setBoolean(CubeQueryConfUtil.LIGHTEST_FACT_FIRST, true);
+ String hqlQuery = rewrite("select dim1, roundedmsr2, msr12 from basecube" + " where " + TWO_DAYS_RANGE, tConf);
+ String expected1 =
+ getExpectedQuery(cubeName, "select basecube.dim1 dim1, sum(basecube.msr12) msr12 FROM ", null,
+ " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
+ String expected2 =
+ getExpectedQuery(cubeName, "select basecube.dim1 dim1, round(sum(basecube.msr2)/1000) roundedmsr2 FROM ", null,
+ " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
+ TestCubeRewriter.compareContains(expected1, hqlQuery);
+ TestCubeRewriter.compareContains(expected2, hqlQuery);
+ String lower = hqlQuery.toLowerCase();
+ assertTrue(
+ lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.roundedmsr2 roundedmsr2, mq1.msr12 msr12 from ")
+ || lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.roundedmsr2 roundedmsr2, mq2.msr12 msr12"
+ + " from "), hqlQuery);
+
+ assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"),
+ hqlQuery);
+ }
+
+ @Test
+ public void testMultiFactQueryWithSingleCommonDimensionWithColumnsSwapped() throws Exception {
// columns in select interchanged
- hqlQuery = rewrite("select dim1, msr12, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ String hqlQuery = rewrite("select dim1, msr12, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ String expected1 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, sum(basecube.msr12) msr12 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, round(sum(basecube.msr2)/1000) roundedmsr2 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
TestCubeRewriter.compareContains(expected2, hqlQuery);
- lower = hqlQuery.toLowerCase();
+ String lower = hqlQuery.toLowerCase();
assertTrue(
lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq2.msr12 msr12, mq1.roundedmsr2 roundedmsr2 from ")
|| lower.startsWith("select coalesce(mq1.dim1, mq2.dim1) dim1, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2"
@@ -184,13 +222,17 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"),
hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryInvolvingThreeFactTables() throws Exception {
// query with 3 fact tables
- hqlQuery = rewrite("select dim1, msr12, roundedmsr2, msr13, msr3 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ String hqlQuery = rewrite("select dim1, msr12, roundedmsr2, msr13, msr3 from basecube where " + TWO_DAYS_RANGE,
+ conf);
+ String expected1 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, sum(basecube.msr12) msr12 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName,
"select basecube.dim1 dim1, round(sum(basecube.msr2)/1000) roundedmsr2, max(basecube.msr3) msr3 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -221,13 +263,16 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
+ " mq2.roundedmsr2 roundedmsr2, mq1.msr13 msr13, mq2.msr3 msr3 from "), hqlQuery);
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.contains("mq2 full outer join ")
&& hqlQuery.endsWith("mq3 on mq1.dim1 <=> mq2.dim1 AND mq1.dim1 <=> mq3.dim1"), hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryWithTwoCommonDimensions() throws Exception {
// query two dim attributes
- hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ String hqlQuery = rewrite("select dim1, dim11, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf);
+ String expected1 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, basecube.dim11 dim11, sum(basecube.msr12) msr12 FROM ",
null, " group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName,
"select basecube.dim1 dim1, basecube.dim11 dim11, round(sum(basecube.msr2)/1000) roundedmsr2 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
@@ -242,13 +287,16 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.contains("mq1 full outer join ")
&& hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1 AND mq1.dim11 <=> mq2.dim11"), hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryWithNoAggregates() throws Exception {
// no aggregates in the query
- hqlQuery = rewrite("select dim1, msr11, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ String hqlQuery = rewrite("select dim1, msr11, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ String expected1 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, basecube.msr11 msr11 FROM ", null, null,
getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, round(basecube.msr2/1000) roundedmsr2 FROM ", null, null,
getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
@@ -260,14 +308,17 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"),
hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryWithColumnAliases() throws Exception {
// query with aliases passed
- hqlQuery =
+ String hqlQuery =
rewrite("select dim1 d1, msr12 `my msr12`, roundedmsr2 m2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ String expected1 =
getExpectedQuery(cubeName, "select basecube.dim1 d1, sum(basecube.msr12) expr2 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName, "select basecube.dim1 d1, round(sum(basecube.msr2)/1000) m2 FROM ", null,
" group by basecube.dim1", getWhereForDailyAndHourly2days(cubeName, "C1_testFact1_BASE"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
@@ -278,13 +329,16 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
hqlQuery.toLowerCase().startsWith("select coalesce(mq1.d1, mq2.d1) d1, mq1.expr2 `my msr12`, mq2.m2 m2 from "),
hqlQuery);
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.d1 <=> mq2.d1"), hqlQuery);
+ }
- // query with non default aggregate
- hqlQuery = rewrite("select dim1, avg(msr12), avg(msr2) from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ @Test
+ public void testMultiFactQueryWithNoDefaultAggregates() throws Exception {
+ // query with non default aggregate
+ String hqlQuery = rewrite("select dim1, avg(msr12), avg(msr2) from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ String expected1 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, avg(basecube.msr12) msr12 FROM ", null,
" group by basecube.dim1", getWhereForHourly2days(cubeName, "C1_testfact2_raw_base"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName, "select basecube.dim1 dim1, avg(basecube.msr2)) msr2 FROM ", null,
" group by basecube.dim1", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
@@ -296,14 +350,17 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim1 <=> mq2.dim1"),
hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryWithJoins() throws Exception {
// query with join
- hqlQuery = rewrite("select testdim2.name, msr12, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ String hqlQuery = rewrite("select testdim2.name, msr12, roundedmsr2 from basecube where " + TWO_DAYS_RANGE, conf);
+ String expected1 =
getExpectedQuery(cubeName, "select testdim2.name name, sum(basecube.msr12) msr12 FROM ", " JOIN " + getDbName()
+ "c1_testdim2tbl testdim2 ON basecube.dim2 = " + " testdim2.id and (testdim2.dt = 'latest') ", null,
" group by testdim2.name", null, getWhereForDailyAndHourly2days(cubeName, "C1_testFact2_BASE"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName, "select testdim2.name name, round(sum(basecube.msr2)/1000) roundedmsr2 FROM ", " JOIN "
+ getDbName() + "c1_testdim2tbl testdim2 ON basecube.dim2 = "
+ " testdim2.id and (testdim2.dt = 'latest') ", null, " group by testdim2.name", null,
@@ -316,14 +373,17 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
"select coalesce(mq1.name, mq2.name) name, mq1.msr12 msr12, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery);
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.name <=> mq2.name"),
hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryWithDenormColumn() throws Exception {
// query with denorm variable
- hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
- expected1 =
+ String hqlQuery = rewrite("select dim2, msr13, roundedmsr2 from basecube" + " where " + TWO_DAYS_RANGE, conf);
+ String expected1 =
getExpectedQuery(cubeName, "select testdim2.id dim2, max(basecube.msr13) msr13 FROM ", " JOIN " + getDbName()
+ "c1_testdim2tbl testdim2 ON basecube.dim12 = " + " testdim2.id and (testdim2.dt = 'latest') ", null,
" group by testdim2.id", null, getWhereForHourly2days(cubeName, "C1_testFact3_RAW_BASE"));
- expected2 =
+ String expected2 =
getExpectedQuery(cubeName, "select basecube.dim2 dim2, round(sum(basecube.msr2)/1000) roundedmsr2 FROM ", null,
" group by basecube.dim2", getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
@@ -334,33 +394,38 @@ public class TestBaseCubeQueries extends TestQueryRewrite {
"select coalesce(mq1.dim2, mq2.dim2) dim2, mq1.msr13 msr13, mq2.roundedmsr2 roundedmsr2 from "), hqlQuery);
assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.dim2 <=> mq2.dim2"),
hqlQuery);
+ }
+ @Test
+ public void testMultiFactQueryWithExpressionInvolvingDenormVariable() throws Exception {
// query with expression
- // TODO allow expression to be answered from denorm columns
- /*
- hqlQuery =
+ // The expression to be answered from denorm columns
+ String hqlQuery =
rewrite(
"select booleancut, round(sum(msr2)/1000), avg(msr13 + msr14) from basecube" + " where " + TWO_DAYS_RANGE,
conf);
- expected1 =
- getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND testdim2.id != 10 expr1,"
+ String expected1 =
+ getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND testdim2.id != 10 booleancut,"
+ " avg(basecube.msr13 + basecube.msr14) expr3 FROM ", " JOIN " + getDbName()
+ "c1_testdim2tbl testdim2 ON basecube.dim12 = " + " testdim2.id and (testdim2.dt = 'latest') ", null,
" group by basecube.dim1 != 'x' AND testdim2.id != 10", null,
getWhereForHourly2days(cubeName, "C1_testfact3_raw_base"));
- expected2 =
- getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND basecube.dim2 != 10 expr1,"
+ String expected2 =
+ getExpectedQuery(cubeName, "select basecube.dim1 != 'x' AND basecube.dim2 != 10 booleancut,"
+ " round(sum(basecube.msr2)/1000) msr2 FROM ", null,
" group by basecube.dim1 != 'x' AND basecube.dim2 != 10",
getWhereForHourly2days(cubeName, "C1_testfact1_raw_base"));
TestCubeRewriter.compareContains(expected1, hqlQuery);
TestCubeRewriter.compareContains(expected2, hqlQuery);
assertTrue(hqlQuery.toLowerCase()
- .startsWith("select coalesce(mq1.expr1, mq2.expr1) expr1, mq2.msr2 msr2, mq1.expr3 expr3 from ")
+ .startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, mq2.msr2 msr2,"
+ + " mq1.expr3 expr3 from ")
|| hqlQuery.toLowerCase()
- .startsWith("select coalesce(mq1.expr1, mq2.expr1) expr1, mq1.msr2 msr2, mq2.expr3 expr3 from "), hqlQuery);
- assertTrue(hqlQuery.contains("mq1 full outer join ") && hqlQuery.endsWith("mq2 on mq1.expr1 <=> mq2.expr1"),
- hqlQuery);*/
+ .startsWith("select coalesce(mq1.booleancut, mq2.booleancut) booleancut, mq1.msr2 msr2,"
+ + " mq2.expr3 expr3 from "), hqlQuery);
+ assertTrue(hqlQuery.contains("mq1 full outer join ")
+ && hqlQuery.endsWith("mq2 on mq1.booleancut <=> mq2.booleancut"),
+ hqlQuery);
}
@Test
[04/50] [abbrv] incubator-lens git commit: [maven-release-plugin]
prepare for next development iteration
Posted by sh...@apache.org.
[maven-release-plugin] prepare for next development iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/68166c97
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/68166c97
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/68166c97
Branch: refs/heads/LENS-581
Commit: 68166c97854384f6ad04407bc15398e00791493b
Parents: 9c45f1c
Author: jdhok <jd...@apache.org>
Authored: Tue Jul 7 16:50:58 2015 +0530
Committer: jdhok <jd...@apache.org>
Committed: Tue Jul 7 16:50:58 2015 +0530
----------------------------------------------------------------------
checkstyle/pom.xml | 6 +-----
lens-api/pom.xml | 2 +-
lens-cli/pom.xml | 2 +-
lens-client/pom.xml | 2 +-
lens-cube/pom.xml | 2 +-
lens-dist/pom.xml | 2 +-
lens-driver-hive/pom.xml | 2 +-
lens-driver-jdbc/pom.xml | 2 +-
lens-examples/pom.xml | 2 +-
lens-ml-dist/pom.xml | 2 +-
lens-ml-lib/pom.xml | 2 +-
lens-query-lib/pom.xml | 2 +-
lens-regression/pom.xml | 2 +-
lens-server-api/pom.xml | 2 +-
lens-server/pom.xml | 2 +-
lens-storage-db/pom.xml | 2 +-
pom.xml | 8 ++++----
17 files changed, 20 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/checkstyle/pom.xml
----------------------------------------------------------------------
diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml
index cd45d00..b57a2f7 100644
--- a/checkstyle/pom.xml
+++ b/checkstyle/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
<name>Lens Checkstyle Rules</name>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
<build>
<plugins>
@@ -49,8 +49,4 @@
</plugin>
</plugins>
</build>
-
- <scm>
- <tag>apache-lens-2.2.0-beta-incubating</tag>
- </scm>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index 250102d..c3f7fb7 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-cli/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml
index 26d34d7..cb83eeb 100644
--- a/lens-cli/pom.xml
+++ b/lens-cli/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-client/pom.xml
----------------------------------------------------------------------
diff --git a/lens-client/pom.xml b/lens-client/pom.xml
index 708c399..d22e9de 100644
--- a/lens-client/pom.xml
+++ b/lens-client/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-cube/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml
index eb750e1..74be429 100644
--- a/lens-cube/pom.xml
+++ b/lens-cube/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-cube</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml
index b5c6b6c..43ddc1c 100644
--- a/lens-dist/pom.xml
+++ b/lens-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-driver-hive/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index cebed79..4489779 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-driver-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml
index e73895d..4d884d4 100644
--- a/lens-driver-jdbc/pom.xml
+++ b/lens-driver-jdbc/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-driver-jdbc</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-examples/pom.xml
----------------------------------------------------------------------
diff --git a/lens-examples/pom.xml b/lens-examples/pom.xml
index e9ae2a8..75b58cc 100644
--- a/lens-examples/pom.xml
+++ b/lens-examples/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-ml-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
index d7b69f5..6130a50 100644
--- a/lens-ml-dist/pom.xml
+++ b/lens-ml-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-ml-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-ml-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index 32a3be8..aad2c0d 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-query-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-query-lib/pom.xml b/lens-query-lib/pom.xml
index 6a7837f..2100d84 100644
--- a/lens-query-lib/pom.xml
+++ b/lens-query-lib/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-query-lib</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-regression/pom.xml
----------------------------------------------------------------------
diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml
index 5679a72..14148ff 100644
--- a/lens-regression/pom.xml
+++ b/lens-regression/pom.xml
@@ -25,7 +25,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-regression</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-server-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml
index 09871f4..fe66040 100644
--- a/lens-server-api/pom.xml
+++ b/lens-server-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-server-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index e436e32..c7df0a1 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/lens-storage-db/pom.xml
----------------------------------------------------------------------
diff --git a/lens-storage-db/pom.xml b/lens-storage-db/pom.xml
index ce27408..0c6886b 100644
--- a/lens-storage-db/pom.xml
+++ b/lens-storage-db/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</parent>
<artifactId>lens-storage-db</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/68166c97/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e665568..ce24f1b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -114,7 +114,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>apache-lens</artifactId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
<name>Lens</name>
<packaging>pom</packaging>
<description>Unified Analytics Platform</description>
@@ -299,7 +299,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-lens.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-lens.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/incubator-lens.git</url>
- <tag>apache-lens-2.2.0-beta-incubating</tag>
+ <tag>HEAD</tag>
</scm>
<reporting>
@@ -574,7 +574,7 @@
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
<executions>
@@ -1322,7 +1322,7 @@
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
- <version>2.2.0-beta-incubating</version>
+ <version>2.2.1-beta-incubating-SNAPSHOT</version>
</dependency>
</dependencies>
<executions>
[44/50] [abbrv] incubator-lens git commit: LENS-686: Repetitive add
partition with same specs should fail after first time
Posted by sh...@apache.org.
LENS-686: Repetitive add partition with same specs should fail after first time
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/11593f46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/11593f46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/11593f46
Branch: refs/heads/LENS-581
Commit: 11593f468d39126ffe6dfc27cb7eb6f3493a13cc
Parents: c5c4f76
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Wed Jul 29 06:56:27 2015 +0530
Committer: Raju Bairishetti <ra...@apache.org>
Committed: Wed Jul 29 06:56:27 2015 +0530
----------------------------------------------------------------------
.../lens/cube/metadata/CubeMetastoreClient.java | 50 ++--
.../org/apache/lens/cube/metadata/Storage.java | 9 +-
.../api/metastore/CubeMetastoreService.java | 34 +--
.../metastore/CubeMetastoreServiceImpl.java | 64 ++---
.../apache/lens/server/metastore/JAXBUtils.java | 12 +
.../server/metastore/MetastoreResource.java | 246 ++++++++-----------
.../server/metastore/TestMetastoreService.java | 30 +++
7 files changed, 218 insertions(+), 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/11593f46/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 0ebc707..f0a0dfe 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -19,6 +19,8 @@
package org.apache.lens.cube.metadata;
+import static org.apache.lens.cube.metadata.MetastoreUtil.getFactOrDimtableStorageTableName;
+
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
@@ -189,7 +191,7 @@ public class CubeMetastoreClient {
if (isFactTable(factOrDimtableName)) {
return Lists.newArrayList();
}
- String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factOrDimtableName, storageName);
+ String storageTableName = getFactOrDimtableStorageTableName(factOrDimtableName, storageName);
Table storageTable = getTable(storageTableName);
List<String> timePartCols = getTimePartColNamesOfTable(storageTable);
List<Partition> latestParts = Lists.newArrayList();
@@ -782,23 +784,26 @@ public class CubeMetastoreClient {
* @param storageName The storage object
* @throws HiveException
*/
- public void addPartition(StoragePartitionDesc partSpec, String storageName) throws HiveException, LensException {
- addPartitions(Collections.singletonList(partSpec), storageName);
+ public List<Partition> addPartition(StoragePartitionDesc partSpec, String storageName)
+ throws HiveException, LensException {
+ return addPartitions(Collections.singletonList(partSpec), storageName);
}
/** batch addition */
- public void addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName)
+ public List<Partition> addPartitions(List<StoragePartitionDesc> storagePartitionDescs, String storageName)
throws HiveException, LensException {
+ List<Partition> partsAdded = Lists.newArrayList();
for (Map.Entry<String, Map<UpdatePeriod, List<StoragePartitionDesc>>> group : groupPartitionDescs(
storagePartitionDescs).entrySet()) {
String factOrDimtable = group.getKey();
for (Map.Entry<UpdatePeriod, List<StoragePartitionDesc>> entry : group.getValue().entrySet()) {
- addPartitions(factOrDimtable, storageName, entry.getKey(), entry.getValue());
+ partsAdded.addAll(addPartitions(factOrDimtable, storageName, entry.getKey(), entry.getValue()));
}
}
+ return partsAdded;
}
- private void addPartitions(String factOrDimTable, String storageName, UpdatePeriod updatePeriod,
+ private List<Partition> addPartitions(String factOrDimTable, String storageName, UpdatePeriod updatePeriod,
List<StoragePartitionDesc> storagePartitionDescs) throws HiveException, LensException {
String storageTableName = MetastoreUtil.getStorageTableName(factOrDimTable.trim(),
Storage.getPrefix(storageName.trim())).toLowerCase();
@@ -810,17 +815,27 @@ public class CubeMetastoreClient {
latestInfos.put(entry.getKey(),
getDimTableLatestInfo(storageTableName, entry.getKey(), getTimePartSpecs(entry.getValue()), updatePeriod));
}
- getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs,
- latestInfos);
+ List<Partition> partsAdded =
+ getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs,
+ latestInfos);
+ ListIterator<Partition> iter = partsAdded.listIterator();
+ while (iter.hasNext()) {
+ if (iter.next().getSpec().values().contains(StorageConstants.LATEST_PARTITION_VALUE)) {
+ iter.remove();
+ }
+ }
latestLookupCache.add(storageTableName);
+ return partsAdded;
} else {
// first update in memory, then add to hive table's partitions. delete is reverse.
partitionTimelineCache.updateForAddition(factOrDimTable, storageName, updatePeriod,
getTimePartSpecs(storagePartitionDescs));
// Adding partition in fact table.
- getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs, null);
+ List<Partition> partsAdded =
+ getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, storagePartitionDescs, null);
// update hive table
alterTablePartitionCache(MetastoreUtil.getStorageTableName(factOrDimTable, Storage.getPrefix(storageName)));
+ return partsAdded;
}
}
@@ -1112,7 +1127,7 @@ public class CubeMetastoreClient {
public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod,
Map<String, Date> partitionTimestamp, Map<String, String> partSpec) throws HiveException {
- String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName);
+ String storageTableName = getFactOrDimtableStorageTableName(factName, storageName);
return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec);
}
@@ -1185,7 +1200,7 @@ public class CubeMetastoreClient {
boolean dimPartitionExists(String dimTblName, String storageName, Map<String, Date> partitionTimestamps)
throws HiveException {
- String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName);
+ String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storageName);
return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName),
partitionTimestamps);
}
@@ -1232,7 +1247,10 @@ public class CubeMetastoreClient {
public Table getHiveTable(String tableName) throws HiveException {
return getTable(tableName);
}
-
+ public List<String> getTimePartColNamesOfTable(String tblName, String storageName) throws HiveException {
+ return getTimePartColNamesOfTable(getFactOrDimtableStorageTableName(tblName,
+ storageName));
+ }
public List<String> getTimePartColNamesOfTable(String storageTableName) throws HiveException {
return getTimePartColNamesOfTable(getTable(storageTableName));
}
@@ -1449,7 +1467,7 @@ public class CubeMetastoreClient {
if (dimTable.getStorages() != null && !dimTable.getStorages().isEmpty()) {
for (String storageName : dimTable.getStorages()) {
if (dimTable.hasStorageSnapshots(storageName)) {
- String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(),
+ String storageTableName = getFactOrDimtableStorageTableName(dimTable.getName(),
storageName);
if (dimLatestPartitionExists(storageTableName,
getDimension(dimTable.getDimName()).getTimedDimension())) {
@@ -2046,7 +2064,7 @@ public class CubeMetastoreClient {
public void dropStorageFromFact(String factName, String storage) throws HiveException {
CubeFactTable cft = getFactTable(factName);
cft.dropStorage(storage);
- dropHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storage));
+ dropHiveTable(getFactOrDimtableStorageTableName(factName, storage));
alterCubeTable(factName, getTable(factName), cft);
updateFactCache(factName);
}
@@ -2054,7 +2072,7 @@ public class CubeMetastoreClient {
// updateFact will be false when fact is fully dropped
private void dropStorageFromFact(String factName, String storage, boolean updateFact) throws HiveException {
CubeFactTable cft = getFactTable(factName);
- dropHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storage));
+ dropHiveTable(getFactOrDimtableStorageTableName(factName, storage));
if (updateFact) {
cft.dropStorage(storage);
alterCubeTable(factName, getTable(factName), cft);
@@ -2076,7 +2094,7 @@ public class CubeMetastoreClient {
// updateDimTbl will be false when dropping dimTbl
private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws HiveException {
CubeDimensionTable cdt = getDimensionTable(dimTblName);
- String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storage);
+ String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storage);
dropHiveTable(storageTableName);
latestLookupCache.remove(storageTableName.trim().toLowerCase());
if (updateDimTbl) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/11593f46/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
index 873aa49..437227c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/Storage.java
@@ -239,11 +239,11 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
* @param latestInfo
* @throws HiveException
*/
- public void addPartition(Hive client, StoragePartitionDesc addPartitionDesc, LatestInfo latestInfo)
+ public List<Partition> addPartition(Hive client, StoragePartitionDesc addPartitionDesc, LatestInfo latestInfo)
throws HiveException {
Map<Map<String, String>, LatestInfo> latestInfos = Maps.newHashMap();
latestInfos.put(addPartitionDesc.getNonTimePartSpec(), latestInfo);
- addPartitions(client, addPartitionDesc.getCubeTableName(), addPartitionDesc.getUpdatePeriod(),
+ return addPartitions(client, addPartitionDesc.getCubeTableName(), addPartitionDesc.getUpdatePeriod(),
Collections.singletonList(addPartitionDesc), latestInfos);
}
@@ -258,7 +258,7 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
* column
* @throws HiveException
*/
- public void addPartitions(Hive client, String factOrDimTable, UpdatePeriod updatePeriod,
+ public List<Partition> addPartitions(Hive client, String factOrDimTable, UpdatePeriod updatePeriod,
List<StoragePartitionDesc> storagePartitionDescs,
Map<Map<String, String>, LatestInfo> latestInfos) throws HiveException {
preAddPartitions(storagePartitionDescs);
@@ -348,8 +348,9 @@ public abstract class Storage extends AbstractCubeTable implements PartitionMeta
}
}
}
- client.createPartitions(addParts);
+ List<Partition> partitionsAdded = client.createPartitions(addParts);
success = true;
+ return partitionsAdded;
} finally {
if (success) {
commitAddPartitions(storagePartitionDescs);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/11593f46/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index d7a033e..c11fd83 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -337,14 +337,14 @@ public interface CubeMetastoreService extends LensService {
* Get all partitions of a dimension table in a storage
*
* @param sessionid The sessionid
- * @param dimTblName The dimension table name
- * @param storage The storage name
+ * @param dimension The dimension table name
+ * @param storageName The storage name
* @param filter The filter for the list of partitions
* @return list of {@link XPartition}
* @throws LensException
*/
- List<XPartition> getAllPartitionsOfDimTableStorage(LensSessionHandle sessionid, String dimTblName,
- String storage, String filter) throws LensException;
+ XPartitionList getAllPartitionsOfDimTableStorage(LensSessionHandle sessionid, String dimension,
+ String storageName, String filter) throws LensException;
/**
* Add partition to dimension table on a storage.
@@ -354,9 +354,10 @@ public interface CubeMetastoreService extends LensService {
* @param storageName The storage name
* @param partition {@link XPartition}
* @throws LensException
+ * @return number of partitions added. Either 0 or 1
*/
- void addPartitionToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName,
- XPartition partition) throws LensException;
+ int addPartitionToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName,
+ XPartition partition) throws LensException;
/**
* Add partitions to dimension table on a storage.
@@ -366,9 +367,10 @@ public interface CubeMetastoreService extends LensService {
* @param storageName The storage name
* @param partitions {@link XPartitionList}
* @throws LensException
+ * @return Number of partitions added
*/
- void addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName,
- XPartitionList partitions) throws LensException;
+ int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName,
+ XPartitionList partitions) throws LensException;
/**
* Get fact table given by name
@@ -475,13 +477,13 @@ public interface CubeMetastoreService extends LensService {
*
* @param sessionid The sessionid
* @param fact The fact table name
- * @param storage The storage name
+ * @param storageName The storage name
* @param filter The filter for partition listing
* @return List of {@link XPartition}
* @throws LensException
*/
- List<XPartition> getAllPartitionsOfFactStorage(LensSessionHandle sessionid, String fact,
- String storage, String filter) throws LensException;
+ XPartitionList getAllPartitionsOfFactStorage(LensSessionHandle sessionid, String fact,
+ String storageName, String filter) throws LensException;
/**
* Add partition to fact on a storage
@@ -491,9 +493,10 @@ public interface CubeMetastoreService extends LensService {
* @param storageName The storage name
* @param partition {@link XPartition}
* @throws LensException
+ * @return number of partitions added. Either 0 or 1
*/
- void addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
- XPartition partition) throws LensException;
+ int addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
+ XPartition partition) throws LensException;
/**
* Add partitions to fact on a storage
@@ -503,9 +506,10 @@ public interface CubeMetastoreService extends LensService {
* @param storageName The storage name
* @param partitions {@link XPartitionList}
* @throws LensException
+ * @return Number of partitions added
*/
- void addPartitionsToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
- XPartitionList partitions) throws LensException;
+ int addPartitionsToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
+ XPartitionList partitions) throws LensException;
/**
* Drop partition from storage with spec specified as comma separated string
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/11593f46/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index cb7a530..37cebfe 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.server.metastore;
+import static org.apache.lens.server.metastore.JAXBUtils.*;
+
import java.util.*;
import javax.ws.rs.BadRequestException;
@@ -692,7 +694,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
- public List<XPartition> getAllPartitionsOfFactStorage(
+ public XPartitionList getAllPartitionsOfFactStorage(
LensSessionHandle sessionid, String fact, String storageName,
String filter) throws LensException {
try {
@@ -703,16 +705,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
storageName);
List<Partition> parts = client.getPartitionsByFilter(storageTableName, filter);
List<String> timePartCols = client.getTimePartColNamesOfTable(storageTableName);
- if (parts != null) {
- List<XPartition> result = new ArrayList<XPartition>(parts.size());
- for (Partition p : parts) {
- XPartition xp = JAXBUtils.xpartitionFromPartition(p, timePartCols);
- result.add(xp);
- }
- return result;
- } else {
- return new ArrayList<>();
- }
+ return xpartitionListFromPartitionList(parts, timePartCols);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -721,14 +714,12 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
- public void addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
+ public int addPartitionToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
XPartition partition) throws LensException {
try {
acquire(sessionid);
checkFactStorage(sessionid, fact, storageName);
- getClient(sessionid).addPartition(
- JAXBUtils.storagePartSpecFromXPartition(partition),
- storageName);
+ return getClient(sessionid).addPartition(storagePartSpecFromXPartition(partition), storageName).size();
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -737,14 +728,12 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
- public void addPartitionsToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
+ public int addPartitionsToFactStorage(LensSessionHandle sessionid, String fact, String storageName,
XPartitionList partitions) throws LensException {
try {
acquire(sessionid);
checkFactStorage(sessionid, fact, storageName);
- getClient(sessionid).addPartitions(
- JAXBUtils.storagePartSpecListFromXPartitionList(partitions),
- storageName);
+ return getClient(sessionid).addPartitions(storagePartSpecListFromXPartitionList(partitions), storageName).size();
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -766,27 +755,17 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
- public List<XPartition> getAllPartitionsOfDimTableStorage(
+ public XPartitionList getAllPartitionsOfDimTableStorage(
LensSessionHandle sessionid, String dimension, String storageName, String filter)
throws LensException {
try {
acquire(sessionid);
checkDimensionStorage(sessionid, dimension, storageName);
CubeMetastoreClient client = getClient(sessionid);
- String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimension,
- storageName);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimension, storageName);
List<Partition> partitions = client.getPartitionsByFilter(storageTableName, filter);
List<String> timePartCols = client.getTimePartColNamesOfTable(storageTableName);
- if (partitions != null) {
- List<XPartition> result = new ArrayList<XPartition>(partitions.size());
- for (Partition p : partitions) {
- XPartition xp = JAXBUtils.xpartitionFromPartition(p, timePartCols);
- result.add(xp);
- }
- return result;
- } else {
- return new ArrayList<XPartition>();
- }
+ return xpartitionListFromPartitionList(partitions, timePartCols);
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -795,15 +774,12 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
- public void addPartitionToDimStorage(LensSessionHandle sessionid,
+ public int addPartitionToDimStorage(LensSessionHandle sessionid,
String dimTblName, String storageName, XPartition partition) throws LensException {
try {
acquire(sessionid);
checkDimensionStorage(sessionid, dimTblName, storageName);
- getClient(sessionid).addPartition(
- JAXBUtils.storagePartSpecFromXPartition(partition),
- storageName);
- log.info("Added partition for dimension: " + dimTblName + " storage: " + storageName);
+ return getClient(sessionid).addPartition(storagePartSpecFromXPartition(partition), storageName).size();
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -822,7 +798,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition)));
JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition);
client.updatePartition(tblName, storageName, existingPartition);
- } catch (HiveException | ClassNotFoundException |InvalidOperationException | UnsupportedOperationException exc) {
+ } catch (HiveException | ClassNotFoundException | InvalidOperationException | UnsupportedOperationException exc) {
throw new LensException(exc);
} finally {
release(sessionid);
@@ -852,14 +828,12 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
@Override
- public void addPartitionsToDimStorage(LensSessionHandle sessionid,
+ public int addPartitionsToDimStorage(LensSessionHandle sessionid,
String dimTblName, String storageName, XPartitionList partitions) throws LensException {
try {
acquire(sessionid);
checkDimensionStorage(sessionid, dimTblName, storageName);
- getClient(sessionid).addPartitions(
- JAXBUtils.storagePartSpecListFromXPartitionList(partitions),
- storageName);
+ return getClient(sessionid).addPartitions(storagePartSpecListFromXPartitionList(partitions), storageName).size();
} catch (HiveException exc) {
throw new LensException(exc);
} finally {
@@ -1481,7 +1455,7 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
boolean isHealthy = true;
StringBuilder details = new StringBuilder();
- try{
+ try {
/** Try to issue command on hive **/
Hive.get(LensServerConf.getHiveConf()).getAllDatabases();
} catch (HiveException e) {
@@ -1498,8 +1472,8 @@ public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMet
}
return isHealthy
- ? new HealthStatus(isHealthy, "Cube metastore service is healthy.")
- : new HealthStatus(isHealthy, details.toString());
+ ? new HealthStatus(isHealthy, "Cube metastore service is healthy.")
+ : new HealthStatus(isHealthy, details.toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/11593f46/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 2d0eba2..b7c4b40 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -785,6 +785,18 @@ public final class JAXBUtils {
return nonTimePartSpec;
}
+ public static XPartitionList xpartitionListFromPartitionList(List<Partition> partitions, List<String> timePartCols)
+ throws HiveException {
+ XPartitionList xPartitionList = new XPartitionList();
+ xPartitionList.getPartition();
+ if (partitions != null) {
+ for (Partition partition : partitions) {
+ xPartitionList.getPartition().add(xpartitionFromPartition(partition, timePartCols));
+ }
+ }
+ return xPartitionList;
+ }
+
public static XPartition xpartitionFromPartition(Partition p, List<String> timePartCols) throws HiveException {
XPartition xp = new XPartition();
xp.setPartitionParameters(new XProperties());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/11593f46/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index d6ef04d..e7ef804 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.server.metastore;
+import static org.apache.lens.api.APIResult.*;
+
import java.util.List;
import javax.ws.rs.*;
@@ -25,7 +27,7 @@ import javax.ws.rs.core.MediaType;
import javax.xml.bind.JAXBElement;
import org.apache.lens.api.*;
-import org.apache.lens.api.APIResult.Status;
+import org.apache.lens.api.APIResult.*;
import org.apache.lens.api.metastore.*;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
@@ -37,7 +39,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.glassfish.jersey.media.multipart.FormDataParam;
import com.google.common.collect.Lists;
-
import lombok.extern.slf4j.Slf4j;
/**
@@ -49,7 +50,6 @@ import lombok.extern.slf4j.Slf4j;
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
@Slf4j
public class MetastoreResource {
- public static final APIResult SUCCESS = new APIResult(Status.SUCCEEDED, "");
public static final ObjectFactory X_CUBE_OBJECT_FACTORY = new ObjectFactory();
public CubeMetastoreService getSvc() {
@@ -135,11 +135,11 @@ public class MetastoreResource {
log.info("Set database:{}", dbName);
try {
getSvc().setCurrentDatabase(sessionid, dbName);
+ return success();
} catch (LensException e) {
log.error("Error changing current database", e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
}
/**
@@ -160,11 +160,11 @@ public class MetastoreResource {
log.info("Drop database {} cascade?{}", dbName, cascade);
try {
getSvc().dropDatabase(sessionid, dbName, cascade);
+ return success();
} catch (LensException e) {
log.error("Error dropping {}", dbName, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
}
/**
@@ -186,11 +186,11 @@ public class MetastoreResource {
try {
getSvc().createDatabase(sessionid, dbName, ignoreIfExisting);
+ return success();
} catch (LensException e) {
log.error("Error creating database {}", dbName, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
}
/**
@@ -289,8 +289,8 @@ public class MetastoreResource {
@Path("cubes")
public APIResult deleteAllCubes(@QueryParam("sessionid") LensSessionHandle sessionid) {
checkSessionId(sessionid);
- boolean failed = false;
List<String> cubeNames = null;
+ LensException exc = null;
int numDeleted = 0;
try {
cubeNames = getSvc().getAllCubeNames(sessionid);
@@ -300,21 +300,9 @@ public class MetastoreResource {
}
} catch (LensException e) {
log.error("Error deleting cubes:", e);
- failed = true;
- }
- if (cubeNames != null && numDeleted == cubeNames.size()) {
- return new APIResult(Status.SUCCEEDED, "Delete of all "
- + "cubes is successful");
- } else {
- assert (failed);
- if (numDeleted == 0) {
- return new APIResult(Status.FAILED, "Delete of all "
- + "cubes has failed");
- } else {
- return new APIResult(Status.PARTIAL, "Delete of all "
- + "cubes is partial");
- }
+ exc = e;
}
+ return successOrPartialOrFailure(numDeleted, cubeNames.size(), exc);
}
/**
@@ -337,9 +325,9 @@ public class MetastoreResource {
checkTableNotFound(e, ((XDerivedCube) cube).getParent());
}
log.error("Error creating cube {}", cube.getName(), e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
@@ -379,9 +367,9 @@ public class MetastoreResource {
}
checkTableNotFound(e, cube.getName());
log.error("Error updating cube {}", cube.getName(), e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -423,9 +411,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, cubeName);
log.error("Error droping cube {}", cubeName, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -463,9 +451,9 @@ public class MetastoreResource {
getSvc().createStorage(sessionid, storage);
} catch (LensException e) {
log.error("Error creating storage {}", storage.getName(), e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -479,8 +467,8 @@ public class MetastoreResource {
@Path("storages")
public APIResult deleteAllStorages(@QueryParam("sessionid") LensSessionHandle sessionid) {
checkSessionId(sessionid);
- boolean failed = false;
List<String> storageNames = null;
+ LensException exc = null;
int numDeleted = 0;
try {
storageNames = getSvc().getAllStorageNames(sessionid);
@@ -490,21 +478,10 @@ public class MetastoreResource {
}
} catch (LensException e) {
log.error("Error deleting storages:", e);
- failed = true;
- }
- if (storageNames != null && numDeleted == storageNames.size()) {
- return new APIResult(Status.SUCCEEDED, "Delete of all "
- + "storages is successful");
- } else {
- assert (failed);
- if (numDeleted == 0) {
- return new APIResult(Status.FAILED, "Delete of all "
- + "storages has failed");
- } else {
- return new APIResult(Status.PARTIAL, "Delete of all "
- + "storages is partial");
- }
+ exc = e;
}
+ assert (numDeleted == storageNames.size() || exc != null);
+ return successOrPartialOrFailure(numDeleted, storageNames.size(), exc);
}
/**
@@ -526,9 +503,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, storageName);
log.error("Error updating storage {}", storageName, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -569,9 +546,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, storageName);
log.error("Error dropping storage {}", storageName, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -609,9 +586,9 @@ public class MetastoreResource {
getSvc().createDimension(sessionid, dimension);
} catch (LensException e) {
log.error("Error creating dimension {}", dimension.getName(), e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -625,9 +602,9 @@ public class MetastoreResource {
@Path("dimensions")
public APIResult deleteAllDimensions(@QueryParam("sessionid") LensSessionHandle sessionid) {
checkSessionId(sessionid);
- boolean failed = false;
List<String> dimNames = null;
int numDeleted = 0;
+ LensException exc = null;
try {
dimNames = getSvc().getAllDimensionNames(sessionid);
for (String dimName : dimNames) {
@@ -636,21 +613,9 @@ public class MetastoreResource {
}
} catch (LensException e) {
log.error("Error deleting dimensions:", e);
- failed = true;
- }
- if (dimNames != null && numDeleted == dimNames.size()) {
- return new APIResult(Status.SUCCEEDED, "Delete of all "
- + "dimensions is successful");
- } else {
- assert (failed);
- if (numDeleted == 0) {
- return new APIResult(Status.FAILED, "Delete of all "
- + "dimensions has failed");
- } else {
- return new APIResult(Status.PARTIAL, "Delete of all "
- + "dimensions is partial");
- }
+ exc = e;
}
+ return successOrPartialOrFailure(numDeleted, dimNames.size(), exc);
}
/**
@@ -672,9 +637,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, dimName);
log.error("Error updating dimension {}", dimName, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -715,9 +680,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, dimName);
log.error("Error dropping dimension {}", dimName, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -789,9 +754,9 @@ public class MetastoreResource {
public APIResult deleteAllFacts(@QueryParam("sessionid") LensSessionHandle sessionid,
@DefaultValue("false") @QueryParam("cascade") boolean cascade) {
checkSessionId(sessionid);
- boolean failed = false;
List<String> factNames = null;
int numDeleted = 0;
+ LensException exc = null;
try {
factNames = getSvc().getAllCubeNames(sessionid);
for (String factName : factNames) {
@@ -800,21 +765,9 @@ public class MetastoreResource {
}
} catch (LensException e) {
log.error("Error deleting cubes:", e);
- failed = true;
- }
- if (factNames != null && numDeleted == factNames.size()) {
- return new APIResult(Status.SUCCEEDED, "Delete of all "
- + "fact tables is successful");
- } else {
- assert (failed);
- if (numDeleted == 0) {
- return new APIResult(Status.FAILED, "Delete of all "
- + "fact tables has failed");
- } else {
- return new APIResult(Status.PARTIAL, "Delete of all "
- + "fact tables is partial");
- }
+ exc = e;
}
+ return successOrPartialOrFailure(numDeleted, factNames.size(), exc);
}
/**
@@ -858,9 +811,9 @@ public class MetastoreResource {
getSvc().createFactTable(sessionid, fact);
} catch (LensException exc) {
log.error("Exception creating fact:", exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -883,9 +836,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error updating fact {}", factName, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -909,9 +862,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error dropping fact {}", factName, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -953,9 +906,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error dropping storages of fact {}", factName, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -978,9 +931,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error adding storage to fact {}", factName, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1004,9 +957,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error dropping storage of fact {}", factName, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1043,10 +996,8 @@ public class MetastoreResource {
@QueryParam("filter") String filter) throws LensException {
checkSessionId(sessionid);
try {
- List<XPartition> partitions = getSvc().getAllPartitionsOfFactStorage(sessionid, factName, storage, filter);
- XPartitionList partList = X_CUBE_OBJECT_FACTORY.createXPartitionList();
- partList.getPartition().addAll(partitions);
- return X_CUBE_OBJECT_FACTORY.createXPartitionList(partList);
+ return X_CUBE_OBJECT_FACTORY
+ .createXPartitionList(getSvc().getAllPartitionsOfFactStorage(sessionid, factName, storage, filter));
} catch (LensException exc) {
checkTableNotFound(exc, factName);
throw exc;
@@ -1076,9 +1027,9 @@ public class MetastoreResource {
} catch (LensException exc) {
log.warn("Got exception while dropping partition.", exc);
checkTableNotFound(exc, factName);
- return new APIResult(Status.PARTIAL, exc.getMessage());
+ return partial(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1100,14 +1051,14 @@ public class MetastoreResource {
checkSessionId(sessionid);
checkNonNullArgs("Partition is null", partition);
try {
- getSvc().addPartitionToFactStorage(sessionid, factName, storage, partition);
+ return successOrPartialOrFailure(getSvc().addPartitionToFactStorage(sessionid, factName, storage, partition), 1);
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
}
+
/**
* updates an existing partition for a storage of fact
*
@@ -1132,9 +1083,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1156,14 +1107,15 @@ public class MetastoreResource {
checkSessionId(sessionid);
checkNonNullArgs("Partition List is null", partitions);
try {
- getSvc().addPartitionsToFactStorage(sessionid, factName, storage, partitions);
+ return successOrPartialOrFailure(getSvc().addPartitionsToFactStorage(sessionid, factName, storage, partitions),
+ partitions.getPartition().size());
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
}
+
/**
* Batch Update partitions for a storage of fact
*
@@ -1188,9 +1140,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, factName);
log.error("Error adding partition to storage of fact {}:{}", factName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1217,9 +1169,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, factName);
log.error("Error dropping partition to storage of fact {}:{}", factName, storage, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1251,9 +1203,9 @@ public class MetastoreResource {
getSvc().createDimensionTable(sessionid, dimensionTable);
} catch (LensException exc) {
log.error("Error creating cube dimension table {}", dimensionTable.getTableName(), exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1275,9 +1227,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, dimTableName);
log.error("Error updating cube dimension table {}", dimTableName, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1300,9 +1252,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, dimension);
log.error("Error dropping cube dimension table {}", dimension, e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1363,9 +1315,9 @@ public class MetastoreResource {
} catch (LensException e) {
checkTableNotFound(e, dimTableName);
log.error("Error creating dimension table storage {}:{}", dimTableName, storageTbl.getStorageName(), e);
- return new APIResult(Status.FAILED, e.getMessage());
+ return failure(e);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1403,9 +1355,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, dimTableName);
log.error("Error dropping storages of dimension table {}", dimTableName, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1428,9 +1380,9 @@ public class MetastoreResource {
} catch (LensException exc) {
checkTableNotFound(exc, dimTableName);
log.error("Error dropping storage of dimension table {}:{}", dimTableName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1451,10 +1403,8 @@ public class MetastoreResource {
@QueryParam("filter") String filter)
throws LensException {
checkSessionId(sessionid);
- List<XPartition> partitions = getSvc().getAllPartitionsOfDimTableStorage(sessionid, dimension, storage, filter);
- XPartitionList partList = X_CUBE_OBJECT_FACTORY.createXPartitionList();
- partList.getPartition().addAll(partitions);
- return X_CUBE_OBJECT_FACTORY.createXPartitionList(partList);
+ return X_CUBE_OBJECT_FACTORY
+ .createXPartitionList(getSvc().getAllPartitionsOfDimTableStorage(sessionid, dimension, storage, filter));
}
/**
@@ -1479,9 +1429,9 @@ public class MetastoreResource {
getSvc().dropPartitionFromStorageByFilter(sessionid, dimTableName, storage, filter);
} catch (LensException exc) {
log.error("Error dropping partition on storage of dimension table {}:{}", dimTableName, storage, exc);
- return new APIResult(Status.PARTIAL, exc.getMessage());
+ return partial(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1506,9 +1456,9 @@ public class MetastoreResource {
values);
} catch (LensException exc) {
log.error("Error dropping partitions on storage of dimension table {}:{}", dimTableName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1530,13 +1480,14 @@ public class MetastoreResource {
checkSessionId(sessionid);
checkNonNullArgs("Partition is null", partition);
try {
- getSvc().addPartitionToDimStorage(sessionid, dimTableName, storage, partition);
+ return successOrPartialOrFailure(getSvc().addPartitionToDimStorage(sessionid, dimTableName, storage, partition),
+ 1);
} catch (LensException exc) {
log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
}
+
/**
* Updates an existing partition for a storage of dimension
*
@@ -1560,9 +1511,9 @@ public class MetastoreResource {
getSvc().updatePartition(sessionid, dimTableName, storage, partition);
} catch (LensException exc) {
log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
@@ -1584,13 +1535,14 @@ public class MetastoreResource {
checkSessionId(sessionid);
checkNonNullArgs("Partition list is null", partitions);
try {
- getSvc().addPartitionsToDimStorage(sessionid, dimTableName, storage, partitions);
+ return successOrPartialOrFailure(getSvc().addPartitionsToDimStorage(sessionid, dimTableName, storage, partitions),
+ partitions.getPartition().size());
} catch (LensException exc) {
log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
}
+
/**
* Add new partitions for a storage of dimension
*
@@ -1614,9 +1566,9 @@ public class MetastoreResource {
getSvc().updatePartitions(sessionid, dimTableName, storage, partitions);
} catch (LensException exc) {
log.error("Error adding partition to storage of dimension table {}:{}", dimTableName, storage, exc);
- return new APIResult(Status.FAILED, exc.getMessage());
+ return failure(exc);
}
- return SUCCESS;
+ return success();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/11593f46/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index 22dfd55..5edbd13 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -1999,6 +1999,12 @@ public class TestMetastoreService extends LensJerseyTest {
.post(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
assertEquals(partAddResult.getStatus(), Status.SUCCEEDED);
+ // add same should fail
+ partAddResult = target().path("metastore/facts/").path(table).path("storages/S2/partition")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .post(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
+ assertEquals(partAddResult.getStatus(), Status.FAILED);
+
xp.setLocation(xp.getLocation() + "/a/b/c");
APIResult partUpdateResult = target().path("metastore/facts/").path(table).path("storages/S2/partition")
.queryParam("sessionid", lensSessionId).request(mediaType)
@@ -2031,6 +2037,14 @@ public class TestMetastoreService extends LensJerseyTest {
partDate.setSeconds(0);
partDate.setTime(partDate.getTime() - partDate.getTime() % 1000);
assertEquals(date.getDate(), partDate);
+ // add two partitions, one of them already added. result should be partial
+ XPartitionList parts = new XPartitionList();
+ parts.getPartition().add(xp);
+ parts.getPartition().add(createPartition(table, DateUtils.addHours(partDate, 1)));
+ partAddResult = target().path("metastore/facts/").path(table).path("storages/S2/partitions")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .post(Entity.xml(cubeObjectFactory.createXPartitionList(parts)), APIResult.class);
+ assertEquals(partAddResult.getStatus(), Status.PARTIAL);
// Drop the partitions
APIResult dropResult = target().path("metastore/facts").path(table).path("storages/S2/partitions")
@@ -2118,6 +2132,13 @@ public class TestMetastoreService extends LensJerseyTest {
.post(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
assertEquals(partAddResult.getStatus(), Status.SUCCEEDED);
+ // create call for same
+ partAddResult = target().path("metastore/dimtables/").path(table).path("storages/test/partition")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .post(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
+ assertEquals(partAddResult.getStatus(), Status.FAILED);
+
+
xp.setLocation(xp.getLocation() + "/a/b/c");
APIResult partUpdateResult = target().path("metastore/dimtables/").path(table).path("storages/test/partition")
.queryParam("sessionid", lensSessionId).request(mediaType)
@@ -2163,6 +2184,15 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(latestPartition.getFullPartitionSpec().getPartSpecElement().get(0).getValue(),
"latest");
+ // create should give partial now:
+ XPartition xp2 = createPartition(table, DateUtils.addHours(partDate, 1));
+ XPartitionList parts = new XPartitionList();
+ parts.getPartition().add(xp);
+ parts.getPartition().add(xp2);
+ partAddResult = target().path("metastore/dimtables/").path(table).path("storages/test/partitions")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .post(Entity.xml(cubeObjectFactory.createXPartitionList(parts)), APIResult.class);
+ assertEquals(partAddResult.getStatus(), Status.PARTIAL);
// Drop the partitions
APIResult dropResult = target().path("metastore/dimtables").path(table).path("storages/test/partitions")
[07/50] [abbrv] incubator-lens git commit: LENS-648 : Fix NPE in
CandidateTableResolver
Posted by sh...@apache.org.
LENS-648 : Fix NPE in CandidateTableResolver
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/c8d38f7c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/c8d38f7c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/c8d38f7c
Branch: refs/heads/LENS-581
Commit: c8d38f7cfdb630558339377e8b427c7fbee8b3cf
Parents: 2e5748a
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Jul 10 11:20:41 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri Jul 10 11:20:41 2015 +0530
----------------------------------------------------------------------
.../lens/cube/parse/CandidateTableResolver.java | 79 +++++++++++---------
.../apache/lens/cube/parse/CubeTestSetup.java | 16 +++-
.../cube/parse/TestDenormalizationResolver.java | 14 ++++
lens-cube/src/test/resources/log4j.properties | 2 +-
4 files changed, 73 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c8d38f7c/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
index 1b5c09a..f861ec7 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CandidateTableResolver.java
@@ -544,23 +544,28 @@ class CandidateTableResolver implements ContextRewriter {
Set<Dimension> dimSet = dimColEntry.getValue();
for (Dimension dim : dimSet) {
OptionalDimCtx optdim = cubeql.getOptionalDimensionMap().get(dim);
- candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates);
+ if (optdim != null) {
+ candidatesReachableThroughRefs.addAll(optdim.requiredForCandidates);
+ }
}
for (Dimension dim : dimSet) {
- for (CandidateTable candidate : removedCandidates.get(dim)) {
- if (!candidatesReachableThroughRefs.contains(candidate)) {
- if (candidate instanceof CandidateFact) {
- if (cubeql.getCandidateFacts().contains(candidate)) {
- LOG.info("Not considering fact:" + candidate + " as its required optional dims are not reachable");
- cubeql.getCandidateFacts().remove(candidate);
- cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
+ if (removedCandidates.get(dim) != null) {
+ for (CandidateTable candidate : removedCandidates.get(dim)) {
+ if (!candidatesReachableThroughRefs.contains(candidate)) {
+ if (candidate instanceof CandidateFact) {
+ if (cubeql.getCandidateFacts().contains(candidate)) {
+ LOG.info("Not considering fact:" + candidate + " as its required optional dims are not reachable");
+ cubeql.getCandidateFacts().remove(candidate);
+ cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
+ CandidateTablePruneCause.columnNotFound(col));
+ }
+ } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
+ LOG.info("Not considering dimtable:" + candidate + " as its required optional dims are not reachable");
+ cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
+ cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(),
+ (CubeDimensionTable) candidate.getTable(),
CandidateTablePruneCause.columnNotFound(col));
}
- } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
- LOG.info("Not considering dimtable:" + candidate + " as its required optional dims are not reachable");
- cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
- cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
- CandidateTablePruneCause.columnNotFound(col));
}
}
}
@@ -583,32 +588,34 @@ class CandidateTableResolver implements ContextRewriter {
Set<Dimension> dimSet = exprColEntry.getValue();
ExpressionContext ec = cubeql.getExprCtx().getExpressionContext(col.getExprCol(), col.getAlias());
for (Dimension dim : dimSet) {
- for (CandidateTable candidate : removedCandidates.get(dim)) {
- // check if evaluable expressions of this candidate are no more evaluable because dimension is not reachable
- // if no evaluable expressions exist, then remove the candidate
- Iterator<ExprSpecContext> escIter = ec.getEvaluableExpressions().get(candidate).iterator();
- while (escIter.hasNext()) {
- ExprSpecContext esc = escIter.next();
- if (esc.getExprDims().contains(dim)) {
- escIter.remove();
+ if (removedCandidates.get(dim) != null) {
+ for (CandidateTable candidate : removedCandidates.get(dim)) {
+ // check if evaluable expressions of this candidate are no more evaluable because dimension is not reachable
+ // if no evaluable expressions exist, then remove the candidate
+ Iterator<ExprSpecContext> escIter = ec.getEvaluableExpressions().get(candidate).iterator();
+ while (escIter.hasNext()) {
+ ExprSpecContext esc = escIter.next();
+ if (esc.getExprDims().contains(dim)) {
+ escIter.remove();
+ }
}
- }
- if (cubeql.getExprCtx().isEvaluable(col.getExprCol(), candidate)) {
- // candidate has other evaluable expressions
- continue;
- }
- if (candidate instanceof CandidateFact) {
- if (cubeql.getCandidateFacts().contains(candidate)) {
- LOG.info("Not considering fact:" + candidate + " as is not reachable through any optional dim");
- cubeql.getCandidateFacts().remove(candidate);
- cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
+ if (cubeql.getExprCtx().isEvaluable(col.getExprCol(), candidate)) {
+ // candidate has other evaluable expressions
+ continue;
+ }
+ if (candidate instanceof CandidateFact) {
+ if (cubeql.getCandidateFacts().contains(candidate)) {
+ LOG.info("Not considering fact:" + candidate + " as is not reachable through any optional dim");
+ cubeql.getCandidateFacts().remove(candidate);
+ cubeql.addFactPruningMsgs(((CandidateFact) candidate).fact,
+ CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol()));
+ }
+ } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
+ LOG.info("Not considering dimtable:" + candidate + " as is not reachable through any optional dim");
+ cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
+ cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol()));
}
- } else if (cubeql.getCandidateDimTables().containsKey(((CandidateDim) candidate).getBaseTable())) {
- LOG.info("Not considering dimtable:" + candidate + " as is not reachable through any optional dim");
- cubeql.getCandidateDimTables().get(((CandidateDim) candidate).getBaseTable()).remove(candidate);
- cubeql.addDimPruningMsgs((Dimension) candidate.getBaseTable(), (CubeDimensionTable) candidate.getTable(),
- CandidateTablePruneCause.expressionNotEvaluable(col.getExprCol()));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c8d38f7c/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index 432c5f4..f939858 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -745,6 +745,8 @@ public class CubeTestSetup {
cubeDimensions2.add(new BaseDimAttribute(new FieldSchema("dim11", "string", "basedim")));
cubeDimensions2.add(new ReferencedDimAtrribute(new FieldSchema("dim12", "int", "ref dim"), "Dim2 refer",
new TableReference("testdim2", "id")));
+ cubeDimensions2.add(new ReferencedDimAtrribute(new FieldSchema("dim22", "int", "ref dim"), "Dim2 refer",
+ "dim2chain", "id", null, null, null));
Map<String, String> cubeProperties = new HashMap<String, String>();
cubeProperties.put(MetastoreUtil.getCubeTimedDimensionListKey(BASE_CUBE_NAME),
@@ -829,6 +831,16 @@ public class CubeTestSetup {
});
}
});
+ add(new JoinChain("dim2chain", "dim2chain", "dim2chain") {
+ {
+ addPath(new ArrayList<TableReference>() {
+ {
+ add(new TableReference("basecube", "dim2"));
+ add(new TableReference("testdim2", "id"));
+ }
+ });
+ }
+ });
}
};
@@ -861,6 +873,7 @@ public class CubeTestSetup {
dimensions.add("dim2");
dimensions.add("dim11");
dimensions.add("dim12");
+ dimensions.add("dim22");
dimensions.add("d_time");
dimensions.add("test_time_dim");
client.createDerivedCube(BASE_CUBE_NAME, DERIVED_CUBE_NAME2, measures, dimensions, derivedProperties, 10L);
@@ -941,7 +954,6 @@ public class CubeTestSetup {
// create fact only with extra measures
factName = "testFact2_BASE";
factColumns = new ArrayList<FieldSchema>();
- factColumns.add(new FieldSchema("msr11", "int", "first measure"));
factColumns.add(new FieldSchema("msr12", "float", "second measure"));
// add dimensions of the cube
@@ -988,6 +1000,8 @@ public class CubeTestSetup {
factColumns.add(new FieldSchema("dim1", "string", "base dim"));
factColumns.add(new FieldSchema("dim11", "string", "base dim"));
factColumns.add(new FieldSchema("dim12", "string", "base dim"));
+ factColumns.add(new FieldSchema("dim22", "string", "base dim"));
+ factColumns.add(new FieldSchema("cityid", "int", "city id"));
storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
updates = new HashSet<UpdatePeriod>();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c8d38f7c/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index 8c3c219..901b95e 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -242,6 +242,20 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
}
@Test
+ public void testCubeQueryWithOptionalDimsRemoved() throws Exception {
+ String hqlQuery = rewrite("select cityzip.code, dim22, msr11 from basecube where " + TWO_DAYS_RANGE,
+ conf);
+ String joinExpr = " join " + getDbName()
+ + "c1_citytable citydim on basecube.cityid = citydim.id and (citydim.dt = 'latest') "
+ + " join " + getDbName() + "c1_ziptable cityzip on citydim.zipcode = cityzip.code and (cityzip.dt = 'latest')";
+ String expected =
+ getExpectedQuery("basecube", "select cityzip.code, basecube.dim22, basecube.msr11 FROM ",
+ joinExpr, null, null, null,
+ getWhereForHourly2days("basecube", "C1_testfact2_raw_base"));
+ TestCubeRewriter.compareQueries(hqlQuery, expected);
+ }
+
+ @Test
public void testDimensionQueryWithTwoRefCols() throws Exception {
Configuration tConf = new Configuration(conf);
tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c8d38f7c/lens-cube/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/resources/log4j.properties b/lens-cube/src/test/resources/log4j.properties
index a87fd8f..deaf1e1 100644
--- a/lens-cube/src/test/resources/log4j.properties
+++ b/lens-cube/src/test/resources/log4j.properties
@@ -32,5 +32,5 @@ log4j.appender.STDOUT.Threshold=WARN
log4j.appender.TEST_LOG_FILE=org.apache.log4j.RollingFileAppender
log4j.appender.TEST_LOG_FILE.File=target/test.log
log4j.appender.TEST_LOG_FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%t] %-5p %c %L %x - %m%n
log4j.appender.TEST_LOG_FILE.Threshold=INFO
[34/50] [abbrv] incubator-lens git commit: LENS-665 : Use Slf4j
logger in lens-cube module
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
index 9d1d9f2..4ad2f1f 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/SingleFactMultiStorageHQLContext.java
@@ -24,8 +24,6 @@ import java.util.Map;
import org.apache.lens.cube.metadata.Dimension;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import lombok.Getter;
@@ -33,8 +31,6 @@ import lombok.Getter;
public class SingleFactMultiStorageHQLContext extends UnionHQLContext {
- public static final Log LOG = LogFactory.getLog(SingleFactMultiStorageHQLContext.class.getName());
-
@Getter
private CubeQueryContext query = null;
private CandidateFact fact = null;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 373c80a..7bfbd86 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -40,8 +40,6 @@ import org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -50,12 +48,14 @@ import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Resolve storages and partitions of all candidate tables and prunes candidate tables with missing storages or
* partitions.
*/
+@Slf4j
class StorageTableResolver implements ContextRewriter {
- public static final Log LOG = LogFactory.getLog(StorageTableResolver.class.getName());
private final Configuration conf;
private final List<String> supportedStorages;
@@ -185,7 +185,7 @@ class StorageTableResolver implements ContextRewriter {
if (isStorageSupported(storage)) {
String tableName = getFactOrDimtableStorageTableName(dimtable.getName(), storage).toLowerCase();
if (validDimTables != null && !validDimTables.contains(tableName)) {
- LOG.info("Not considering dim storage table:" + tableName + " as it is not a valid dim storage");
+ log.info("Not considering dim storage table:{} as it is not a valid dim storage", tableName);
skipStorageCauses.put(tableName, new SkipStorageCause(SkipStorageCode.INVALID));
continue;
}
@@ -194,9 +194,9 @@ class StorageTableResolver implements ContextRewriter {
// check if partition exists
foundPart = client.dimTableLatestPartitionExists(tableName);
if (foundPart) {
- LOG.info("Adding existing partition" + StorageConstants.LATEST_PARTITION_VALUE);
+ log.info("Adding existing partition {}", StorageConstants.LATEST_PARTITION_VALUE);
} else {
- LOG.info("Partition " + StorageConstants.LATEST_PARTITION_VALUE + " does not exist on " + tableName);
+ log.info("Partition {} does not exist on {}", StorageConstants.LATEST_PARTITION_VALUE, tableName);
}
if (!failOnPartialData || foundPart) {
storageTables.add(tableName);
@@ -205,7 +205,7 @@ class StorageTableResolver implements ContextRewriter {
StorageConstants.getPartitionsForLatest());
whereClauses.put(tableName, whereClause);
} else {
- LOG.info("Not considering dim storage table:" + tableName + " as no dim partitions exist");
+ log.info("Not considering dim storage table:{} as no dim partitions exist", tableName);
skipStorageCauses.put(tableName, new SkipStorageCause(SkipStorageCode.NO_PARTITIONS));
}
} else {
@@ -213,7 +213,7 @@ class StorageTableResolver implements ContextRewriter {
foundPart = true;
}
} else {
- LOG.info("Storage:" + storage + " is not supported");
+ log.info("Storage:{} is not supported", storage);
skipStorageCauses.put(storage, new SkipStorageCause(SkipStorageCode.UNSUPPORTED));
}
}
@@ -221,7 +221,7 @@ class StorageTableResolver implements ContextRewriter {
addNonExistingParts(dim.getName(), StorageConstants.getPartitionsForLatest());
}
if (storageTables.isEmpty()) {
- LOG.info("Not considering dim table:" + dimtable + " as no candidate storage tables eixst");
+ log.info("Not considering dim table:{} as no candidate storage tables eixst", dimtable);
cubeql.addDimPruningMsgs(dim, dimtable, noCandidateStorages(skipStorageCauses));
i.remove();
continue;
@@ -254,7 +254,7 @@ class StorageTableResolver implements ContextRewriter {
String storage = entry.getKey();
// skip storages that are not supported
if (!isStorageSupported(storage)) {
- LOG.info("Skipping storage: " + storage + " as it is not supported");
+ log.info("Skipping storage: {} as it is not supported", storage);
skipStorageCauses.put(storage, new SkipStorageCause(SkipStorageCode.UNSUPPORTED));
continue;
}
@@ -271,12 +271,12 @@ class StorageTableResolver implements ContextRewriter {
Map<String, SkipUpdatePeriodCode> skipUpdatePeriodCauses = new HashMap<String, SkipUpdatePeriodCode>();
for (UpdatePeriod updatePeriod : entry.getValue()) {
if (maxInterval != null && updatePeriod.compareTo(maxInterval) > 0) {
- LOG.info("Skipping update period " + updatePeriod + " for fact" + fact);
+ log.info("Skipping update period {} for fact {}", updatePeriod, fact);
skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.QUERY_INTERVAL_BIGGER);
continue;
}
if (validUpdatePeriods != null && !validUpdatePeriods.contains(updatePeriod.name().toLowerCase())) {
- LOG.info("Skipping update period " + updatePeriod + " for fact" + fact + " for storage" + storage);
+ log.info("Skipping update period {} for fact {} for storage {}", updatePeriod, fact, storage);
skipUpdatePeriodCauses.put(updatePeriod.toString(), SkipUpdatePeriodCode.INVALID);
continue;
}
@@ -286,7 +286,7 @@ class StorageTableResolver implements ContextRewriter {
storageTableMap.put(updatePeriod, storageTables);
}
isStorageAdded = true;
- LOG.info("Adding storage table:" + table + " for fact:" + fact + " for update period" + updatePeriod);
+ log.info("Adding storage table:{} for fact:{} for update period {}", table, fact, updatePeriod);
storageTables.add(table);
}
if (!isStorageAdded) {
@@ -294,7 +294,7 @@ class StorageTableResolver implements ContextRewriter {
}
}
if (storageTableMap.isEmpty()) {
- LOG.info("Not considering fact table:" + fact + " as it does not" + " have any storage tables");
+ log.info("Not considering fact table:{} as it does not have any storage tables", fact);
cubeql.addFactPruningMsgs(fact, noCandidateStorages(skipStorageCauses));
i.remove();
}
@@ -310,7 +310,7 @@ class StorageTableResolver implements ContextRewriter {
String getStorageTableName(CubeFactTable fact, String storage, List<String> validFactStorageTables) {
String tableName = getFactOrDimtableStorageTableName(fact.getName(), storage).toLowerCase();
if (validFactStorageTables != null && !validFactStorageTables.contains(tableName)) {
- LOG.info("Skipping storage table " + tableName + " as it is not valid");
+ log.info("Skipping storage table {} as it is not valid", tableName);
return null;
}
return tableName;
@@ -386,7 +386,7 @@ class StorageTableResolver implements ContextRewriter {
break;
}
TimeRange fallBackRange = getFallbackRange(prevRange, cfact, cubeql);
- LOG.info("No partitions for range:" + range + ". fallback range: " + fallBackRange);
+ log.info("No partitions for range:{}. fallback range: {}", range, fallBackRange);
if (fallBackRange == null) {
break;
}
@@ -402,7 +402,7 @@ class StorageTableResolver implements ContextRewriter {
}
whereClauseForFallback.put(range, extraWhereClause.toString());
if (rangeParts.isEmpty()) {
- LOG.info("No partitions for fallback range:" + range);
+ log.info("No partitions for fallback range:{}", range);
noPartsForRange = true;
continue;
}
@@ -428,8 +428,8 @@ class StorageTableResolver implements ContextRewriter {
cfact.getRangeToWhereClause().put(range, joinWithAnd(rangeWhereClause, extraWhereClause.toString()));
}
if (!unsupportedTimeDims.isEmpty()) {
- LOG.info("Not considering fact table:" + cfact.fact + " as it doesn't support time dimensions: "
- + unsupportedTimeDims);
+ log.info("Not considering fact table:{} as it doesn't support time dimensions: {}", cfact.fact,
+ unsupportedTimeDims);
cubeql.addFactPruningMsgs(cfact.fact, timeDimNotSupported(unsupportedTimeDims));
i.remove();
continue;
@@ -439,8 +439,8 @@ class StorageTableResolver implements ContextRewriter {
addNonExistingParts(cfact.fact.getName(), nonExistingParts);
}
if (cfact.getNumQueriedParts() == 0 || (failOnPartialData && (noPartsForRange || !nonExistingParts.isEmpty()))) {
- LOG.info("Not considering fact table:" + cfact.fact + " as it could" + " not find partition for given ranges: "
- + cubeql.getTimeRanges());
+ log.info("Not considering fact table:{} as it could not find partition for given ranges: {}", cfact.fact,
+ cubeql.getTimeRanges());
/*
* This fact is getting discarded because of any of following reasons:
* 1. Has missing partitions
@@ -464,7 +464,7 @@ class StorageTableResolver implements ContextRewriter {
Map<String, Set<FactPartition>> minimalStorageTables = new LinkedHashMap<String, Set<FactPartition>>();
StorageUtil.getMinimalAnsweringTables(answeringParts, minimalStorageTables);
if (minimalStorageTables.isEmpty()) {
- LOG.info("Not considering fact table:" + cfact + " as it does not" + " have any storage tables");
+ log.info("Not considering fact table:{} as it does not have any storage tables", cfact);
cubeql.addFactPruningMsgs(cfact.fact, noCandidateStorages(skipStorageCauses));
i.remove();
continue;
@@ -494,7 +494,7 @@ class StorageTableResolver implements ContextRewriter {
}
cfact.getRangeToStorageWhereMap().put(trange, rangeToWhere);
}
- LOG.info("Resolved partitions for fact " + cfact + ": " + answeringParts + " storageTables:" + storageTables);
+ log.info("Resolved partitions for fact {}: {} storageTables:{}", cfact, answeringParts, storageTables);
}
}
@@ -533,16 +533,16 @@ class StorageTableResolver implements ContextRewriter {
boolean addNonExistingParts, Map<String, SkipStorageCause> skipStorageCauses,
PartitionRangesForPartitionColumns missingPartitions)
throws Exception {
- LOG.info("getPartitions for " + fact + " from fromDate:" + fromDate + " toDate:" + toDate);
+ log.info("getPartitions for {} from fromDate:{} toDate:{}", fact, fromDate, toDate);
if (fromDate.equals(toDate) || fromDate.after(toDate)) {
return true;
}
UpdatePeriod interval = CubeFactTable.maxIntervalInRange(fromDate, toDate, updatePeriods);
if (interval == null) {
- LOG.info("No max interval for range:" + fromDate + " to " + toDate);
+ log.info("No max interval for range: {} to {}", fromDate, toDate);
return false;
}
- LOG.info("Max interval for " + fact + " is:" + interval);
+ log.info("Max interval for {} is: {}", fact, interval);
Set<String> storageTbls = new LinkedHashSet<String>();
storageTbls.addAll(validStorageMap.get(fact).get(interval));
@@ -550,7 +550,7 @@ class StorageTableResolver implements ContextRewriter {
while (it.hasNext()) {
String storageTableName = it.next();
if (!client.partColExists(storageTableName, partCol)) {
- LOG.info(partCol + " does not exist in" + storageTableName);
+ log.info("{} does not exist in {}", partCol, storageTableName);
skipStorageCauses.put(storageTableName, SkipStorageCause.partColDoesNotExist(partCol));
it.remove();
continue;
@@ -573,24 +573,24 @@ class StorageTableResolver implements ContextRewriter {
Date dt = iter.next();
Date nextDt = iter.peekNext();
FactPartition part = new FactPartition(partCol, dt, interval, null, partWhereClauseFormat);
- LOG.info("candidate storage tables for searching partitions: " + storageTbls);
+ log.info("candidate storage tables for searching partitions: {}", storageTbls);
updateFactPartitionStorageTablesFrom(fact, part, storageTbls);
- LOG.info("Storage tables containing Partition " + part + " are: " + part.getStorageTables());
+ log.info("Storage tables containing Partition {} are: {}", part, part.getStorageTables());
if (part.isFound()) {
- LOG.info("Adding existing partition" + part);
+ log.info("Adding existing partition {}", part);
partitions.add(part);
- LOG.info("Looking for look ahead process time partitions for " + part);
+ log.info("Looking for look ahead process time partitions for {}", part);
if (processTimePartCol == null) {
- LOG.info("processTimePartCol is null");
+ log.info("processTimePartCol is null");
} else if (partCol.equals(processTimePartCol)) {
- LOG.info("part column is process time col");
+ log.info("part column is process time col");
} else if (updatePeriods.first().equals(interval)) {
- LOG.info("Update period is the least update period");
+ log.info("Update period is the least update period");
} else if ((iter.getNumIters() - iter.getCounter()) > lookAheadNumParts) {
// see if this is the part of the last-n look ahead partitions
- LOG.info("Not a look ahead partition");
+ log.info("Not a look ahead partition");
} else {
- LOG.info("Looking for look ahead process time partitions for " + part);
+ log.info("Looking for look ahead process time partitions for {}", part);
// check if finer partitions are required
// final partitions are required if no partitions from
// look-ahead
@@ -605,32 +605,32 @@ class StorageTableResolver implements ContextRewriter {
updateFactPartitionStorageTablesFrom(fact, processTimePartition,
part.getStorageTables());
if (processTimePartition.isFound()) {
- LOG.info("Finer parts not required for look-ahead partition :" + part);
+ log.info("Finer parts not required for look-ahead partition :{}", part);
} else {
- LOG.info("Looked ahead process time partition " + processTimePartition + " is not found");
+ log.info("Looked ahead process time partition {} is not found", processTimePartition);
TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>();
newset.addAll(updatePeriods);
newset.remove(interval);
- LOG.info("newset of update periods:" + newset);
+ log.info("newset of update periods:{}", newset);
if (!newset.isEmpty()) {
// Get partitions for look ahead process time
- LOG.info("Looking for process time partitions between " + pdt + " and " + nextPdt);
+ log.info("Looking for process time partitions between {} and {}", pdt, nextPdt);
Set<FactPartition> processTimeParts =
getPartitions(fact, TimeRange.getBuilder().fromDate(pdt).toDate(nextPdt).partitionColumn(
processTimePartCol).build(), newset, false, skipStorageCauses, missingPartitions);
- LOG.info("Look ahead partitions: " + processTimeParts);
+ log.info("Look ahead partitions: {}", processTimeParts);
TimeRange timeRange = TimeRange.getBuilder().fromDate(dt).toDate(nextDt).build();
for (FactPartition pPart : processTimeParts) {
- LOG.info("Looking for finer partitions in pPart: " + pPart);
+ log.info("Looking for finer partitions in pPart: {}", pPart);
for (Date date : timeRange.iterable(pPart.getPeriod(), 1)) {
partitions.add(new FactPartition(partCol, date, pPart.getPeriod(), pPart,
partWhereClauseFormat));
}
- LOG.info("added all sub partitions blindly in pPart: " + pPart);
+ log.info("added all sub partitions blindly in pPart: {}", pPart);
// if (!getPartitions(fact, dt, cal.getTime(), partCol, pPart, partitions,
// newset, false,
// skipStorageCauses, nonExistingParts)) {
- // LOG.info("No partitions found in look ahead range");
+ // log.info("No partitions found in look ahead range");
// }
}
}
@@ -638,14 +638,14 @@ class StorageTableResolver implements ContextRewriter {
}
}
} else {
- LOG.info("Partition:" + part + " does not exist in any storage table");
+ log.info("Partition:{} does not exist in any storage table", part);
TreeSet<UpdatePeriod> newset = new TreeSet<UpdatePeriod>();
newset.addAll(updatePeriods);
newset.remove(interval);
if (!getPartitions(fact, dt, nextDt, partCol, partitions, newset, false, skipStorageCauses,
missingPartitions)) {
- LOG.info("Adding non existing partition" + part);
+ log.info("Adding non existing partition {}", part);
if (addNonExistingParts) {
// Add non existing partitions for all cases of whether we populate all non existing or not.
missingPartitions.add(part);
@@ -655,11 +655,11 @@ class StorageTableResolver implements ContextRewriter {
part.getStorageTables().addAll(storageTbls);
}
} else {
- LOG.info("No finer granual partitions exist for" + part);
+ log.info("No finer granual partitions exist for {}", part);
return false;
}
} else {
- LOG.info("Finer granual partitions added for " + part);
+ log.info("Finer granual partitions added for {}", part);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/cb28e069/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
index 04a13a6..67b3f40 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageUtil.java
@@ -23,16 +23,11 @@ import java.util.*;
import org.apache.lens.cube.metadata.FactPartition;
import org.apache.lens.cube.metadata.StorageConstants;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
public final class StorageUtil {
private StorageUtil() {
}
- private static final Log LOG = LogFactory.getLog(StorageUtil.class.getName());
-
public static String getWherePartClause(String timeDimName, String tableName, Collection<String> parts) {
if (parts.size() == 0) {
return "";
[19/50] [abbrv] incubator-lens git commit: LENS-673 : Fix patch apply
command on committer's page in website
Posted by sh...@apache.org.
LENS-673 : Fix patch apply command on committer's page in website
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/9492add3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/9492add3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/9492add3
Branch: refs/heads/LENS-581
Commit: 9492add320edf54c0f6c5b4c16431d15644ac09b
Parents: dfeea69
Author: Yash Sharma <ya...@gmail.com>
Authored: Mon Jul 20 17:42:33 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Jul 20 17:42:33 2015 +0530
----------------------------------------------------------------------
src/site/apt/developer/commit.apt | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9492add3/src/site/apt/developer/commit.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/developer/commit.apt b/src/site/apt/developer/commit.apt
index 04abc62..6982ead 100644
--- a/src/site/apt/developer/commit.apt
+++ b/src/site/apt/developer/commit.apt
@@ -69,11 +69,12 @@ Committer Guide
* Ensure that 24 hours have elapsed since the jira is made patch available or the review request is up. As a practice
we should observe this, but it should be possible to consciously override and commit with a shorter turnaround time.
- * Apply the patch attached on jira. The patch should licensed under apache license.
+ * Apply the patch attached on jira. The patch should be licensed under apache license.
+---+
- git apply -p0 <final-patch>.patch
+ git apply --check lens_patch.patch
+ git apply lens_patch.patch
+---+
[50/50] [abbrv] incubator-lens git commit: Merge branch 'master' into
LENS-581
Posted by sh...@apache.org.
Merge branch 'master' into LENS-581
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/6c805705
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/6c805705
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/6c805705
Branch: refs/heads/LENS-581
Commit: 6c805705078d147b840b21df879601550a6829ac
Parents: ace0dd2 11593f4
Author: Sharad Agarwal <sh...@flipkarts-mbp.fkipl.flipkart.com>
Authored: Wed Jul 29 10:27:52 2015 +0530
Committer: Sharad Agarwal <sh...@flipkarts-mbp.fkipl.flipkart.com>
Committed: Wed Jul 29 10:27:52 2015 +0530
----------------------------------------------------------------------
checkstyle/pom.xml | 3 +-
lens-api/pom.xml | 17 +-
.../java/org/apache/lens/api/APIResult.java | 48 +-
.../org/apache/lens/api/query/LensQuery.java | 4 +
.../org/apache/lens/api/query/QueryCost.java | 80 ---
.../apache/lens/api/query/QueryCostType.java | 26 +
.../api/query/QueryHandleWithResultSet.java | 7 +
.../org/apache/lens/api/query/QueryPlan.java | 4 +-
.../org/apache/lens/api/query/QueryStatus.java | 14 +-
.../lens/api/query/QuerySubmitResult.java | 4 +-
.../query/SupportedQuerySubmitOperations.java | 3 +-
.../org/apache/lens/api/result/QueryCostTO.java | 75 +++
lens-api/src/main/resources/cube-0.1.xsd | 1 +
.../SupportedQuerySubmitOperationsTest.java | 37 ++
.../lens/api/serialize/SerializationTest.java | 38 ++
lens-cli/pom.xml | 3 +-
.../lens/cli/commands/BaseLensCommand.java | 2 +-
.../cli/commands/LensConnectionCommands.java | 33 +-
.../commands/LensDimensionTableCommands.java | 35 +-
.../lens/cli/commands/LensFactCommands.java | 42 ++
.../cli/commands/PhysicalTableCrudCommand.java | 15 +
.../apache/lens/cli/LensCliApplicationTest.java | 2 +
.../lens/cli/TestLensConnectionCliCommands.java | 21 +-
.../cli/TestLensDimensionTableCommands.java | 107 ++--
.../apache/lens/cli/TestLensFactCommands.java | 42 +-
.../apache/lens/cli/TestLensQueryCommands.java | 19 +-
.../src/test/resources/dim1-local-parts.xml | 5 +-
.../src/test/resources/fact1-local-parts.xml | 4 +-
lens-client/pom.xml | 3 +-
.../java/org/apache/lens/client/LensClient.java | 78 ++-
.../lens/client/LensClientSingletonWrapper.java | 9 +-
.../org/apache/lens/client/LensConnection.java | 16 +-
.../apache/lens/client/LensMetadataClient.java | 251 ++++----
.../org/apache/lens/client/LensStatement.java | 6 +-
lens-cube/pom.xml | 14 +-
.../cube/error/ColUnAvailableInTimeRange.java | 3 +-
.../lens/cube/error/ConflictingFields.java | 3 +-
.../lens/cube/metadata/AbstractBaseTable.java | 12 +-
.../lens/cube/metadata/AbstractCubeTable.java | 9 +-
.../lens/cube/metadata/BaseDimAttribute.java | 10 +-
.../org/apache/lens/cube/metadata/Cube.java | 15 +-
.../lens/cube/metadata/CubeDimensionTable.java | 5 +-
.../lens/cube/metadata/CubeMetastoreClient.java | 156 +++--
.../apache/lens/cube/metadata/Dimension.java | 7 +-
.../lens/cube/metadata/FactPartition.java | 16 +-
.../apache/lens/cube/metadata/HDFSStorage.java | 16 +
.../lens/cube/metadata/MetastoreUtil.java | 6 +-
.../lens/cube/metadata/PartitionMetahook.java | 29 +-
.../org/apache/lens/cube/metadata/Storage.java | 56 +-
.../lens/cube/metadata/TimePartition.java | 20 +-
.../apache/lens/cube/metadata/UpdatePeriod.java | 54 +-
.../metadata/timeline/PartitionTimeline.java | 17 +-
.../lens/cube/parse/AggregateResolver.java | 9 +-
.../apache/lens/cube/parse/CandidateFact.java | 13 +-
.../cube/parse/CandidateTablePruneCause.java | 9 +-
.../lens/cube/parse/CandidateTableResolver.java | 161 +++---
.../lens/cube/parse/CubeQueryContext.java | 38 +-
.../org/apache/lens/cube/parse/DateUtil.java | 10 +-
.../cube/parse/DenormalizationResolver.java | 14 +-
.../apache/lens/cube/parse/DimHQLContext.java | 4 -
.../lens/cube/parse/DimOnlyHQLContext.java | 4 -
.../lens/cube/parse/ExpressionResolver.java | 65 ++-
.../apache/lens/cube/parse/FactHQLContext.java | 9 +-
.../apache/lens/cube/parse/GroupbyResolver.java | 10 +-
.../apache/lens/cube/parse/JoinResolver.java | 54 +-
.../lens/cube/parse/LeastPartitionResolver.java | 11 +-
.../cube/parse/LightestDimensionResolver.java | 10 +-
.../lens/cube/parse/LightestFactResolver.java | 11 +-
.../cube/parse/MaxCoveringFactResolver.java | 13 +-
.../lens/cube/parse/MultiFactHQLContext.java | 4 -
.../lens/cube/parse/SimpleHQLContext.java | 11 +-
.../lens/cube/parse/SingleFactHQLContext.java | 4 -
.../parse/SingleFactMultiStorageHQLContext.java | 4 -
.../lens/cube/parse/StorageTableResolver.java | 116 ++--
.../org/apache/lens/cube/parse/StorageUtil.java | 5 -
.../FactPartitionBasedQueryCostCalculator.java | 98 ++++
.../apache/lens/driver/cube/RewriteUtil.java | 393 -------------
.../apache/lens/driver/cube/RewriterPlan.java | 2 +-
.../error/ColumnUnAvailableInTimeRangeTest.java | 37 ++
.../lens/cube/error/ConflictingFieldsTest.java | 43 ++
.../cube/metadata/TestCubeMetastoreClient.java | 538 ++++++++---------
.../lens/cube/metadata/TestFactPartition.java | 57 ++
.../apache/lens/cube/parse/CubeTestSetup.java | 77 ++-
.../lens/cube/parse/TestBaseCubeQueries.java | 171 ++++--
.../lens/cube/parse/TestCubeRewriter.java | 50 +-
.../cube/parse/TestDenormalizationResolver.java | 16 +
.../lens/cube/parse/TestTimeRangeResolver.java | 4 +-
.../parse/TestTimeRangeWriterWithQuery.java | 45 +-
...stFactPartitionBasedQueryCostCalculator.java | 96 ++++
.../apache/lens/driver/cube/TestRewriting.java | 464 ---------------
lens-cube/src/test/resources/log4j.properties | 2 +-
lens-dist/pom.xml | 3 +-
lens-docker/lens-test/lens-bootstrap.sh | 27 +-
lens-driver-hive/pom.xml | 3 +-
.../org/apache/lens/driver/hive/HiveDriver.java | 193 ++++---
.../apache/lens/driver/hive/HiveQueryPlan.java | 27 +-
.../driver/hive/RemoteThriftConnection.java | 19 +-
.../DurationBasedQueryPriorityDecider.java | 139 -----
.../src/main/resources/hivedriver-default.xml | 33 +-
.../apache/lens/driver/hive/TestHiveDriver.java | 162 +++---
.../lens/driver/hive/TestRemoteHiveDriver.java | 14 +-
lens-driver-jdbc/pom.xml | 36 +-
.../additional/java/ClassLoaderTestClass.java | 23 +
.../src/additional/java/DatabaseJarSerde.java | 36 ++
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 48 +-
.../jdbc/DataSourceConnectionProvider.java | 13 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 103 ++--
.../driver/jdbc/JDBCDriverConfConstants.java | 7 +
.../apache/lens/driver/jdbc/JDBCResultSet.java | 11 +-
.../src/main/resources/jdbcdriver-default.xml | 25 +
.../driver/jdbc/TestColumnarSQLRewriter.java | 243 ++++----
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 50 +-
.../testdata/ClassLoaderTestClass.java | 23 -
.../testdata/ClassLoaderTestClass2.java | 23 -
lens-driver-jdbc/testdata/DatabaseJarSerde.java | 36 --
lens-driver-jdbc/testdata/serde.jar | Bin 733 -> 0 bytes
lens-driver-jdbc/testdata/test.jar | Bin 726 -> 0 bytes
lens-driver-jdbc/testdata/test2.jar | Bin 700 -> 0 bytes
lens-examples/pom.xml | 3 +-
.../lens/examples/PopulateSampleMetastore.java | 37 +-
.../apache/lens/examples/SampleMetastore.java | 1 +
.../src/main/resources/cube-queries.sql | 1 +
.../src/main/resources/db-storage-schema.sql | 4 +
.../resources/sales-aggr-continuous-fact.xml | 57 ++
lens-ml-lib/pom.xml | 3 +-
.../apache/lens/client/LensMLJerseyClient.java | 10 +-
.../lens/ml/algo/spark/BaseSparkAlgo.java | 14 +-
.../ml/algo/spark/ColumnFeatureFunction.java | 10 +-
.../apache/lens/ml/algo/spark/HiveTableRDD.java | 4 -
.../lens/ml/algo/spark/SparkMLDriver.java | 28 +-
.../lens/ml/algo/spark/TableTrainingSpec.java | 20 +-
.../java/org/apache/lens/ml/impl/HiveMLUDF.java | 12 +-
.../java/org/apache/lens/ml/impl/MLRunner.java | 15 +-
.../java/org/apache/lens/ml/impl/MLTask.java | 21 +-
.../org/apache/lens/ml/impl/ModelLoader.java | 24 +-
.../apache/lens/ml/impl/TableTestingSpec.java | 17 +-
.../apache/lens/ml/server/MLServiceImpl.java | 16 +-
.../lens/ml/server/MLServiceResource.java | 14 +-
.../java/org/apache/lens/rdd/LensRDDClient.java | 19 +-
.../java/org/apache/lens/ml/ExampleUtils.java | 11 +-
.../java/org/apache/lens/ml/TestMLRunner.java | 10 +-
lens-ml-lib/testjars/serde.jar | Bin 1369 -> 0 bytes
lens-ml-lib/testjars/test.jar | Bin 697 -> 0 bytes
lens-query-lib/pom.xml | 3 +-
lens-regression/pom.xml | 7 +-
.../core/helpers/LensServerHelper.java | 10 +-
.../core/helpers/MetastoreHelper.java | 12 +-
.../regression/core/helpers/QueryHelper.java | 73 ++-
.../core/helpers/ServiceManagerHelper.java | 10 +-
.../regression/core/helpers/SessionHelper.java | 24 +-
.../apache/lens/regression/util/AssertUtil.java | 7 -
.../org/apache/lens/regression/util/Util.java | 20 +-
.../lens/regression/sanity/ITSmokeTest.java | 23 +-
lens-server-api/pom.xml | 14 +-
.../lens/server/api/LensConfConstants.java | 21 +-
.../org/apache/lens/server/api/LensService.java | 25 +
.../ConfigBasedObjectCreationFactory.java | 33 ++
.../lens/server/api/driver/DriverQueryHook.java | 50 ++
.../lens/server/api/driver/DriverQueryPlan.java | 12 +-
.../lens/server/api/driver/LensDriver.java | 28 +-
.../server/api/driver/MinQueryCostSelector.java | 31 +-
.../server/api/driver/NoOpDriverQueryHook.java | 35 ++
.../lens/server/api/error/LensException.java | 15 +-
.../server/api/events/LensEventService.java | 4 +-
.../lens/server/api/health/HealthCheckable.java | 29 +
.../lens/server/api/health/HealthStatus.java | 32 ++
.../api/metastore/CubeMetastoreService.java | 42 +-
.../api/metrics/MethodMetricsFactory.java | 9 +-
.../lens/server/api/metrics/MetricsService.java | 12 +-
.../api/priority/CostToPriorityRangeConf.java | 74 ---
.../api/priority/QueryPriorityDecider.java | 39 --
.../lens/server/api/priority/RangeConf.java | 111 ----
.../server/api/query/AbstractQueryContext.java | 79 +--
.../api/query/DriverSelectorQueryContext.java | 60 +-
.../server/api/query/FinishedLensQuery.java | 11 +
.../lens/server/api/query/QueryContext.java | 44 +-
.../server/api/query/QueryExecutionService.java | 8 +
.../lens/server/api/query/QueryRewriter.java | 50 --
...erSpecificWaitingQueriesSelectionPolicy.java | 42 ++
...ficWaitingQueriesSelectionPolicyFactory.java | 33 ++
.../EstimatedImmutableQueryCollection.java | 65 +++
.../query/collect/ImmutableQueryCollection.java | 53 ++
.../collect/WaitingQueriesSelectionPolicy.java | 39 ++
.../MaxConcurrentDriverQueriesConstraint.java | 47 ++
...oncurrentDriverQueriesConstraintFactory.java | 39 ++
.../constraint/QueryLaunchingConstraint.java | 34 ++
.../query/cost/FactPartitionBasedQueryCost.java | 67 +++
.../lens/server/api/query/cost/QueryCost.java | 40 ++
.../api/query/cost/QueryCostCalculator.java | 32 ++
.../api/query/cost/QueryCostTOBuilder.java | 50 ++
.../priority/CostRangePriorityDecider.java | 42 ++
.../query/priority/CostToPriorityRangeConf.java | 74 +++
.../query/priority/QueryPriorityDecider.java | 33 ++
.../server/api/query/priority/RangeConf.java | 111 ++++
.../api/query/rewrite/Phase1Rewriter.java | 29 +
.../server/api/query/rewrite/QueryRewriter.java | 50 ++
.../lens/server/api/user/UserConfigLoader.java | 37 +-
.../apache/lens/server/api/util/LensUtil.java | 36 ++
.../server/model/LogSegregationContext.java | 27 +-
.../MappedDiagnosticLogSegregationContext.java | 23 +-
.../lens/server/api/driver/MockDriver.java | 22 +-
.../lens/server/api/driver/MockFailDriver.java | 2 +-
.../lens/server/api/query/MockQueryContext.java | 5 +
...ecificWaitingQueriesSelectionPolicyTest.java | 51 ++
...axConcurrentDriverQueriesConstraintTest.java | 57 ++
.../cost/TestFactPartitionBasedQueryCost.java | 83 +++
.../server/api/user/MockDriverQueryHook.java | 32 ++
.../server/api/user/MockUserConfigLoader.java | 47 --
...stMappedDiagnosticLogSegregationContext.java | 60 ++
lens-server/pom.xml | 53 +-
.../additional/java/ClassLoaderTestClass.java | 23 +
.../additional/java/ClassLoaderTestClass2.java | 23 +
.../src/additional/java/DatabaseJarSerde.java | 36 ++
.../lens/server/AuthenticationFilter.java | 9 +-
.../org/apache/lens/server/BaseLensService.java | 433 ++++++++++++++
.../apache/lens/server/EventServiceImpl.java | 32 +-
.../org/apache/lens/server/LensApplication.java | 18 +-
.../lens/server/LensApplicationListener.java | 12 +-
.../server/LensRequestContextInitFilter.java | 2 +-
.../java/org/apache/lens/server/LensServer.java | 26 +-
.../org/apache/lens/server/LensService.java | 426 --------------
.../org/apache/lens/server/LensServices.java | 69 ++-
.../org/apache/lens/server/LogResource.java | 87 +++
.../healthcheck/LensServiceHealthCheck.java | 45 ++
.../metastore/CubeMetastoreServiceImpl.java | 168 ++++--
.../apache/lens/server/metastore/JAXBUtils.java | 77 ++-
.../server/metastore/MetastoreResource.java | 444 ++++++++-------
.../lens/server/metrics/MetricsServiceImpl.java | 62 +-
.../lens/server/query/QueryEndNotifier.java | 17 +-
.../server/query/QueryExecutionServiceImpl.java | 570 ++++++++++++++-----
.../lens/server/query/QueryServiceResource.java | 7 +-
.../lens/server/query/ResultFormatter.java | 31 +-
.../DefaultEstimatedQueryCollection.java | 169 ++++++
.../query/collect/DefaultQueryCollection.java | 104 ++++
.../query/collect/EstimatedQueryCollection.java | 30 +
.../IntersectingWaitingQueriesSelector.java | 107 ++++
.../query/collect/MutableQueryCollection.java | 65 +++
.../server/query/collect/QueryCollection.java | 31 +
.../ThreadSafeEstimatedQueryCollection.java | 91 +++
.../collect/ThreadSafeQueryCollection.java | 75 +++
...erSpecificWaitingQueriesSelectionPolicy.java | 44 ++
...ficWaitingQueriesSelectionPolicyFactory.java | 34 ++
.../query/collect/WaitingQueriesSelector.java | 43 ++
...DefaultQueryLaunchingConstraintsChecker.java | 68 +++
.../QueryLaunchingConstraintsChecker.java | 33 ++
.../TotalQueryCostCeilingConstraint.java | 74 +++
.../TotalQueryCostCeilingConstraintFactory.java | 47 ++
.../lens/server/quota/QuotaServiceImpl.java | 21 +-
.../apache/lens/server/rewrite/RewriteUtil.java | 390 +++++++++++++
.../rewrite/UserQueryToCubeQueryRewriter.java | 69 +++
.../scheduler/QuerySchedulerServiceImpl.java | 22 +-
.../server/session/DatabaseResourceService.java | 45 +-
.../lens/server/session/HiveSessionService.java | 83 ++-
.../lens/server/session/LensSessionImpl.java | 24 +-
.../lens/server/session/SessionResource.java | 15 +-
.../store/log/StatisticsLogFileScannerTask.java | 21 +-
.../lens/server/ui/MetastoreUIResource.java | 46 +-
.../lens/server/ui/QueryServiceUIResource.java | 12 +-
.../lens/server/ui/SessionUIResource.java | 11 +-
.../lens/server/ui/StaticFileResource.java | 5 -
.../server/user/DatabaseUserConfigLoader.java | 5 +-
.../lens/server/user/FixedUserConfigLoader.java | 25 +-
.../user/PropertyBasedUserConfigLoader.java | 15 +-
.../server/user/UserConfigLoaderFactory.java | 14 +-
.../server/util/FairPriorityBlockingQueue.java | 107 ++++
.../apache/lens/server/util/UtilityMethods.java | 20 +
.../src/main/resources/lensserver-default.xml | 55 +-
.../org/apache/lens/server/LensJerseyTest.java | 20 +-
.../org/apache/lens/server/LensTestUtil.java | 29 +-
.../apache/lens/server/TestLensApplication.java | 13 +
.../org/apache/lens/server/TestLogResource.java | 131 +++++
.../org/apache/lens/server/TestResource.java | 6 -
.../apache/lens/server/TestServerRestart.java | 152 ++---
.../lens/server/TestStartupOnMetastoreDown.java | 9 +-
.../lens/server/TestStaticFileResource.java | 5 -
.../lens/server/common/RestAPITestUtil.java | 6 +-
.../server/healthcheck/TestHealthChecks.java | 127 +++++
.../server/metastore/TestMetastoreService.java | 141 +++--
.../metrics/TestResourceMethodMetrics.java | 16 +-
.../lens/server/query/TestEventService.java | 44 +-
.../apache/lens/server/query/TestLensDAO.java | 9 +-
.../server/query/TestQueryEndEmailNotifier.java | 4 -
.../lens/server/query/TestQueryService.java | 338 +++++------
.../lens/server/query/TestResultFormatting.java | 9 +-
.../DefaultEstimatedQueryCollectionTest.java | 127 +++++
.../collect/DefaultQueryCollectionTest.java | 107 ++++
.../IntersectingWaitingQueriesSelectorTest.java | 185 ++++++
.../server/query/collect/QueryCollectUtil.java | 102 ++++
.../server/query/collect/ThreadSafetyTest.java | 37 ++
...ecificWaitingQueriesSelectionPolicyTest.java | 52 ++
...ultQueryLaunchingConstraintsCheckerTest.java | 149 +++++
...alQueryCostCeilingConstraintFactoryTest.java | 56 ++
.../TotalQueryCostCeilingConstraintTest.java | 75 +++
.../lens/server/rewrite/TestRewriting.java | 461 +++++++++++++++
.../TestUserQueryToCubeQueryRewriter.java | 82 +++
.../server/session/TestSessionClassLoaders.java | 30 +-
.../lens/server/user/FooBarConfigLoader.java | 11 +-
.../lens/server/user/TestUserConfigLoader.java | 3 -
lens-server/src/test/resources/lens-site.xml | 3 +-
lens-server/src/test/resources/log4j.properties | 15 +-
lens-server/testdata/ClassLoaderTestClass.java | 23 -
lens-server/testdata/ClassLoaderTestClass2.java | 23 -
lens-server/testdata/DatabaseJarSerde.java | 36 --
lens-server/testdata/serde.jar | Bin 1033 -> 0 bytes
lens-server/testdata/test.jar | Bin 726 -> 0 bytes
lens-server/testdata/test2.jar | Bin 700 -> 0 bytes
lens-storage-db/pom.xml | 3 +-
.../org/apache/lens/storage/db/DBStorage.java | 16 +
.../apache/lens/storage/db/TestDBStorage.java | 2 +-
pom.xml | 19 +-
src/site/apt/admin/config.apt | 140 +++--
src/site/apt/admin/deployment.apt | 24 +
src/site/apt/admin/hivedriver-config.apt | 16 +-
src/site/apt/admin/jdbcdriver-config.apt | 58 +-
src/site/apt/developer/codestructure.apt | 55 ++
src/site/apt/developer/commit.apt | 5 +-
src/site/apt/developer/contribute.apt | 11 +-
src/site/apt/releases/download.apt | 6 +-
src/site/apt/releases/release-history.apt | 23 +
src/site/apt/user/cli.apt | 16 +-
src/site/site.xml | 55 +-
tools/scripts/lens-cli.sh | 2 +-
322 files changed, 10934 insertions(+), 5094 deletions(-)
----------------------------------------------------------------------
[49/50] [abbrv] incubator-lens git commit: Revert "Lens-581 :
Rafactored all API to new api (Vikas Singh via sharad)"
Posted by sh...@apache.org.
Revert "Lens-581 : Rafactored all API to new api (Vikas Singh via sharad)"
This reverts commit 64934716e9a79e8e4c34f6e37a62f1da10f5e90a.
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/ace0dd24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/ace0dd24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/ace0dd24
Branch: refs/heads/LENS-581
Commit: ace0dd241b2d8b3b6cea2f424bb7b21abaa07102
Parents: 6493471
Author: Sharad Agarwal <sh...@flipkarts-mbp.fkipl.flipkart.com>
Authored: Wed Jul 29 10:27:24 2015 +0530
Committer: Sharad Agarwal <sh...@flipkarts-mbp.fkipl.flipkart.com>
Committed: Wed Jul 29 10:27:24 2015 +0530
----------------------------------------------------------------------
lens-ml-lib/data/lr/lr_train.data | 11 -
.../org/apache/lens/client/LensMLClient.java | 336 ++++----
.../apache/lens/client/LensMLJerseyClient.java | 221 ++++--
.../org/apache/lens/ml/algo/api/AlgoParam.java | 53 ++
.../org/apache/lens/ml/algo/api/Algorithm.java | 40 +-
.../org/apache/lens/ml/algo/api/MLAlgo.java | 53 ++
.../org/apache/lens/ml/algo/api/MLDriver.java | 2 +-
.../org/apache/lens/ml/algo/api/MLModel.java | 79 ++
.../apache/lens/ml/algo/api/TrainedModel.java | 30 -
.../apache/lens/ml/algo/lib/AlgoArgParser.java | 74 +-
.../org/apache/lens/ml/algo/lib/Algorithms.java | 52 +-
.../lens/ml/algo/lib/ClassifierBaseModel.java | 7 +-
.../lens/ml/algo/lib/ForecastingModel.java | 23 +-
.../lens/ml/algo/spark/BaseSparkAlgo.java | 155 ++--
.../spark/BaseSparkClassificationModel.java | 50 +-
.../ml/algo/spark/ColumnFeatureFunction.java | 34 +-
.../apache/lens/ml/algo/spark/HiveTableRDD.java | 8 +-
.../lens/ml/algo/spark/SparkMLDriver.java | 101 +--
.../lens/ml/algo/spark/TableTrainingSpec.java | 358 ++++-----
.../lens/ml/algo/spark/dt/DecisionTreeAlgo.java | 28 +-
.../dt/DecisionTreeClassificationModel.java | 13 +-
.../algo/spark/dt/SparkDecisionTreeModel.java | 4 +-
.../lens/ml/algo/spark/kmeans/KMeansAlgo.java | 114 ++-
.../spark/kmeans/KMeansClusteringModel.java | 54 +-
.../algo/spark/lr/LogisticRegressionAlgo.java | 34 +-
.../lr/LogitRegressionClassificationModel.java | 13 +-
.../lens/ml/algo/spark/nb/NaiveBayesAlgo.java | 21 +-
.../spark/nb/NaiveBayesClassificationModel.java | 13 +-
.../apache/lens/ml/algo/spark/svm/SVMAlgo.java | 30 +-
.../algo/spark/svm/SVMClassificationModel.java | 13 +-
.../main/java/org/apache/lens/ml/api/Algo.java | 37 -
.../java/org/apache/lens/ml/api/AlgoParam.java | 55 --
.../java/org/apache/lens/ml/api/AlgoSpec.java | 39 -
.../java/org/apache/lens/ml/api/DataSet.java | 49 --
.../java/org/apache/lens/ml/api/Evaluation.java | 77 --
.../java/org/apache/lens/ml/api/Feature.java | 63 --
.../java/org/apache/lens/ml/api/LensML.java | 183 ++---
.../org/apache/lens/ml/api/MLConfConstants.java | 86 ---
.../java/org/apache/lens/ml/api/MLProcess.java | 50 --
.../org/apache/lens/ml/api/MLTestReport.java | 95 +++
.../main/java/org/apache/lens/ml/api/Model.java | 63 --
.../org/apache/lens/ml/api/ModelInstance.java | 86 ---
.../org/apache/lens/ml/api/ModelMetadata.java | 118 +++
.../java/org/apache/lens/ml/api/Prediction.java | 83 --
.../java/org/apache/lens/ml/api/Status.java | 34 -
.../java/org/apache/lens/ml/api/TestReport.java | 125 +++
.../lens/ml/dao/InMemoryMetaStoreClient.java | 226 ------
.../java/org/apache/lens/ml/dao/MLDBUtils.java | 105 +++
.../org/apache/lens/ml/dao/MetaStoreClient.java | 217 ------
.../apache/lens/ml/impl/BatchPredictSpec.java | 366 ---------
.../java/org/apache/lens/ml/impl/HiveMLUDF.java | 91 +--
.../org/apache/lens/ml/impl/LensMLImpl.java | 723 ++++++++++++------
.../lens/ml/impl/MLProcessLifeCycleManager.java | 759 -------------------
.../java/org/apache/lens/ml/impl/MLRunner.java | 168 ++++
.../java/org/apache/lens/ml/impl/MLTask.java | 285 +++++++
.../java/org/apache/lens/ml/impl/MLUtils.java | 54 +-
.../org/apache/lens/ml/impl/ModelLoader.java | 183 +++--
.../org/apache/lens/ml/impl/QueryRunner.java | 7 +-
.../apache/lens/ml/impl/TableTestingSpec.java | 278 +++----
.../java/org/apache/lens/ml/server/MLApp.java | 6 +
.../apache/lens/ml/server/MLServiceImpl.java | 290 ++++---
.../lens/ml/server/MLServiceResource.java | 343 +++++++--
.../java/org/apache/lens/rdd/LensRDDClient.java | 61 +-
.../java/org/apache/lens/ml/ExampleUtils.java | 15 +-
.../java/org/apache/lens/ml/TestMLResource.java | 171 ++++-
.../java/org/apache/lens/ml/TestMLRunner.java | 37 +-
.../java/org/apache/lens/ml/TestMLServices.java | 281 -------
lens-ml-lib/src/test/resources/lens-site.xml | 271 ++++---
lens-ml-lib/src/test/resources/log4j.properties | 85 ---
src/site/apt/user/cli.apt | 6 +-
70 files changed, 3695 insertions(+), 4600 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/data/lr/lr_train.data
----------------------------------------------------------------------
diff --git a/lens-ml-lib/data/lr/lr_train.data b/lens-ml-lib/data/lr/lr_train.data
deleted file mode 100644
index 8ecf84c..0000000
--- a/lens-ml-lib/data/lr/lr_train.data
+++ /dev/null
@@ -1,11 +0,0 @@
-1 2 3 0 0
-0 3 5 1 1
-1 3 5 1 1
-0 6 7 0 0
-0 5 1 1 1
-0 3 1 1 1
-1 8 0 0 0
-0 1 1 0 0
-0 3 1 0 1
-1 3 0 0 0
-1 4 9 1 0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
index fc6d112..6dd0ecf 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLClient.java
@@ -20,55 +20,40 @@ package org.apache.lens.client;
import java.io.Closeable;
import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import javax.ws.rs.core.Form;
+
import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.ml.api.*;
-import org.apache.lens.ml.server.MLService;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.ServiceProvider;
-import org.apache.lens.server.api.ServiceProviderFactory;
+import org.apache.lens.ml.algo.api.MLAlgo;
+import org.apache.lens.ml.algo.api.MLModel;
+import org.apache.lens.ml.api.LensML;
+import org.apache.lens.ml.api.MLTestReport;
+import org.apache.lens.ml.api.ModelMetadata;
+import org.apache.lens.ml.api.TestReport;
import org.apache.lens.server.api.error.LensException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-public class LensMLClient implements LensML, Closeable {
- private static final Log LOG = LogFactory.getLog(LensMLClient.class);
- private static final HiveConf HIVE_CONF;
+import org.apache.commons.lang.StringUtils;
- static {
- HIVE_CONF = new HiveConf();
- // Add default config so that we know the service provider implementation
- HIVE_CONF.addResource("lensserver-default.xml");
- HIVE_CONF.addResource("lens-site.xml");
- }
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
- /**
- * The ml service.
- */
- MLService mlService;
- /**
- * The service provider.
- */
- ServiceProvider serviceProvider;
- /**
- * The service provider factory.
- */
- ServiceProviderFactory serviceProviderFactory;
- /**
- * The client.
- */
- private LensMLJerseyClient client;
+import lombok.extern.slf4j.Slf4j;
- /**
- * Instantiates a new ML service resource.
- */
- public LensMLClient() {
+/**
+ * Client side implementation of LensML
+ */
+@Slf4j
+public class LensMLClient implements LensML, Closeable {
- }
+ /** The client. */
+ private LensMLJerseyClient client;
public LensMLClient(String password) {
this(new LensClientConfig(), password);
@@ -87,147 +72,232 @@ public class LensMLClient implements LensML, Closeable {
}
public LensMLClient(LensClient lensClient) {
- client = new LensMLJerseyClient(lensClient.getConnection(), lensClient.getConnection().getSessionHandle());
- serviceProviderFactory = getServiceProviderFactory(HIVE_CONF);
- }
-
- private ServiceProvider getServiceProvider() {
- if (serviceProvider == null) {
- serviceProvider = serviceProviderFactory.getServiceProvider();
- }
- return serviceProvider;
+ client = new LensMLJerseyClient(lensClient.getConnection(), lensClient
+ .getConnection().getSessionHandle());
}
/**
- * Gets the service provider factory.
+ * Get list of available machine learning algorithms
*
- * @param conf the conf
- * @return the service provider factory
+ * @return
*/
- private ServiceProviderFactory getServiceProviderFactory(HiveConf conf) {
- Class<?> spfClass = conf.getClass(LensConfConstants.SERVICE_PROVIDER_FACTORY, ServiceProviderFactory.class);
- try {
- return (ServiceProviderFactory) spfClass.newInstance();
- } catch (InstantiationException e) {
- throw new RuntimeException(e);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- private MLService getMlService() {
- if (mlService == null) {
- mlService = (MLService) getServiceProvider().getService(MLService.NAME);
- }
- return mlService;
- }
-
@Override
- public void close() throws IOException {
- client.close();
- }
-
- @Override
- public List<Algo> getAlgos() {
- return getMlService().getAlgos();
- }
-
- public List<String> getAlgoNames() {
- return null;
- }
-
- @Override
- public Algo getAlgo(String name) throws LensException {
- return null;
- }
-
- @Override
- public String createDataSet(String name, String dataTable, String dataBase) throws LensException {
-
- return client.createDataSet(name, dataTable, dataBase);
- }
-
- public void test() {
- client.test();
+ public List<String> getAlgorithms() {
+ return client.getAlgoNames();
}
+ /**
+ * Get user friendly information about parameters accepted by the algorithm.
+ *
+ * @param algorithm the algorithm
+ * @return map of param key to its help message
+ */
@Override
- public String createDataSetFromQuery(String name, String query) {
- return null;
+ public Map<String, String> getAlgoParamDescription(String algorithm) {
+ List<String> paramDesc = client.getParamDescriptionOfAlgo(algorithm);
+ // convert paramDesc to map
+ Map<String, String> paramDescMap = new LinkedHashMap<String, String>();
+ for (String str : paramDesc) {
+ String[] keyHelp = StringUtils.split(str, ":");
+ paramDescMap.put(keyHelp[0].trim(), keyHelp[1].trim());
+ }
+ return paramDescMap;
}
+ /**
+ * Get a algo object instance which could be used to generate a model of the given algorithm.
+ *
+ * @param algorithm the algorithm
+ * @return the algo for name
+ * @throws LensException the lens exception
+ */
@Override
- public DataSet getDataSet(String name) throws LensException {
- return null;
+ public MLAlgo getAlgoForName(String algorithm) throws LensException {
+ throw new UnsupportedOperationException("MLAlgo cannot be accessed from client");
}
+ /**
+ * Create a model using the given HCatalog table as input. The arguments should contain information needeed to
+ * generate the model.
+ *
+ * @param table the table
+ * @param algorithm the algorithm
+ * @param args the args
+ * @return Unique ID of the model created after training is complete
+ * @throws LensException the lens exception
+ */
@Override
- public String createModel(String name, String algo, Map<String, String> algoParams, List<Feature> features,
- Feature label, LensSessionHandle lensSessionHandle) throws LensException {
- return getMlService().createModel(name, algo, algoParams, features, label, lensSessionHandle);
+ public String train(String table, String algorithm, String[] args) throws LensException {
+ Form trainParams = new Form();
+ trainParams.param("table", table);
+ for (int i = 0; i < args.length; i += 2) {
+ trainParams.param(args[i], args[i + 1]);
+ }
+ return client.trainModel(algorithm, trainParams);
}
+ /**
+ * Get model IDs for the given algorithm.
+ *
+ * @param algorithm the algorithm
+ * @return the models
+ * @throws LensException the lens exception
+ */
@Override
- public Model getModel(String modelId) throws LensException {
- return getMlService().getModel(modelId);
+ public List<String> getModels(String algorithm) throws LensException {
+ return client.getModelsForAlgorithm(algorithm);
}
+ /**
+ * Get a model instance given the algorithm name and model ID.
+ *
+ * @param algorithm the algorithm
+ * @param modelId the model id
+ * @return the model
+ * @throws LensException the lens exception
+ */
@Override
- public String trainModel(String modelId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- return getMlService().trainModel(modelId, dataSetName, lensSessionHandle);
- }
+ public MLModel getModel(String algorithm, String modelId) throws LensException {
+ ModelMetadata metadata = client.getModelMetadata(algorithm, modelId);
+ String modelPathURI = metadata.getModelPath();
- @Override
- public ModelInstance getModelInstance(String modelInstanceId) throws LensException {
- return getMlService().getModelInstance(modelInstanceId);
- }
+ ObjectInputStream in = null;
+ try {
+ URI modelURI = new URI(modelPathURI);
+ Path modelPath = new Path(modelURI);
+ FileSystem fs = FileSystem.get(modelURI, client.getConf());
+ in = new ObjectInputStream(fs.open(modelPath));
+ MLModel<?> model = (MLModel) in.readObject();
+ return model;
+ } catch (IOException e) {
+ throw new LensException(e);
+ } catch (URISyntaxException e) {
+ throw new LensException(e);
+ } catch (ClassNotFoundException e) {
+ throw new LensException(e);
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ log.error("Error closing stream.", e);
+ }
+ }
+ }
- @Override
- public List<ModelInstance> getAllModelInstances(String modelId) {
- return null;
}
+ /**
+ * Get the FS location where model instance is saved.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @return the model path
+ */
@Override
- public String evaluate(String modelInstanceId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- return getMlService().evaluate(modelInstanceId, dataSetName, lensSessionHandle);
+ public String getModelPath(String algorithm, String modelID) {
+ ModelMetadata metadata = client.getModelMetadata(algorithm, modelID);
+ return metadata.getModelPath();
}
+ /**
+ * Evaluate model by running it against test data contained in the given table.
+ *
+ * @param session the session
+ * @param table the table
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @return Test report object containing test output table, and various evaluation metrics
+ * @throws LensException the lens exception
+ */
@Override
- public Evaluation getEvaluation(String evalId) throws LensException {
- return getMlService().getEvaluation(evalId);
+ public MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
+ String outputTable) throws LensException {
+ String reportID = client.testModel(table, algorithm, modelID, outputTable);
+ return getTestReport(algorithm, reportID);
}
+ /**
+ * Get test reports for an algorithm.
+ *
+ * @param algorithm the algorithm
+ * @return the test reports
+ * @throws LensException the lens exception
+ */
@Override
- public String predict(String modelInstanceId, String dataSetName, LensSessionHandle lensSessionHandle)
- throws LensException {
- return getMlService().predict(modelInstanceId, dataSetName, lensSessionHandle);
+ public List<String> getTestReports(String algorithm) throws LensException {
+ return client.getTestReportsOfAlgorithm(algorithm);
}
+ /**
+ * Get a test report by ID.
+ *
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @return the test report
+ * @throws LensException the lens exception
+ */
@Override
- public boolean cancelModelInstance(String modelInstanceId) {
- return getMlService().cancelModelInstance(modelInstanceId);
+ public MLTestReport getTestReport(String algorithm, String reportID) throws LensException {
+ TestReport report = client.getTestReport(algorithm, reportID);
+ MLTestReport mlTestReport = new MLTestReport();
+ mlTestReport.setAlgorithm(report.getAlgorithm());
+ mlTestReport.setFeatureColumns(Arrays.asList(report.getFeatureColumns().split("\\,+")));
+ mlTestReport.setLensQueryID(report.getQueryID());
+ mlTestReport.setLabelColumn(report.getLabelColumn());
+ mlTestReport.setModelID(report.getModelID());
+ mlTestReport.setOutputColumn(report.getOutputColumn());
+ mlTestReport.setPredictionResultColumn(report.getOutputColumn());
+ mlTestReport.setQueryID(report.getQueryID());
+ mlTestReport.setReportID(report.getReportID());
+ mlTestReport.setTestTable(report.getTestTable());
+ return mlTestReport;
}
+ /**
+ * Online predict call given a model ID, algorithm name and sample feature values.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @param features the features
+ * @return prediction result
+ * @throws LensException the lens exception
+ */
@Override
- public boolean cancelEvaluation(String evalId) {
- return getMlService().cancelEvaluation(evalId);
+ public Object predict(String algorithm, String modelID, Object[] features) throws LensException {
+ return getModel(algorithm, modelID).predict(features);
}
+ /**
+ * Permanently delete a model instance.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @throws LensException the lens exception
+ */
@Override
- public boolean cancelPrediction(String predicitonId) {
- return getMlService().cancelPrediction(predicitonId);
+ public void deleteModel(String algorithm, String modelID) throws LensException {
+ client.deleteModel(algorithm, modelID);
}
+ /**
+ * Permanently delete a test report instance.
+ *
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @throws LensException the lens exception
+ */
@Override
- public Prediction getPrediction(String predictionId) throws LensException {
- return getMlService().getPrediction(predictionId);
+ public void deleteTestReport(String algorithm, String reportID) throws LensException {
+ client.deleteTestReport(algorithm, reportID);
}
+ /**
+ * Close connection
+ */
@Override
- public String predict(String modelInstanceId, Map<String, String> featureVector) throws LensException {
- return getMlService().predict(modelInstanceId, featureVector);
+ public void close() throws IOException {
+ client.close();
}
public LensSessionHandle getSessionHandle() {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
index e757d07..c68dd12 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/client/LensMLJerseyClient.java
@@ -21,46 +21,46 @@ package org.apache.lens.client;
import java.util.List;
import java.util.Map;
+import javax.ws.rs.NotFoundException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Form;
import javax.ws.rs.core.MediaType;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
-import org.apache.lens.api.result.LensAPIResult;
-import org.apache.lens.ml.api.Evaluation;
-import org.apache.lens.ml.api.Feature;
+import org.apache.lens.ml.api.ModelMetadata;
+import org.apache.lens.ml.api.TestReport;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.glassfish.jersey.media.multipart.MultiPartFeature;
-public class LensMLJerseyClient {
+/*
+ * Client code to invoke server side ML API
+ */
- /**
- * The Constant LENS_ML_RESOURCE_PATH.
- */
+/**
+ * The Class LensMLJerseyClient.
+ */
+public class LensMLJerseyClient {
+ /** The Constant LENS_ML_RESOURCE_PATH. */
public static final String LENS_ML_RESOURCE_PATH = "lens.ml.resource.path";
- /**
- * The Constant DEFAULT_ML_RESOURCE_PATH.
- */
+ /** The Constant DEFAULT_ML_RESOURCE_PATH. */
public static final String DEFAULT_ML_RESOURCE_PATH = "ml";
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(LensMLJerseyClient.class);
- /**
- * The connection.
- */
+ /** The connection. */
private final LensConnection connection;
private final LensSessionHandle sessionHandle;
@@ -86,14 +86,6 @@ public class LensMLJerseyClient {
this.sessionHandle = sessionHandle;
}
- public void close() {
- try {
- connection.close();
- } catch (Exception exc) {
- LOG.error("Error closing connection", exc);
- }
- }
-
protected WebTarget getMLWebTarget() {
Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).build();
LensConnectionParams connParams = connection.getLensConnectionParams();
@@ -102,48 +94,177 @@ public class LensMLJerseyClient {
return client.target(baseURI).path(mlURI);
}
- public List<String> getAlgoNames() {
- StringList algoNames = getMLWebTarget().path("algonames").request().get(StringList.class);
- return algoNames.getElements() == null ? null : algoNames.getElements();
+ /**
+ * Gets the model metadata.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @return the model metadata
+ */
+ public ModelMetadata getModelMetadata(String algorithm, String modelID) {
+ try {
+ return getMLWebTarget().path("models").path(algorithm).path(modelID).request().get(ModelMetadata.class);
+ } catch (NotFoundException exc) {
+ return null;
+ }
}
- public String createModel(String name, String algo, Map<String, String> algoParams,
- List<Feature> features, Feature label) {
- return "";
+ /**
+ * Delete model.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ */
+ public void deleteModel(String algorithm, String modelID) {
+ getMLWebTarget().path("models").path(algorithm).path(modelID).request().delete();
}
- public LensSessionHandle getSessionHandle() {
- if (sessionHandle != null) {
- return sessionHandle;
+ /**
+ * Gets the models for algorithm.
+ *
+ * @param algorithm the algorithm
+ * @return the models for algorithm
+ */
+ public List<String> getModelsForAlgorithm(String algorithm) {
+ try {
+ StringList models = getMLWebTarget().path("models").path(algorithm).request().get(StringList.class);
+ return models == null ? null : models.getElements();
+ } catch (NotFoundException exc) {
+ return null;
}
- return connection.getSessionHandle();
}
- public String createDataSet(String dataSetName, String dataTableName, String dataBase) {
+ public List<String> getAlgoNames() {
+ StringList algoNames = getMLWebTarget().path("algos").request().get(StringList.class);
+ return algoNames == null ? null : algoNames.getElements();
+ }
+
+ /**
+ * Train model.
+ *
+ * @param algorithm the algorithm
+ * @param params the params
+ * @return the string
+ */
+ public String trainModel(String algorithm, Form params) {
+ return getMLWebTarget().path(algorithm).path("train").request(MediaType.APPLICATION_JSON_TYPE)
+ .post(Entity.entity(params, MediaType.APPLICATION_FORM_URLENCODED_TYPE), String.class);
+ }
+
+ /**
+ * Test model.
+ *
+ * @param table the table
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @param outputTable the output table name
+ * @return the string
+ */
+ public String testModel(String table, String algorithm, String modelID, String outputTable) {
+ WebTarget modelTestTarget = getMLWebTarget().path("test").path(table).path(algorithm).path(modelID);
+
FormDataMultiPart mp = new FormDataMultiPart();
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("dataSetName").build(), dataSetName));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("dataTableName").build(), dataTableName));
- LensAPIResult result = getMLWebTarget().path("dataset")
- .queryParam("dataSetName", dataSetName)
- .queryParam("dataTableName", dataTableName)
- .queryParam("dataBaseName", dataBase)
- .request(MediaType.APPLICATION_XML)
- .post(Entity.xml(dataSetName), LensAPIResult.class);
+ LensSessionHandle sessionHandle = this.sessionHandle == null ? connection.getSessionHandle() : this.sessionHandle;
+
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), sessionHandle,
+ MediaType.APPLICATION_XML_TYPE));
- LensAPIResult<String> apiResult = getMLWebTarget().path("dataset").request().post(
- Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), LensAPIResult.class);
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("outputTable").build(), outputTable));
+ return modelTestTarget.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), String.class);
+ }
+
+ /**
+ * Gets the test reports of algorithm.
+ *
+ * @param algorithm the algorithm
+ * @return the test reports of algorithm
+ */
+ public List<String> getTestReportsOfAlgorithm(String algorithm) {
+ try {
+ StringList list = getMLWebTarget().path("reports").path(algorithm).request().get(StringList.class);
+ return list == null ? null : list.getElements();
+ } catch (NotFoundException exc) {
+ return null;
+ }
+ }
- return apiResult.getData();
+ /**
+ * Gets the test report.
+ *
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @return the test report
+ */
+ public TestReport getTestReport(String algorithm, String reportID) {
+ try {
+ return getMLWebTarget().path("reports").path(algorithm).path(reportID).request().get(TestReport.class);
+ } catch (NotFoundException exc) {
+ return null;
+ }
+ }
+ /**
+ * Delete test report.
+ *
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @return the string
+ */
+ public String deleteTestReport(String algorithm, String reportID) {
+ return getMLWebTarget().path("reports").path(algorithm).path(reportID).request().delete(String.class);
}
- public void test() {
+ /**
+ * Predict single.
+ *
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @param features the features
+ * @return the string
+ */
+ public String predictSingle(String algorithm, String modelID, Map<String, String> features) {
+ WebTarget target = getMLWebTarget().path("predict").path(algorithm).path(modelID);
- Evaluation result = getMLWebTarget().path("evaluation")
- .request(MediaType.APPLICATION_XML)
- .get(Evaluation.class);
+ for (Map.Entry<String, String> entry : features.entrySet()) {
+ target.queryParam(entry.getKey(), entry.getValue());
+ }
+ return target.request().get(String.class);
}
+ /**
+ * Gets the param description of algo.
+ *
+ * @param algorithm the algorithm
+ * @return the param description of algo
+ */
+ public List<String> getParamDescriptionOfAlgo(String algorithm) {
+ try {
+ StringList paramHelp = getMLWebTarget().path("algos").path(algorithm).request(MediaType.APPLICATION_XML)
+ .get(StringList.class);
+ return paramHelp.getElements();
+ } catch (NotFoundException exc) {
+ return null;
+ }
+ }
+
+ public Configuration getConf() {
+ return connection.getLensConnectionParams().getConf();
+ }
+
+ public void close() {
+ try {
+ connection.close();
+ } catch (Exception exc) {
+ LOG.error("Error closing connection", exc);
+ }
+ }
+
+ public LensSessionHandle getSessionHandle() {
+ if (sessionHandle != null) {
+ return sessionHandle;
+ }
+ return connection.getSessionHandle();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/AlgoParam.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/AlgoParam.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/AlgoParam.java
new file mode 100644
index 0000000..e0d13c0
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/AlgoParam.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.algo.api;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * The Interface AlgoParam.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+public @interface AlgoParam {
+
+ /**
+ * Name.
+ *
+ * @return the string
+ */
+ String name();
+
+ /**
+ * Help.
+ *
+ * @return the string
+ */
+ String help();
+
+ /**
+ * Default value.
+ *
+ * @return the string
+ */
+ String defaultValue() default "None";
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/Algorithm.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/Algorithm.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/Algorithm.java
index 187c524..29bde29 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/Algorithm.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/Algorithm.java
@@ -18,31 +18,29 @@
*/
package org.apache.lens.ml.algo.api;
-import java.util.List;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
-import org.apache.lens.api.LensConf;
-import org.apache.lens.ml.api.AlgoParam;
-import org.apache.lens.ml.api.DataSet;
-import org.apache.lens.ml.api.Model;
-import org.apache.lens.server.api.error.LensException;
-
-public interface Algorithm {
-
- String getName();
-
- String getDescription();
-
- List<AlgoParam> getParams();
+/**
+ * The Interface Algorithm.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface Algorithm {
/**
- * Configure.
+ * Name.
*
- * @param configuration the configuration
+ * @return the string
*/
- void configure(LensConf configuration);
-
- LensConf getConf();
-
- TrainedModel train(Model model, DataSet dataTable) throws LensException;
+ String name();
+ /**
+ * Description.
+ *
+ * @return the string
+ */
+ String description();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLAlgo.java
new file mode 100644
index 0000000..65373c6
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLAlgo.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.algo.api;
+
+import org.apache.lens.api.LensConf;
+import org.apache.lens.server.api.error.LensException;
+
+/**
+ * The Interface MLAlgo.
+ */
+public interface MLAlgo {
+ String getName();
+
+ String getDescription();
+
+ /**
+ * Configure.
+ *
+ * @param configuration the configuration
+ */
+ void configure(LensConf configuration);
+
+ LensConf getConf();
+
+ /**
+ * Train.
+ *
+ * @param conf the conf
+ * @param db the db
+ * @param table the table
+ * @param modelId the model id
+ * @param params the params
+ * @return the ML model
+ * @throws LensException the lens exception
+ */
+ MLModel train(LensConf conf, String db, String table, String modelId, String... params) throws LensException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLDriver.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLDriver.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLDriver.java
index 9814008..d2a2748 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLDriver.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLDriver.java
@@ -43,7 +43,7 @@ public interface MLDriver {
* @return the algo instance
* @throws LensException the lens exception
*/
- Algorithm getAlgoInstance(String algo) throws LensException;
+ MLAlgo getAlgoInstance(String algo) throws LensException;
/**
* Inits the.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLModel.java
new file mode 100644
index 0000000..73717ac
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/MLModel.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.algo.api;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.List;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+/**
+ * Instantiates a new ML model.
+ */
+@NoArgsConstructor
+@ToString
+public abstract class MLModel<PREDICTION> implements Serializable {
+
+ /** The id. */
+ @Getter
+ @Setter
+ private String id;
+
+ /** The created at. */
+ @Getter
+ @Setter
+ private Date createdAt;
+
+ /** The algo name. */
+ @Getter
+ @Setter
+ private String algoName;
+
+ /** The table. */
+ @Getter
+ @Setter
+ private String table;
+
+ /** The params. */
+ @Getter
+ @Setter
+ private List<String> params;
+
+ /** The label column. */
+ @Getter
+ @Setter
+ private String labelColumn;
+
+ /** The feature columns. */
+ @Getter
+ @Setter
+ private List<String> featureColumns;
+
+ /**
+ * Predict.
+ *
+ * @param args the args
+ * @return the prediction
+ */
+ public abstract PREDICTION predict(Object... args);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/TrainedModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/TrainedModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/TrainedModel.java
deleted file mode 100644
index 2f23912..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/api/TrainedModel.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.algo.api;
-
-import java.io.Serializable;
-import java.util.Map;
-
-import org.apache.lens.server.api.error.LensException;
-
-public interface TrainedModel<PREDICTION> extends Serializable {
-
- PREDICTION predict(Map<String, String> featureVector) throws LensException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
index 2f808f2..00f20fc 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/AlgoArgParser.java
@@ -19,39 +19,51 @@
package org.apache.lens.ml.algo.lib;
import java.lang.reflect.Field;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import org.apache.lens.ml.algo.api.Algorithm;
-import org.apache.lens.ml.api.AlgoParam;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.lens.ml.algo.api.AlgoParam;
+import org.apache.lens.ml.algo.api.MLAlgo;
import lombok.extern.slf4j.Slf4j;
/**
- * AlgoArgParser class. Parses and sets algo params.
+ * The Class AlgoArgParser.
*/
@Slf4j
public final class AlgoArgParser {
+ private AlgoArgParser() {
+ }
+
/**
- * The Constant LOG.
+ * The Class CustomArgParser.
+ *
+ * @param <E> the element type
*/
- public static final Log LOG = LogFactory.getLog(AlgoArgParser.class);
+ public abstract static class CustomArgParser<E> {
- private AlgoArgParser() {
+ /**
+ * Parses the.
+ *
+ * @param value the value
+ * @return the e
+ */
+ public abstract E parse(String value);
}
/**
- * Extracts all the variables annotated with @AlgoParam checks if any input key matches the name
- * if so replaces the value.
+ * Extracts feature names. If the algo has any parameters associated with @AlgoParam annotation, those are set
+ * as well.
*
- * @param algo
- * @param algoParameters
+ * @param algo the algo
+ * @param args the args
+ * @return List of feature column names.
*/
- public static void parseArgs(Algorithm algo, Map<String, String> algoParameters) {
- Class<? extends Algorithm> algoClass = algo.getClass();
+ public static List<String> parseArgs(MLAlgo algo, String[] args) {
+ List<String> featureColumns = new ArrayList<String>();
+ Class<? extends MLAlgo> algoClass = algo.getClass();
// Get param fields
Map<String, Field> fieldMap = new HashMap<String, Field>();
@@ -63,11 +75,14 @@ public final class AlgoArgParser {
}
}
- for (Map.Entry<String, String> entry : algoParameters.entrySet()) {
- String key = entry.getKey();
- String value = entry.getValue();
+ for (int i = 0; i < args.length; i += 2) {
+ String key = args[i].trim();
+ String value = args[i + 1].trim();
+
try {
- if (fieldMap.containsKey(key)) {
+ if ("feature".equalsIgnoreCase(key)) {
+ featureColumns.add(value);
+ } else if (fieldMap.containsKey(key)) {
Field f = fieldMap.get(key);
if (String.class.equals(f.getType())) {
f.set(algo, value);
@@ -86,29 +101,14 @@ public final class AlgoArgParser {
CustomArgParser<?> parser = clz.newInstance();
f.set(algo, parser.parse(value));
} else {
- LOG.warn("Ignored param " + key + "=" + value + " as no parser found");
+ log.warn("Ignored param " + key + "=" + value + " as no parser found");
}
}
}
} catch (Exception exc) {
- LOG.error("Error while setting param " + key + " to " + value + " for algo " + algo, exc);
+ log.error("Error while setting param " + key + " to " + value + " for algo " + algo, exc);
}
}
- }
-
- /**
- * The Class CustomArgParser.
- *
- * @param <E> the element type
- */
- public abstract static class CustomArgParser<E> {
-
- /**
- * Parses the.
- *
- * @param value the value
- * @return the e
- */
- public abstract E parse(String value);
+ return featureColumns;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/Algorithms.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/Algorithms.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/Algorithms.java
index 87ed8ee..ad37403 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/Algorithms.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/Algorithms.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.lens.ml.algo.api.Algorithm;
+import org.apache.lens.ml.algo.api.MLAlgo;
import org.apache.lens.server.api.error.LensException;
/**
@@ -32,44 +33,50 @@ import org.apache.lens.server.api.error.LensException;
*/
public class Algorithms {
- /**
- * The algorithm classes.
- */
-
- private final Map<String, Class<? extends Algorithm>> algorithmClasses
- = new HashMap<String, Class<? extends Algorithm>>();
+ /** The algorithm classes. */
+ private final Map<String, Class<? extends MLAlgo>> algorithmClasses
+ = new HashMap<String, Class<? extends MLAlgo>>();
/**
- * Registers algorithm
+ * Register.
*
- * @param name
- * @param algoClass
+ * @param algoClass the algo class
*/
- public void register(String name, Class<? extends Algorithm> algoClass) {
- if (algoClass != null) {
- algorithmClasses.put(name, algoClass);
+ public void register(Class<? extends MLAlgo> algoClass) {
+ if (algoClass != null && algoClass.getAnnotation(Algorithm.class) != null) {
+ algorithmClasses.put(algoClass.getAnnotation(Algorithm.class).name(), algoClass);
+ } else {
+ throw new IllegalArgumentException("Not a valid algorithm class: " + algoClass);
}
}
- public Algorithm getAlgoForName(String name) throws LensException {
- Class<? extends Algorithm> algoClass = algorithmClasses.get(name);
-
+ /**
+ * Gets the algo for name.
+ *
+ * @param name the name
+ * @return the algo for name
+ * @throws LensException the lens exception
+ */
+ public MLAlgo getAlgoForName(String name) throws LensException {
+ Class<? extends MLAlgo> algoClass = algorithmClasses.get(name);
if (algoClass == null) {
return null;
}
+ Algorithm algoAnnotation = algoClass.getAnnotation(Algorithm.class);
+ String description = algoAnnotation.description();
try {
- Constructor<? extends Algorithm> constructor = algoClass.getConstructor();
- return constructor.newInstance();
- } catch (Exception e) {
- throw new LensException("Unable to get Algorithm " + name, e);
+ Constructor<? extends MLAlgo> algoConstructor = algoClass.getConstructor(String.class, String.class);
+ return algoConstructor.newInstance(name, description);
+ } catch (Exception exc) {
+ throw new LensException("Unable to get algo: " + name, exc);
}
}
/**
- * Checks if algorithm is supported
+ * Checks if is algo supported.
*
- * @param name
- * @return
+ * @param name the name
+ * @return true, if is algo supported
*/
public boolean isAlgoSupported(String name) {
return algorithmClasses.containsKey(name);
@@ -78,4 +85,5 @@ public class Algorithms {
public List<String> getAlgorithmNames() {
return new ArrayList<String>(algorithmClasses.keySet());
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ClassifierBaseModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ClassifierBaseModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ClassifierBaseModel.java
index 6fb2101..a960a4a 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ClassifierBaseModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ClassifierBaseModel.java
@@ -18,12 +18,13 @@
*/
package org.apache.lens.ml.algo.lib;
-import org.apache.lens.ml.algo.api.TrainedModel;
+import org.apache.lens.ml.algo.api.MLModel;
/**
- * The Class ClassifierBaseModel
+ * Return a single double value as a prediction. This is useful in classifiers where the classifier returns a single
+ * class label as a prediction.
*/
-public abstract class ClassifierBaseModel implements TrainedModel<Double> {
+public abstract class ClassifierBaseModel extends MLModel<Double> {
/**
* Gets the feature vector.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ForecastingModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ForecastingModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ForecastingModel.java
index 7176797..16a6180 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ForecastingModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/lib/ForecastingModel.java
@@ -19,24 +19,21 @@
package org.apache.lens.ml.algo.lib;
import java.util.List;
-import java.util.Map;
-import org.apache.lens.ml.algo.api.TrainedModel;
-import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.ml.algo.api.MLModel;
/**
* The Class ForecastingModel.
*/
-public class ForecastingModel implements TrainedModel<MultiPrediction> {
+public class ForecastingModel extends MLModel<MultiPrediction> {
/*
* (non-Javadoc)
*
- * @see org.apache.lens.ml.TrainedModel#predict(java.lang.Object[])
+ * @see org.apache.lens.ml.MLModel#predict(java.lang.Object[])
*/
-
@Override
- public MultiPrediction predict(Map<String, String> featureVector) throws LensException {
+ public MultiPrediction predict(Object... args) {
return new ForecastingPredictions(null);
}
@@ -45,9 +42,7 @@ public class ForecastingModel implements TrainedModel<MultiPrediction> {
*/
public static class ForecastingPredictions implements MultiPrediction {
- /**
- * The values.
- */
+ /** The values. */
private final List<LabelledPrediction> values;
/**
@@ -70,14 +65,10 @@ public class ForecastingModel implements TrainedModel<MultiPrediction> {
*/
public static class ForecastingLabel implements LabelledPrediction<Long, Double> {
- /**
- * The timestamp.
- */
+ /** The timestamp. */
private final Long timestamp;
- /**
- * The value.
- */
+ /** The value. */
private final double value;
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
index 30bbf88..86cab38 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkAlgo.java
@@ -22,18 +22,15 @@ import java.lang.reflect.Field;
import java.util.*;
import org.apache.lens.api.LensConf;
+import org.apache.lens.ml.algo.api.AlgoParam;
import org.apache.lens.ml.algo.api.Algorithm;
-import org.apache.lens.ml.algo.api.TrainedModel;
-import org.apache.lens.ml.api.AlgoParam;
-import org.apache.lens.ml.api.DataSet;
-import org.apache.lens.ml.api.Feature;
-import org.apache.lens.ml.api.Model;
+import org.apache.lens.ml.algo.api.MLAlgo;
+import org.apache.lens.ml.algo.api.MLModel;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.rdd.RDD;
@@ -41,63 +38,44 @@ import org.apache.spark.rdd.RDD;
/**
* The Class BaseSparkAlgo.
*/
-public abstract class BaseSparkAlgo implements Algorithm {
+public abstract class BaseSparkAlgo implements MLAlgo {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(BaseSparkAlgo.class);
- /**
- * The name.
- */
+ /** The name. */
private final String name;
- /**
- * The description.
- */
+ /** The description. */
private final String description;
- /**
- * The spark context.
- */
+ /** The spark context. */
protected JavaSparkContext sparkContext;
- /**
- * The params.
- */
+ /** The params. */
protected Map<String, String> params;
- /**
- * The conf.
- */
+ /** The conf. */
protected transient LensConf conf;
- /**
- * The training fraction.
- */
+ /** The training fraction. */
@AlgoParam(name = "trainingFraction", help = "% of dataset to be used for training", defaultValue = "0")
protected double trainingFraction;
- /**
- * The label.
- */
- @AlgoParam(name = "label", help = "column name, feature name which is used as a training label for supervised "
- + "learning")
- protected Feature label;
- /**
- * The partition filter.
- */
+
+ /** The use training fraction. */
+ private boolean useTrainingFraction;
+
+ /** The label. */
+ @AlgoParam(name = "label", help = "Name of column which is used as a training label for supervised learning")
+ protected String label;
+
+ /** The partition filter. */
@AlgoParam(name = "partition", help = "Partition filter used to create create HCatInputFormats")
protected String partitionFilter;
- /**
- * The features.
- */
- @AlgoParam(name = "feature", help = "sample features containing feature name and column name")
- protected List<Feature> features;
- /**
- * The use training fraction.
- */
- private boolean useTrainingFraction;
+
+ /** The features. */
+ @AlgoParam(name = "feature", help = "Column name(s) which are to be used as sample features")
+ protected List<String> features;
/**
* Instantiates a new base spark algo.
@@ -132,26 +110,17 @@ public abstract class BaseSparkAlgo implements Algorithm {
/*
* (non-Javadoc)
*
- * @see org.apache.lens.ml.TrainedModel#train(Model model, String dataTable)
+ * @see org.apache.lens.ml.MLAlgo#train(org.apache.lens.api.LensConf, java.lang.String, java.lang.String,
+ * java.lang.String, java.lang.String[])
*/
@Override
- public TrainedModel train(Model model, DataSet dataTable) throws LensException {
- parseParams(model.getAlgoSpec().getAlgoParams());
- features = model.getFeatureSpec();
- String database = dataTable.getDatabase();
- if (database.isEmpty()) {
- if (SessionState.get() != null) {
- database = SessionState.get().getCurrentDatabase();
- } else {
- database = "default";
- }
- }
-
+ public MLModel<?> train(LensConf conf, String db, String table, String modelId, String... params)
+ throws LensException {
+ parseParams(params);
TableTrainingSpec.TableTrainingSpecBuilder builder = TableTrainingSpec.newBuilder().hiveConf(toHiveConf(conf))
- .database(database).table(dataTable.getTableName()).partitionFilter(partitionFilter)
- .featureColumns(model.getFeatureSpec())
- .labelColumn(model.getLabelSpec());
+ .database(db).table(table).partitionFilter(partitionFilter).featureColumns(features).labelColumn(label);
+
if (useTrainingFraction) {
builder.trainingFraction(trainingFraction);
}
@@ -162,8 +131,12 @@ public abstract class BaseSparkAlgo implements Algorithm {
spec.createRDDs(sparkContext);
RDD<LabeledPoint> trainingRDD = spec.getTrainingRDD();
- BaseSparkClassificationModel<?> trainedModel = trainInternal(trainingRDD);
- return trainedModel;
+ BaseSparkClassificationModel<?> model = trainInternal(modelId, trainingRDD);
+ model.setTable(table);
+ model.setParams(Arrays.asList(params));
+ model.setLabelColumn(label);
+ model.setFeatureColumns(features);
+ return model;
}
/**
@@ -185,13 +158,29 @@ public abstract class BaseSparkAlgo implements Algorithm {
*
* @param args the args
*/
+ public void parseParams(String[] args) {
+ if (args.length % 2 != 0) {
+ throw new IllegalArgumentException("Invalid number of params " + args.length);
+ }
- public void parseParams(Map<String, String> args) {
+ params = new LinkedHashMap<String, String>();
- params = new HashMap();
+ for (int i = 0; i < args.length; i += 2) {
+ if ("f".equalsIgnoreCase(args[i]) || "feature".equalsIgnoreCase(args[i])) {
+ if (features == null) {
+ features = new ArrayList<String>();
+ }
+ features.add(args[i + 1]);
+ } else if ("l".equalsIgnoreCase(args[i]) || "label".equalsIgnoreCase(args[i])) {
+ label = args[i + 1];
+ } else {
+ params.put(args[i].replaceAll("\\-+", ""), args[i + 1]);
+ }
+ }
- if (args.containsKey("trainingFraction")) {
- String trainingFractionStr = args.get("trainingFraction");
+ if (params.containsKey("trainingFraction")) {
+ // Get training Fraction
+ String trainingFractionStr = params.get("trainingFraction");
try {
trainingFraction = Double.parseDouble(trainingFractionStr);
useTrainingFraction = true;
@@ -199,11 +188,12 @@ public abstract class BaseSparkAlgo implements Algorithm {
throw new IllegalArgumentException("Invalid training fraction", nfe);
}
}
- if (args.containsKey("partition") || args.containsKey("p")) {
- partitionFilter = args.containsKey("partition") ? args.get("partition") : args.get("p");
+
+ if (params.containsKey("partition") || params.containsKey("p")) {
+ partitionFilter = params.containsKey("partition") ? params.get("partition") : params.get("p");
}
- parseAlgoParams(args);
+ parseAlgoParams(params);
}
/**
@@ -253,8 +243,12 @@ public abstract class BaseSparkAlgo implements Algorithm {
public Map<String, String> getArgUsage() {
Map<String, String> usage = new LinkedHashMap<String, String>();
Class<?> clz = this.getClass();
- usage.put("Algorithm Name", name);
- usage.put("Algorithm Description", description);
+ // Put class name and description as well as part of the usage
+ Algorithm algorithm = clz.getAnnotation(Algorithm.class);
+ if (algorithm != null) {
+ usage.put("Algorithm Name", algorithm.name());
+ usage.put("Algorithm Description", algorithm.description());
+ }
// Get all algo params including base algo params
while (clz != null) {
@@ -283,22 +277,11 @@ public abstract class BaseSparkAlgo implements Algorithm {
/**
* Train internal.
*
+ * @param modelId the model id
* @param trainingRDD the training rdd
* @return the base spark classification model
* @throws LensException the lens exception
*/
- protected abstract BaseSparkClassificationModel trainInternal(RDD<LabeledPoint> trainingRDD)
+ protected abstract BaseSparkClassificationModel trainInternal(String modelId, RDD<LabeledPoint> trainingRDD)
throws LensException;
-
- @Override
- public List<AlgoParam> getParams() {
- ArrayList<AlgoParam> paramList = new ArrayList();
- for (Field field : this.getClass().getDeclaredFields()) {
- AlgoParam param = field.getAnnotation(AlgoParam.class);
- if (param != null) {
- paramList.add(param);
- }
- }
- return paramList;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkClassificationModel.java
index 945d97d..806dc1f 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkClassificationModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/BaseSparkClassificationModel.java
@@ -18,56 +18,48 @@
*/
package org.apache.lens.ml.algo.spark;
-import java.util.List;
-import java.util.Map;
-
import org.apache.lens.ml.algo.lib.ClassifierBaseModel;
-import org.apache.lens.ml.api.Feature;
-import org.apache.lens.server.api.error.LensException;
import org.apache.spark.mllib.classification.ClassificationModel;
import org.apache.spark.mllib.linalg.Vectors;
/**
- * The class BaseSparkClassificationModel
+ * The Class BaseSparkClassificationModel.
*
- * @param <MODEL>
+ * @param <MODEL> the generic type
*/
public class BaseSparkClassificationModel<MODEL extends ClassificationModel> extends ClassifierBaseModel {
+ /** The model id. */
+ private final String modelId;
- /**
- * The spark model.
- */
+ /** The spark model. */
private final MODEL sparkModel;
- private List<Feature> featureList;
/**
- * initializes BaseSparkClassificationModel
+ * Instantiates a new base spark classification model.
*
- * @param featureList
- * @param model
+ * @param modelId the model id
+ * @param model the model
*/
- public BaseSparkClassificationModel(List<Feature> featureList, MODEL model) {
+ public BaseSparkClassificationModel(String modelId, MODEL model) {
+ this.modelId = modelId;
this.sparkModel = model;
- this.featureList = featureList;
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.MLModel#predict(java.lang.Object[])
+ */
+ @Override
+ public Double predict(Object... args) {
+ return sparkModel.predict(Vectors.dense(getFeatureVector(args)));
+ }
@Override
- public Double predict(Map<String, String> featureVector) throws LensException {
- String[] featureArray = new String[featureList.size()];
- int i = 0;
- for (Feature feature : featureList) {
- String featureValue = featureVector.get(feature.getName());
- if (featureValue == null || featureValue.isEmpty()) {
- throw new LensException("Error while predicting: input featureVector doesn't contain all required features : "
- + "Feature Name: " + feature.getName());
- } else {
- featureArray[i++] = featureVector.get(feature.getName());
- }
- }
- return sparkModel.predict(Vectors.dense(getFeatureVector(featureArray)));
+ public String getId() {
+ return modelId;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
index 4676ced..d75efc0 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/ColumnFeatureFunction.java
@@ -33,34 +33,22 @@ import scala.Tuple2;
*/
public class ColumnFeatureFunction extends FeatureFunction {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Logger LOG = Logger.getLogger(ColumnFeatureFunction.class);
- /**
- * The feature value mappers.
- */
+ /** The feature value mappers. */
private final FeatureValueMapper[] featureValueMappers;
- /**
- * The feature positions.
- */
+ /** The feature positions. */
private final int[] featurePositions;
- /**
- * The label column pos.
- */
+ /** The label column pos. */
private final int labelColumnPos;
- /**
- * The num features.
- */
+ /** The num features. */
private final int numFeatures;
- /**
- * The default labeled point.
- */
+ /** The default labeled point. */
private final LabeledPoint defaultLabeledPoint;
/**
@@ -74,12 +62,11 @@ public class ColumnFeatureFunction extends FeatureFunction {
* @param defaultLabel default lable to be used for null records
*/
public ColumnFeatureFunction(int[] featurePositions, FeatureValueMapper[] valueMappers, int labelColumnPos,
- int numFeatures, double defaultLabel) {
+ int numFeatures, double defaultLabel) {
Preconditions.checkNotNull(valueMappers, "Value mappers argument is required");
Preconditions.checkNotNull(featurePositions, "Feature positions are required");
- Preconditions.checkArgument(valueMappers.length == featurePositions.length + 1,
- "Mismatch between number of value mappers and feature positions. There should be value mappers for features and "
- + "one additional mapper for label");
+ Preconditions.checkArgument(valueMappers.length == featurePositions.length,
+ "Mismatch between number of value mappers and feature positions");
this.featurePositions = featurePositions;
this.featureValueMappers = valueMappers;
@@ -109,8 +96,7 @@ public class ColumnFeatureFunction extends FeatureFunction {
features[i] = featureValueMappers[i].call(record.get(featurePos));
}
- //Feature mapper for label is stored after label mappers at position numFeatures.
- double label = featureValueMappers[numFeatures].call(record.get(labelColumnPos));
+ double label = featureValueMappers[labelColumnPos].call(record.get(labelColumnPos));
return new LabeledPoint(label, Vectors.dense(features));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
index c87a2d8..4960e1e 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/HiveTableRDD.java
@@ -33,11 +33,11 @@ import org.apache.spark.api.java.JavaSparkContext;
* Create a JavaRDD based on a Hive table using HCatInputFormat.
*/
public final class HiveTableRDD {
- public static final Log LOG = LogFactory.getLog(HiveTableRDD.class);
-
private HiveTableRDD() {
}
+ public static final Log LOG = LogFactory.getLog(HiveTableRDD.class);
+
/**
* Creates the hive table rdd.
*
@@ -50,9 +50,7 @@ public final class HiveTableRDD {
* @throws IOException Signals that an I/O exception has occurred.
*/
public static JavaPairRDD<WritableComparable, HCatRecord> createHiveTableRDD(JavaSparkContext javaSparkContext,
- Configuration conf, String db,
- String table, String partitionFilter)
- throws IOException {
+ Configuration conf, String db, String table, String partitionFilter) throws IOException {
HCatInputFormat.setInput(conf, db, table, partitionFilter);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
index 3e9b94a..21ed87d 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
@@ -24,11 +24,10 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lens.api.LensConf;
-import org.apache.lens.ml.algo.api.Algorithm;
+import org.apache.lens.ml.algo.api.MLAlgo;
import org.apache.lens.ml.algo.api.MLDriver;
import org.apache.lens.ml.algo.lib.Algorithms;
import org.apache.lens.ml.algo.spark.dt.DecisionTreeAlgo;
-import org.apache.lens.ml.algo.spark.kmeans.KMeansAlgo;
import org.apache.lens.ml.algo.spark.lr.LogisticRegressionAlgo;
import org.apache.lens.ml.algo.spark.nb.NaiveBayesAlgo;
import org.apache.lens.ml.algo.spark.svm.SVMAlgo;
@@ -45,33 +44,40 @@ import org.apache.spark.api.java.JavaSparkContext;
*/
public class SparkMLDriver implements MLDriver {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(SparkMLDriver.class);
- /**
- * Supported algorithms.
- */
- private final Algorithms algorithms = new Algorithms();
- /**
- * If the driver owns spark's context.
- */
+
+ /** The owns spark context. */
private boolean ownsSparkContext = true;
+
/**
- * Spark's client mode.
+ * The Enum SparkMasterMode.
*/
+ private enum SparkMasterMode {
+ // Embedded mode used in tests
+ /** The embedded. */
+ EMBEDDED,
+ // Yarn client and Yarn cluster modes are used when deploying the app to Yarn cluster
+ /** The yarn client. */
+ YARN_CLIENT,
+
+ /** The yarn cluster. */
+ YARN_CLUSTER
+ }
+
+ /** The algorithms. */
+ private final Algorithms algorithms = new Algorithms();
+
+ /** The client mode. */
private SparkMasterMode clientMode = SparkMasterMode.EMBEDDED;
- /**
- * If the driver is started.
- */
+
+ /** The is started. */
private boolean isStarted;
- /**
- * The spark conf.
- */
+
+ /** The spark conf. */
private SparkConf sparkConf;
- /**
- * The spark context.
- */
+
+ /** The spark context. */
private JavaSparkContext sparkContext;
/**
@@ -94,22 +100,20 @@ public class SparkMLDriver implements MLDriver {
return algorithms.isAlgoSupported(name);
}
- /**
- * Returns Algorithm Instance for it's name
+ /*
+ * (non-Javadoc)
*
- * @param name
- * @return
- * @throws LensException
+ * @see org.apache.lens.ml.MLDriver#getAlgoInstance(java.lang.String)
*/
@Override
- public Algorithm getAlgoInstance(String name) throws LensException {
+ public MLAlgo getAlgoInstance(String name) throws LensException {
checkStarted();
if (!isAlgoSupported(name)) {
return null;
}
- Algorithm algo = null;
+ MLAlgo algo = null;
try {
algo = algorithms.getAlgoForName(name);
if (algo instanceof BaseSparkAlgo) {
@@ -122,21 +126,19 @@ public class SparkMLDriver implements MLDriver {
}
/**
- * Register Algorithms
+ * Register algos.
*/
private void registerAlgos() {
- algorithms.register("spark_logistic_regression", LogisticRegressionAlgo.class);
- algorithms.register("spark_naive_bayes", NaiveBayesAlgo.class);
- algorithms.register("spark_k_means", KMeansAlgo.class);
- algorithms.register("spark_svm", SVMAlgo.class);
- algorithms.register("spark_decision_tree", DecisionTreeAlgo.class);
+ algorithms.register(NaiveBayesAlgo.class);
+ algorithms.register(SVMAlgo.class);
+ algorithms.register(LogisticRegressionAlgo.class);
+ algorithms.register(DecisionTreeAlgo.class);
}
- /**
- * Initializes the driver
+ /*
+ * (non-Javadoc)
*
- * @param conf the conf
- * @throws LensException
+ * @see org.apache.lens.ml.MLDriver#init(org.apache.lens.api.LensConf)
*/
@Override
public void init(LensConf conf) throws LensException {
@@ -280,25 +282,4 @@ public class SparkMLDriver implements MLDriver {
return sparkContext;
}
- /**
- * The Enum SparkMasterMode.
- */
- private enum SparkMasterMode {
- // Embedded mode used in tests
- /**
- * The embedded.
- */
- EMBEDDED,
- // Yarn client and Yarn cluster modes are used when deploying the app to Yarn cluster
- /**
- * The yarn client.
- */
- YARN_CLIENT,
-
- /**
- * The yarn cluster.
- */
- YARN_CLUSTER
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
index 4362945..897aacb 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/TableTrainingSpec.java
@@ -23,7 +23,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lens.ml.api.Feature;
import org.apache.lens.server.api.error.LensException;
import org.apache.commons.logging.Log;
@@ -51,212 +50,67 @@ import lombok.ToString;
@ToString
public class TableTrainingSpec implements Serializable {
- /**
- * The Constant LOG.
- */
+ /** The Constant LOG. */
public static final Log LOG = LogFactory.getLog(TableTrainingSpec.class);
- /**
- * The label pos.
- */
- int labelPos;
- /**
- * The feature positions.
- */
- int[] featurePositions;
- /**
- * The num features.
- */
- int numFeatures;
- /**
- * The labeled rdd.
- */
- transient JavaRDD<LabeledPoint> labeledRDD;
- /**
- * The training rdd.
- */
+
+ /** The training rdd. */
@Getter
private transient RDD<LabeledPoint> trainingRDD;
- /**
- * The testing rdd.
- */
+
+ /** The testing rdd. */
@Getter
private transient RDD<LabeledPoint> testingRDD;
- /**
- * The database.
- */
+
+ /** The database. */
@Getter
private String database;
- /**
- * The table.
- */
+
+ /** The table. */
@Getter
private String table;
- // By default all samples are considered for training
- /**
- * The partition filter.
- */
+ /** The partition filter. */
@Getter
private String partitionFilter;
- /**
- * The feature columns.
- */
+
+ /** The feature columns. */
@Getter
- private List<Feature> featureColumns;
- /**
- * The label column.
- */
+ private List<String> featureColumns;
+
+ /** The label column. */
@Getter
- private Feature labelColumn;
- /**
- * The conf.
- */
+ private String labelColumn;
+
+ /** The conf. */
@Getter
private transient HiveConf conf;
- /**
- * The split training.
- */
- private boolean splitTraining;
- /**
- * The training fraction.
- */
- private double trainingFraction = 1.0;
-
- /**
- * New builder.
- *
- * @return the table training spec builder
- */
- public static TableTrainingSpecBuilder newBuilder() {
- return new TableTrainingSpecBuilder();
- }
-
- /**
- * Validate.
- *
- * @return true, if successful
- */
- boolean validate() {
- List<HCatFieldSchema> columns;
- try {
- HCatInputFormat.setInput(conf, database == null ? "default" : database, table, partitionFilter);
- HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
- columns = tableSchema.getFields();
- } catch (IOException exc) {
- LOG.error("Error getting table info " + toString(), exc);
- return false;
- }
- LOG.info(table + " columns " + columns.toString());
-
- boolean valid = false;
- if (columns != null && !columns.isEmpty()) {
- // Check labeled column
- List<String> columnNamesInTable = new ArrayList();
- List<String> columnNamesInFeatureList = new ArrayList();
- for (HCatFieldSchema col : columns) {
- columnNamesInTable.add(col.getName());
- }
-
- for (Feature feature : featureColumns) {
- columnNamesInFeatureList.add(feature.getDataColumn());
- }
-
- String labelColumnName = labelColumn.getDataColumn();
+ // By default all samples are considered for training
+ /** The split training. */
+ private boolean splitTraining;
- // Need at least one feature column and one label column
- valid = columnNamesInTable.contains(labelColumnName);
+ /** The training fraction. */
+ private double trainingFraction = 1.0;
- if (valid) {
- labelPos = columnNamesInTable.indexOf(labelColumnName);
+ /** The label pos. */
+ int labelPos;
- // Check feature columns
- if (featureColumns == null || featureColumns.isEmpty()) {
- // feature columns are not provided, so all columns except label column are feature columns
- featurePositions = new int[columnNamesInTable.size() - 1];
- int p = 0;
- for (int i = 0; i < columnNamesInTable.size(); i++) {
- if (i == labelPos) {
- continue;
- }
- featurePositions[p++] = i;
- }
+ /** The feature positions. */
+ int[] featurePositions;
- columnNamesInTable.remove(labelPos);
- featureColumns = new ArrayList<Feature>();
- for (String featureName : columnNamesInTable) {
- featureColumns.add(new Feature(featureName, null, null, featureName));
- }
- } else {
- // Feature columns were provided, verify all feature columns are present in the table
- valid = columnNamesInTable.containsAll(columnNamesInFeatureList);
- if (valid) {
- // Get feature positions
- featurePositions = new int[columnNamesInFeatureList.size()];
- for (int i = 0; i < columnNamesInFeatureList.size(); i++) {
- featurePositions[i] = columnNamesInTable.indexOf(featureColumns.get(i).getDataColumn());
- }
- }
- }
- numFeatures = featureColumns.size();
- }
- }
+ /** The num features. */
+ int numFeatures;
- return valid;
- }
+ /** The labeled rdd. */
+ transient JavaRDD<LabeledPoint> labeledRDD;
/**
- * Creates the rd ds.
+ * New builder.
*
- * @param sparkContext the spark context
- * @throws LensException the lens exception
+ * @return the table training spec builder
*/
- public void createRDDs(JavaSparkContext sparkContext) throws LensException {
- // Validate the spec
- if (!validate()) {
- throw new LensException("Table spec not valid: " + toString());
- }
-
- LOG.info("Creating RDDs with spec " + toString());
-
- // Get the RDD for table
- JavaPairRDD<WritableComparable, HCatRecord> tableRDD;
- try {
- tableRDD = HiveTableRDD.createHiveTableRDD(sparkContext, conf, database, table, partitionFilter);
- } catch (IOException e) {
- throw new LensException(e);
- }
-
- // Map into trainable RDD
- // TODO: Figure out a way to use custom value mappers
- FeatureValueMapper[] valueMappers = new FeatureValueMapper[numFeatures + 1];
- final DoubleValueMapper doubleMapper = new DoubleValueMapper();
- for (int i = 0; i < numFeatures; i++) {
- valueMappers[i] = doubleMapper;
- }
- valueMappers[numFeatures] = doubleMapper; //label mapper
- ColumnFeatureFunction trainPrepFunction = new ColumnFeatureFunction(featurePositions, valueMappers, labelPos,
- numFeatures, 0);
- labeledRDD = tableRDD.map(trainPrepFunction);
-
- if (splitTraining) {
- // We have to split the RDD between a training RDD and a testing RDD
- LOG.info("Splitting RDD for table " + database + "." + table + " with split fraction " + trainingFraction);
- JavaRDD<DataSample> sampledRDD = labeledRDD.map(new Function<LabeledPoint, DataSample>() {
- @Override
- public DataSample call(LabeledPoint v1) throws Exception {
- return new DataSample(v1);
- }
- });
-
- trainingRDD = sampledRDD.filter(new TrainingFilter(trainingFraction)).map(new GetLabeledPoint()).rdd();
- testingRDD = sampledRDD.filter(new TestingFilter(trainingFraction)).map(new GetLabeledPoint()).rdd();
- } else {
- LOG.info("Using same RDD for train and test");
- trainingRDD = labeledRDD.rdd();
- testingRDD = trainingRDD;
- }
- LOG.info("Generated RDDs");
+ public static TableTrainingSpecBuilder newBuilder() {
+ return new TableTrainingSpecBuilder();
}
/**
@@ -264,9 +118,7 @@ public class TableTrainingSpec implements Serializable {
*/
public static class TableTrainingSpecBuilder {
- /**
- * The spec.
- */
+ /** The spec. */
final TableTrainingSpec spec;
/**
@@ -326,7 +178,7 @@ public class TableTrainingSpec implements Serializable {
* @param labelColumn the label column
* @return the table training spec builder
*/
- public TableTrainingSpecBuilder labelColumn(Feature labelColumn) {
+ public TableTrainingSpecBuilder labelColumn(String labelColumn) {
spec.labelColumn = labelColumn;
return this;
}
@@ -337,7 +189,7 @@ public class TableTrainingSpec implements Serializable {
* @param featureColumns the feature columns
* @return the table training spec builder
*/
- public TableTrainingSpecBuilder featureColumns(List<Feature> featureColumns) {
+ public TableTrainingSpecBuilder featureColumns(List<String> featureColumns) {
spec.featureColumns = featureColumns;
return this;
}
@@ -371,14 +223,10 @@ public class TableTrainingSpec implements Serializable {
*/
public static class DataSample implements Serializable {
- /**
- * The labeled point.
- */
+ /** The labeled point. */
private final LabeledPoint labeledPoint;
- /**
- * The sample.
- */
+ /** The sample. */
private final double sample;
/**
@@ -397,9 +245,7 @@ public class TableTrainingSpec implements Serializable {
*/
public static class TrainingFilter implements Function<DataSample, Boolean> {
- /**
- * The training fraction.
- */
+ /** The training fraction. */
private double trainingFraction;
/**
@@ -427,9 +273,7 @@ public class TableTrainingSpec implements Serializable {
*/
public static class TestingFilter implements Function<DataSample, Boolean> {
- /**
- * The training fraction.
- */
+ /** The training fraction. */
private double trainingFraction;
/**
@@ -468,4 +312,122 @@ public class TableTrainingSpec implements Serializable {
}
}
+ /**
+ * Validate.
+ *
+ * @return true, if successful
+ */
+ boolean validate() {
+ List<HCatFieldSchema> columns;
+ try {
+ HCatInputFormat.setInput(conf, database == null ? "default" : database, table, partitionFilter);
+ HCatSchema tableSchema = HCatInputFormat.getTableSchema(conf);
+ columns = tableSchema.getFields();
+ } catch (IOException exc) {
+ LOG.error("Error getting table info " + toString(), exc);
+ return false;
+ }
+
+ LOG.info(table + " columns " + columns.toString());
+
+ boolean valid = false;
+ if (columns != null && !columns.isEmpty()) {
+ // Check labeled column
+ List<String> columnNames = new ArrayList<String>();
+ for (HCatFieldSchema col : columns) {
+ columnNames.add(col.getName());
+ }
+
+ // Need at least one feature column and one label column
+ valid = columnNames.contains(labelColumn) && columnNames.size() > 1;
+
+ if (valid) {
+ labelPos = columnNames.indexOf(labelColumn);
+
+ // Check feature columns
+ if (featureColumns == null || featureColumns.isEmpty()) {
+ // feature columns are not provided, so all columns except label column are feature columns
+ featurePositions = new int[columnNames.size() - 1];
+ int p = 0;
+ for (int i = 0; i < columnNames.size(); i++) {
+ if (i == labelPos) {
+ continue;
+ }
+ featurePositions[p++] = i;
+ }
+
+ columnNames.remove(labelPos);
+ featureColumns = columnNames;
+ } else {
+ // Feature columns were provided, verify all feature columns are present in the table
+ valid = columnNames.containsAll(featureColumns);
+ if (valid) {
+ // Get feature positions
+ featurePositions = new int[featureColumns.size()];
+ for (int i = 0; i < featureColumns.size(); i++) {
+ featurePositions[i] = columnNames.indexOf(featureColumns.get(i));
+ }
+ }
+ }
+ numFeatures = featureColumns.size();
+ }
+ }
+
+ return valid;
+ }
+
+ /**
+ * Creates the rd ds.
+ *
+ * @param sparkContext the spark context
+ * @throws LensException the lens exception
+ */
+ public void createRDDs(JavaSparkContext sparkContext) throws LensException {
+ // Validate the spec
+ if (!validate()) {
+ throw new LensException("Table spec not valid: " + toString());
+ }
+
+ LOG.info("Creating RDDs with spec " + toString());
+
+ // Get the RDD for table
+ JavaPairRDD<WritableComparable, HCatRecord> tableRDD;
+ try {
+ tableRDD = HiveTableRDD.createHiveTableRDD(sparkContext, conf, database, table, partitionFilter);
+ } catch (IOException e) {
+ throw new LensException(e);
+ }
+
+ // Map into trainable RDD
+ // TODO: Figure out a way to use custom value mappers
+ FeatureValueMapper[] valueMappers = new FeatureValueMapper[numFeatures];
+ final DoubleValueMapper doubleMapper = new DoubleValueMapper();
+ for (int i = 0; i < numFeatures; i++) {
+ valueMappers[i] = doubleMapper;
+ }
+
+ ColumnFeatureFunction trainPrepFunction = new ColumnFeatureFunction(featurePositions, valueMappers, labelPos,
+ numFeatures, 0);
+ labeledRDD = tableRDD.map(trainPrepFunction);
+
+ if (splitTraining) {
+ // We have to split the RDD between a training RDD and a testing RDD
+ LOG.info("Splitting RDD for table " + database + "." + table + " with split fraction " + trainingFraction);
+ JavaRDD<DataSample> sampledRDD = labeledRDD.map(new Function<LabeledPoint, DataSample>() {
+ @Override
+ public DataSample call(LabeledPoint v1) throws Exception {
+ return new DataSample(v1);
+ }
+ });
+
+ trainingRDD = sampledRDD.filter(new TrainingFilter(trainingFraction)).map(new GetLabeledPoint()).rdd();
+ testingRDD = sampledRDD.filter(new TestingFilter(trainingFraction)).map(new GetLabeledPoint()).rdd();
+ } else {
+ LOG.info("Using same RDD for train and test");
+ trainingRDD = labeledRDD.rdd();
+ testingRDD = trainingRDD;
+ }
+ LOG.info("Generated RDDs");
+ }
+
}
[39/50] [abbrv] incubator-lens git commit: LENS-251 : Add support for
continuous update period
Posted by sh...@apache.org.
LENS-251 : Add support for continuous update period
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/75001a1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/75001a1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/75001a1b
Branch: refs/heads/LENS-581
Commit: 75001a1be17b8a649c1fc4df34583e5bf9774008
Parents: 651a9b6
Author: Sharad Agarwal <sh...@flipkarts-MacBook-Pro.local>
Authored: Mon Jul 27 16:06:08 2015 +0530
Committer: Sharad Agarwal <sh...@flipkarts-MacBook-Pro.local>
Committed: Mon Jul 27 16:06:08 2015 +0530
----------------------------------------------------------------------
lens-api/src/main/resources/cube-0.1.xsd | 1 +
.../apache/lens/cube/metadata/UpdatePeriod.java | 5 +-
.../org/apache/lens/cube/parse/DateUtil.java | 3 ++
.../lens/cube/parse/StorageTableResolver.java | 13 +++++
.../apache/lens/cube/parse/CubeTestSetup.java | 51 +++++++++++++++++-
.../lens/cube/parse/TestCubeRewriter.java | 50 ++++++++++++++++-
.../cube/parse/TestDenormalizationResolver.java | 2 +
.../lens/cube/parse/TestTimeRangeResolver.java | 4 +-
.../parse/TestTimeRangeWriterWithQuery.java | 45 +++++++++++-----
.../lens/examples/PopulateSampleMetastore.java | 33 +++++++++++-
.../apache/lens/examples/SampleMetastore.java | 1 +
.../src/main/resources/cube-queries.sql | 1 +
.../src/main/resources/db-storage-schema.sql | 4 ++
.../resources/sales-aggr-continuous-fact.xml | 57 ++++++++++++++++++++
14 files changed, 249 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-api/src/main/resources/cube-0.1.xsd
----------------------------------------------------------------------
diff --git a/lens-api/src/main/resources/cube-0.1.xsd b/lens-api/src/main/resources/cube-0.1.xsd
index ab7d3ff..7ebb5a1 100644
--- a/lens-api/src/main/resources/cube-0.1.xsd
+++ b/lens-api/src/main/resources/cube-0.1.xsd
@@ -727,6 +727,7 @@
</xs:documentation>
</xs:annotation>
<xs:restriction base="xs:string">
+ <xs:enumeration value="CONTINUOUS"/>
<xs:enumeration value="SECONDLY"/>
<xs:enumeration value="MINUTELY"/>
<xs:enumeration value="HOURLY"/>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
index 253323b..b4dff46 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/UpdatePeriod.java
@@ -39,7 +39,8 @@ public enum UpdatePeriod implements Named {
WEEKLY(WEEK_OF_YEAR, 7 * DAILY.weight(), 0.7f, "yyyy-'W'ww"),
MONTHLY(MONTH, 30 * DAILY.weight(), 0.6f, "yyyy-MM"),
QUARTERLY(MONTH, 3 * MONTHLY.weight(), 0.55f, "yyyy-MM"),
- YEARLY(YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy");
+ YEARLY(YEAR, 12 * MONTHLY.weight(), 0.52f, "yyyy"),
+ CONTINUOUS(Calendar.SECOND, 1, 1.5f, "yyyy-MM-dd-HH-mm-ss");
public static final long MIN_INTERVAL = values()[0].weight();
private final int calendarField;
@@ -179,6 +180,8 @@ public enum UpdatePeriod implements Named {
public DateFormat format() {
switch (this) {
+ case CONTINUOUS:
+ return getSecondlyFormat();
case SECONDLY:
return getSecondlyFormat();
case MINUTELY:
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
index 9e295d3..5c77548 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/DateUtil.java
@@ -224,6 +224,7 @@ public final class DateUtil {
break;
}
case SECONDLY:
+ case CONTINUOUS:
if (cal.get(Calendar.MILLISECOND) != 0) {
hasFraction = true;
}
@@ -258,6 +259,7 @@ public final class DateUtil {
case MINUTELY:
cal.set(Calendar.SECOND, 0);
case SECONDLY:
+ case CONTINUOUS:
cal.set(Calendar.MILLISECOND, 0);
break;
case WEEKLY:
@@ -381,6 +383,7 @@ public final class DateUtil {
static CoveringInfo getCoveringInfo(Date from, Date to, UpdatePeriod interval) {
switch (interval) {
case SECONDLY:
+ case CONTINUOUS:
return getMilliSecondCoveringInfo(from, to, 1000);
case MINUTELY:
return getMilliSecondCoveringInfo(from, to, 1000 * 60);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
index 7bfbd86..5fb2441 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageTableResolver.java
@@ -546,6 +546,19 @@ class StorageTableResolver implements ContextRewriter {
Set<String> storageTbls = new LinkedHashSet<String>();
storageTbls.addAll(validStorageMap.get(fact).get(interval));
+ if (interval == UpdatePeriod.CONTINUOUS && rangeWriter.getClass().equals(BetweenTimeRangeWriter.class)) {
+ for (String storageTbl : storageTbls) {
+ FactPartition part = new FactPartition(partCol, fromDate, interval, null, partWhereClauseFormat);
+ partitions.add(part);
+ part.getStorageTables().add(storageTbl);
+ part = new FactPartition(partCol, toDate, interval, null, partWhereClauseFormat);
+ partitions.add(part);
+ part.getStorageTables().add(storageTbl);
+ log.info("Added continuous fact partition for storage table " + storageTbl);
+ }
+ return true;
+ }
+
Iterator<String> it = storageTbls.iterator();
while (it.hasNext()) {
String storageTableName = it.next();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
index e0338f5..1d4e7dd 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/CubeTestSetup.java
@@ -125,6 +125,7 @@ public class CubeTestSetup {
public static final String TWO_DAYS_RANGE_BEFORE_4_DAYS;
private static boolean zerothHour;
+ private static String c0 = "C0";
private static String c1 = "C1";
private static String c2 = "C2";
private static String c3 = "C3";
@@ -592,6 +593,8 @@ public class CubeTestSetup {
"No aggregateMsr", null, null, null));
cubeMeasures.add(new ColumnMeasure(new FieldSchema("newmeasure", "bigint", "measure available from now"),
"New measure", null, null, null, NOW, null, 100.0));
+ cubeMeasures.add(new ColumnMeasure(new FieldSchema("msr15", "int", "first measure"), "Measure15", null, "SUM",
+ "RS"));
cubeDimensions = new HashSet<CubeDimAttribute>();
cubeDimensions.add(new BaseDimAttribute(new FieldSchema("d_time", "timestamp", "d time")));
@@ -1045,11 +1048,45 @@ public class CubeTestSetup {
}
+ private void createCubeContinuousFact(CubeMetastoreClient client) throws Exception{
+ // create continuous raw fact only with extra measures
+ String factName = "testFact_CONTINUOUS";
+ List<FieldSchema> factColumns = new ArrayList<FieldSchema>();
+ factColumns.add(new FieldSchema("msr11", "double", "third measure"));
+ factColumns.add(new FieldSchema("msr15", "int", "fifteenth measure"));
+
+ // add dimensions of the cube
+ factColumns.add(new FieldSchema("d_time", "timestamp", "event time"));
+ factColumns.add(new FieldSchema("processing_time", "timestamp", "processing time"));
+ factColumns.add(new FieldSchema("dim1", "string", "base dim"));
+ factColumns.add(new FieldSchema("dim11", "string", "base dim"));
+ factColumns.add(new FieldSchema("dim12", "string", "base dim"));
+
+ Map<String, Set<UpdatePeriod>> storageAggregatePeriods = new HashMap<String, Set<UpdatePeriod>>();
+ Set<UpdatePeriod> updates = new HashSet<UpdatePeriod>();
+ updates.add(CONTINUOUS);
+ storageAggregatePeriods.put(c0, updates);
+
+ StorageTableDesc s0 = new StorageTableDesc();
+ s0.setInputFormat(TextInputFormat.class.getCanonicalName());
+ s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+
+ Map<String, StorageTableDesc> storageTables = new HashMap<String, StorageTableDesc>();
+ storageTables.put(c0, s0);
+ Map<String, String> properties = Maps.newHashMap(factValidityProperties);
+ properties.put(MetastoreConstants.FACT_ABSOLUTE_START_TIME, DateUtil.relativeToAbsolute("now.day - 3 days"));
+
+ client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 100L, properties,
+ storageTables);
+ }
+
private void createCubeFact(CubeMetastoreClient client) throws Exception {
String factName = "testFact";
List<FieldSchema> factColumns = new ArrayList<FieldSchema>(cubeMeasures.size());
for (CubeMeasure measure : cubeMeasures) {
- factColumns.add(measure.getColumn());
+ if (!measure.getColumn().getName().equals("msr15")) { //do not add msr15
+ factColumns.add(measure.getColumn());
+ }
}
factColumns.add(new FieldSchema("msr5", "double", "msr5"));
@@ -1100,9 +1137,19 @@ public class CubeTestSetup {
storageTables.put(c4, s2);
storageTables.put(c2, s1);
storageTables.put(c3, s1);
+
+ //add storage with continuous update period
+ updates.add(CONTINUOUS);
+ storageAggregatePeriods.put(c0, updates);
+ StorageTableDesc s0 = new StorageTableDesc();
+ s0.setInputFormat(TextInputFormat.class.getCanonicalName());
+ s0.setOutputFormat(HiveIgnoreKeyTextOutputFormat.class.getCanonicalName());
+ storageTables.put(c0, s0);
+
// create cube fact
client.createCubeFactTable(TEST_CUBE_NAME, factName, factColumns, storageAggregatePeriods, 5L,
factValidityProperties, storageTables);
+
CubeFactTable fact = client.getFactTable(factName);
Table table = client.getTable(MetastoreUtil.getStorageTableName(fact.getName(),
@@ -2076,6 +2123,7 @@ public class CubeTestSetup {
Hive.get(conf).createDatabase(database);
SessionState.get().setCurrentDatabase(dbName);
CubeMetastoreClient client = CubeMetastoreClient.getInstance(conf);
+ client.createStorage(new HDFSStorage(c0));
client.createStorage(new HDFSStorage(c1));
client.createStorage(new HDFSStorage(c2));
client.createStorage(new HDFSStorage(c3));
@@ -2084,6 +2132,7 @@ public class CubeTestSetup {
createCube(client);
createBaseAndDerivedCubes(client);
createCubeFact(client);
+ createCubeContinuousFact(client);
createCubeCheapFact(client);
// commenting this as the week date format throws IllegalPatternException
// createCubeFactWeekly(client);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
index e04d04c..434313e 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestCubeRewriter.java
@@ -24,6 +24,8 @@ import static org.apache.lens.cube.parse.CubeTestSetup.*;
import static org.testng.Assert.*;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.*;
import org.apache.lens.cube.metadata.*;
@@ -56,7 +58,7 @@ public class TestCubeRewriter extends TestQueryRewrite {
@BeforeTest
public void setupDriver() throws Exception {
conf = new Configuration();
- conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C1,C2");
+ conf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C0,C1,C2");
conf.setBoolean(CubeQueryConfUtil.DISABLE_AUTO_JOINS, true);
conf.setBoolean(CubeQueryConfUtil.ENABLE_SELECT_TO_GROUPBY, true);
conf.setBoolean(CubeQueryConfUtil.ENABLE_GROUP_BY_TO_SELECT, true);
@@ -76,6 +78,50 @@ public class TestCubeRewriter extends TestQueryRewrite {
}
@Test
+ public void testQueryWithContinuousUpdatePeriod() throws Exception {
+ Configuration conf = getConf();
+ conf.setClass(CubeQueryConfUtil.TIME_RANGE_WRITER_CLASS, BetweenTimeRangeWriter.class, TimeRangeWriter.class);
+
+ DateFormat qFmt = new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss");
+ Calendar qCal = Calendar.getInstance();
+ Date toDate = qCal.getTime();
+ String qTo = qFmt.format(toDate);
+ qCal.setTime(TWODAYS_BACK);
+ Date from2DaysBackDate = qCal.getTime();
+ String qFrom = qFmt.format(from2DaysBackDate);
+
+ CubeQueryContext rewrittenQuery = rewriteCtx("select SUM(msr15) from testCube where"
+ + " time_range_in(d_time, '"+ qFrom + "', '" + qTo + "')", conf);
+
+ DateFormat fmt = UpdatePeriod.CONTINUOUS.format();
+ String to = fmt.format(toDate);
+ String from = fmt.format(from2DaysBackDate);
+
+ String expected = "select SUM((testCube.msr15)) from TestQueryRewrite.c0_testFact_CONTINUOUS testcube"
+ + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))";
+ System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL());
+ System.out.println("expected " + expected);
+ compareQueries(expected, rewrittenQuery.toHQL());
+
+ //test with msr2 on different fact
+ rewrittenQuery = rewriteCtx("select SUM(msr2) from testCube where" + " time_range_in(d_time, '"
+ + qFrom + "', '" + qTo + "')", conf);
+ expected = "select SUM((testCube.msr2)) from TestQueryRewrite.c0_testFact testcube"
+ + " WHERE ((( testcube . dt ) between '" + from + "' and '" + to + "' ))";
+ System.out.println("rewrittenQuery.toHQL() " + rewrittenQuery.toHQL());
+ System.out.println("expected " + expected);
+ compareQueries(expected, rewrittenQuery.toHQL());
+
+ //from date 4 days back
+ qCal.setTime(BEFORE_4_DAYS_START);
+ Date from4DaysBackDate = qCal.getTime();
+ String qFrom4DaysBackDate = qFmt.format(from4DaysBackDate);
+ SemanticException th = getSemanticExceptionInRewrite("select SUM(msr15) from testCube where"
+ + " time_range_in(d_time, '"+ qFrom4DaysBackDate + "', '" + qTo + "')", getConf());
+ assertEquals(th.getCanonicalErrorMsg().getErrorCode(), ErrorMsg.NO_CANDIDATE_FACT_AVAILABLE.getErrorCode());
+ }
+
+ @Test
public void testCandidateTables() throws Exception {
SemanticException th = getSemanticExceptionInRewrite(
"select dim12, SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, getConf());
@@ -255,6 +301,8 @@ public class TestCubeRewriter extends TestQueryRewrite {
System.err.println("__FAILED__ " + method + "\n\tExpected: " + expected + "\n\t---------\n\tActual: " + actual);
}
+ log.info("expectedTrimmed " + expectedTrimmed);
+ log.info("actualTrimmed " + actualTrimmed);
assertTrue(expectedTrimmed.equalsIgnoreCase(actualTrimmed));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
index 901b95e..7d9183c 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestDenormalizationResolver.java
@@ -157,6 +157,8 @@ public class TestDenormalizationResolver extends TestQueryRewrite {
{
put("summary2,testfact2_raw,summary3",
Arrays.asList(new CandidateTablePruneCause(CandidateTablePruneCode.INVALID_DENORM_TABLE)));
+ put("testfact_continuous",
+ Arrays.asList(CandidateTablePruneCause.columnNotFound("msr2", "msr3")));
put("summary4", Arrays.asList(CandidateTablePruneCause.noCandidateStorages(
new HashMap<String, CandidateTablePruneCause.SkipStorageCause>() {
{
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
index f2d7990..a30a114 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeResolver.java
@@ -62,8 +62,8 @@ public class TestTimeRangeResolver extends TestQueryRewrite {
getSemanticExceptionInRewrite("cube select msr2 from " + cubeName + " where " + LAST_YEAR_RANGE,
getConf());
PruneCauses.BriefAndDetailedError causes = extractPruneCause(e);
- assertTrue(causes.getBrief().contains("No facts available for all of these time ranges:"));
- assertEquals(causes.getDetails().size(), 1);
+ assertTrue(causes.getBrief().contains("Columns [msr2] are not present in any table"));
+ assertEquals(causes.getDetails().size(), 2);
assertEquals(causes.getDetails().values().iterator().next().size(), 1);
assertEquals(causes.getDetails().values().iterator().next().iterator().next().getCause(),
FACT_NOT_AVAILABLE_IN_RANGE);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
index 0e22ce2..00d92b5 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/parse/TestTimeRangeWriterWithQuery.java
@@ -21,6 +21,8 @@ package org.apache.lens.cube.parse;
import static org.apache.lens.cube.parse.CubeTestSetup.*;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
@@ -89,30 +91,45 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
}
// hourly partitions for two days
conf.setBoolean(CubeQueryConfUtil.FAIL_QUERY_ON_PARTIAL_DATA, true);
- String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf);
+
+ DateFormat qFmt = new SimpleDateFormat("yyyy-MM-dd-HH:mm:ss");
+ Calendar qCal = Calendar.getInstance();
+ Date toDate = qCal.getTime();
+ String qTo = qFmt.format(toDate);
+ qCal.setTime(TWODAYS_BACK);
+ Date from2DaysBackDate = qCal.getTime();
+ String qFrom = qFmt.format(from2DaysBackDate);
+ String twoDaysInRangeClause = " time_range_in(d_time, '"+ qFrom + "', '" + qTo + "')";
+
+ String hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + twoDaysInRangeClause, conf);
Map<String, String> whereClauses = new HashMap<String, String>();
whereClauses.put(
- CubeTestSetup.getDbName() + "c1_testfact2",
- TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", CubeTestSetup.TWODAYS_BACK,
- getOneLess(CubeTestSetup.NOW, UpdatePeriod.HOURLY.calendarField()), UpdatePeriod.HOURLY.format()));
+ CubeTestSetup.getDbName() + "c1_testfact",
+ TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", from2DaysBackDate, toDate,
+ UpdatePeriod.CONTINUOUS.format()));
String expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses);
System.out.println("HQL:" + hqlQuery);
TestCubeRewriter.compareQueries(expected, hqlQuery);
// multiple range query
+ //from date 4 days back
+ qCal.setTime(BEFORE_4_DAYS_START);
+ Date from4DaysBackDate = qCal.getTime();
+ String qFrom4Days = qFmt.format(from4DaysBackDate);
+ String fourDaysInRangeClause = " time_range_in(d_time, '"+ qFrom4Days + "', '" + qTo + "')";
+
hqlQuery =
- rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE + " OR "
- + CubeTestSetup.TWO_DAYS_RANGE_BEFORE_4_DAYS, conf);
+ rewrite("select SUM(msr2) from testCube" + " where " + twoDaysInRangeClause + " OR "
+ + fourDaysInRangeClause, conf);
whereClauses = new HashMap<String, String>();
whereClauses.put(
- CubeTestSetup.getDbName() + "c1_testfact2",
- TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", CubeTestSetup.TWODAYS_BACK,
- getOneLess(CubeTestSetup.NOW, UpdatePeriod.HOURLY.calendarField()), UpdatePeriod.HOURLY.format())
+ CubeTestSetup.getDbName() + "c1_testfact",
+ TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", from2DaysBackDate, toDate,
+ UpdatePeriod.CONTINUOUS.format())
+ " OR"
- + TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", CubeTestSetup.BEFORE_4_DAYS_START,
- getOneLess(CubeTestSetup.BEFORE_4_DAYS_END, UpdatePeriod.HOURLY.calendarField()),
- UpdatePeriod.HOURLY.format()));
+ + TestBetweenTimeRangeWriter.getBetweenClause(cubeName, "dt", from4DaysBackDate, toDate,
+ UpdatePeriod.CONTINUOUS.format()));
expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses);
System.out.println("HQL:" + hqlQuery);
TestCubeRewriter.compareQueries(expected, hqlQuery);
@@ -121,9 +138,9 @@ public class TestTimeRangeWriterWithQuery extends TestQueryRewrite {
conf.set(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT, "yyyy-MM-dd HH:mm:ss");
hqlQuery = rewrite("select SUM(msr2) from testCube" + " where " + TWO_DAYS_RANGE, conf);
whereClauses = new HashMap<String, String>();
- whereClauses.put(CubeTestSetup.getDbName() + "c1_testfact2", TestBetweenTimeRangeWriter.getBetweenClause(cubeName,
+ whereClauses.put(CubeTestSetup.getDbName() + "c1_testfact", TestBetweenTimeRangeWriter.getBetweenClause(cubeName,
"dt", getUptoHour(CubeTestSetup.TWODAYS_BACK),
- getUptoHour(getOneLess(CubeTestSetup.NOW, UpdatePeriod.HOURLY.calendarField())), TestTimeRangeWriter.DB_FORMAT));
+ getUptoHour(CubeTestSetup.NOW), TestTimeRangeWriter.DB_FORMAT));
expected = getExpectedQuery(cubeName, "select sum(testcube.msr2) FROM ", null, null, whereClauses);
System.out.println("HQL:" + hqlQuery);
TestCubeRewriter.compareQueries(expected, hqlQuery);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
index 7ecabfa..5ba0ea6 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
@@ -19,6 +19,11 @@
package org.apache.lens.examples;
import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import javax.xml.bind.JAXBException;
@@ -64,7 +69,7 @@ public class PopulateSampleMetastore {
}
}
- public void populateAll() throws JAXBException, IOException {
+ public void populateAll() throws Exception {
populateDimTables();
populateFactTables();
}
@@ -94,6 +99,29 @@ public class PopulateSampleMetastore {
}
}
+ private void createContinuousFactData() throws Exception {
+ Class.forName("org.hsqldb.jdbcDriver");
+ Connection con = DriverManager.getConnection(
+ "jdbc:hsqldb:/tmp/db-storage.db", "SA", "");
+
+ con.setAutoCommit(true);
+ Statement statement = con.createStatement();
+ try {
+ Date date = new Date(System.currentTimeMillis());
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ String nowTime = format.format(date);
+ statement.execute("INSERT INTO mydb_sales_aggr_continuous_fact (order_time, delivery_time, customer_id, "
+ + "product_id, promotion_id, customer_city_id, production_city_id, delivery_city_id, unit_sales, "
+ + "store_sales, store_cost, max_line_item_price, max_line_item_discount) values "
+ + "('" + nowTime + "','" + nowTime + "',2,2,1,2,2,2,1,8,2,10,2)");
+
+ } finally {
+ statement.close();
+ con.close();
+ }
+
+ }
+
private void createDimTablePartitions(String fileName, String dimTable, String storage)
throws JAXBException, IOException {
XPartitionList partitionList = (XPartitionList) SampleMetastore.readFromXML(fileName);
@@ -144,7 +172,7 @@ public class PopulateSampleMetastore {
}
}
- public void populateFactTables() throws JAXBException, IOException {
+ public void populateFactTables() throws Exception {
createFactPartition("fact1-local-part1.xml", "fact1", "local");
createFactPartition("fact1-local-part2.xml", "fact1", "local");
createFactPartition("fact1-local-part3.xml", "fact1", "local");
@@ -162,5 +190,6 @@ public class PopulateSampleMetastore {
createFactPartitions("sales-aggr-fact2-local-parts.xml", "sales_aggr_fact2", "local");
createFactPartitions("sales-aggr-fact1-mydb-parts.xml", "sales_aggr_fact1", "mydb");
createFactPartitions("sales-aggr-fact2-mydb-parts.xml", "sales_aggr_fact2", "mydb");
+ createContinuousFactData();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
index dee43ef..4699dbb 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
@@ -159,6 +159,7 @@ public class SampleMetastore {
createFact("sales-raw-fact.xml");
createFact("sales-aggr-fact1.xml");
createFact("sales-aggr-fact2.xml");
+ createFact("sales-aggr-continuous-fact.xml");
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-examples/src/main/resources/cube-queries.sql
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/cube-queries.sql b/lens-examples/src/main/resources/cube-queries.sql
index 7668f54..13c5204 100644
--- a/lens-examples/src/main/resources/cube-queries.sql
+++ b/lens-examples/src/main/resources/cube-queries.sql
@@ -103,5 +103,6 @@ cube select product_details.category, store_sales from sales where time_range_in
cube select product_details.category, store_sales from sales where time_range_in(order_time, '2015-04-12-00', '2015-04-13-00')
cube select product_details.color, store_sales from sales where time_range_in(delivery_time, '2015-04-11-00', '2015-04-13-00') and product_details.category='Stationary'
cube select product_details.color, store_sales from sales where time_range_in(delivery_time, '2015-04-12-00', '2015-04-13-00') and product_details.category='Stationary'
+cube select store_sales from sales where time_range_in(order_time, 'now.second-2 days', 'now.second')
-- The following query sees that ot part col is not present and falls back on querying on dt part col
cube select customer_city_name, store_cost from sales where time_range_in(order_time, '2015-04-13-03', '2015-04-13-04') --
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-examples/src/main/resources/db-storage-schema.sql
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/db-storage-schema.sql b/lens-examples/src/main/resources/db-storage-schema.sql
index bbb62e8..0aceeab 100644
--- a/lens-examples/src/main/resources/db-storage-schema.sql
+++ b/lens-examples/src/main/resources/db-storage-schema.sql
@@ -44,6 +44,10 @@ CREATE TABLE mydb_sales_aggr_fact1 (order_time timestamp, delivery_time timestam
INSERT INTO mydb_sales_aggr_fact1 (order_time, delivery_time, customer_id, product_id, promotion_id, customer_city_id, production_city_id, delivery_city_id, unit_sales, store_sales, store_cost, max_line_item_price, max_line_item_discount) values ('2015-04-12 00:00:00','2015-04-12 00:00:00',1,1,1,1,1,1,1,5,0,5,0)
INSERT INTO mydb_sales_aggr_fact1 (order_time, delivery_time, customer_id, product_id, promotion_id, customer_city_id, production_city_id, delivery_city_id, unit_sales, store_sales, store_cost, max_line_item_price, max_line_item_discount) values ('2015-04-12 00:00:00','2015-04-12 00:00:00',2,2,1,2,2,2,1,8,2,10,2)
+DROP TABLE IF EXISTS mydb_sales_aggr_continuous_fact
+CREATE TABLE mydb_sales_aggr_continuous_fact (order_time timestamp, delivery_time timestamp, customer_id integer, product_id integer, promotion_id integer, customer_city_id integer, production_city_id integer, delivery_city_id integer, unit_sales double, store_sales double, store_cost double, max_line_item_price float, max_line_item_discount float)
+
+
DROP TABLE IF EXISTS mydb_product_db_table
CREATE TABLE mydb_product_db_table (id integer, SKU_number integer, description varchar(255), color varchar(50), category varchar(255), weight float, manufacturer varchar(255))
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/75001a1b/lens-examples/src/main/resources/sales-aggr-continuous-fact.xml
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/resources/sales-aggr-continuous-fact.xml b/lens-examples/src/main/resources/sales-aggr-continuous-fact.xml
new file mode 100644
index 0000000..781a615
--- /dev/null
+++ b/lens-examples/src/main/resources/sales-aggr-continuous-fact.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<x_fact_table cube_name="sales" name="sales_aggr_continuous_fact" weight="50.0" xmlns="uri:lens:cube:0.1"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+ <columns>
+ <column comment="" name="order_time" type="TIMESTAMP"/>
+ <column comment="" name="delivery_time" type="TIMESTAMP"/>
+ <column comment="" name="customer_id" type="INT"/>
+ <column comment="" name="product_id" type="INT"/>
+ <column comment="" name="promotion_id" type="INT"/>
+ <column comment="" name="customer_city_id" type="INT"/>
+ <column comment="" name="production_city_id" type="INT"/>
+ <column comment="" name="delivery_city_id" type="INT"/>
+ <column comment="" name="unit_sales" type="BIGINT"/>
+ <column comment="" name="store_sales" type="DOUBLE"/>
+ <column comment="" name="store_cost" type="DOUBLE"/>
+ <column comment="" name="max_line_item_price" type="FLOAT"/>
+ <column comment="" name="max_line_item_discount" type="FLOAT"/>
+ </columns>
+ <properties>
+ <property name="cube.fact.is.aggregated" value="true"/>
+ <property name="cube.fact.relative.start.time" value="now.second - 2 days"/>
+ </properties>
+ <storage_tables>
+ <storage_table>
+ <update_periods>
+ <update_period>CONTINUOUS</update_period>
+ </update_periods>
+ <storage_name>mydb</storage_name>
+ <table_desc external="true" field_delimiter="," table_location="/tmp/db-storage.db"
+ storage_handler_name="org.apache.lens.storage.db.DBStorageHandler">
+ <table_parameters>
+ <property name="lens.metastore.native.db.name" value="default"/>
+ </table_parameters>
+ </table_desc>
+ </storage_table>
+ </storage_tables>
+</x_fact_table>
[48/50] [abbrv] incubator-lens git commit: Revert "Lens-581 :
Rafactored all API to new api (Vikas Singh via sharad)"
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeAlgo.java
index 4dd9f50..7810a9a 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeAlgo.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeAlgo.java
@@ -20,9 +20,10 @@ package org.apache.lens.ml.algo.spark.dt;
import java.util.Map;
+import org.apache.lens.ml.algo.api.AlgoParam;
+import org.apache.lens.ml.algo.api.Algorithm;
import org.apache.lens.ml.algo.spark.BaseSparkAlgo;
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.AlgoParam;
import org.apache.lens.server.api.error.LensException;
import org.apache.spark.mllib.regression.LabeledPoint;
@@ -40,27 +41,19 @@ import scala.Enumeration;
/**
* The Class DecisionTreeAlgo.
*/
+@Algorithm(name = "spark_decision_tree", description = "Spark Decision Tree classifier algo")
public class DecisionTreeAlgo extends BaseSparkAlgo {
- static final String DESCRIPTION = "Spark decision tree algo";
- static final String NAME = "spark_decision_tree";
-
- /**
- * The algo.
- */
+ /** The algo. */
@AlgoParam(name = "algo", help = "Decision tree algorithm. Allowed values are 'classification' and 'regression'")
private Enumeration.Value algo;
- /**
- * The decision tree impurity.
- */
+ /** The decision tree impurity. */
@AlgoParam(name = "impurity", help = "Impurity measure used by the decision tree. "
+ "Allowed values are 'gini', 'entropy' and 'variance'")
private Impurity decisionTreeImpurity;
- /**
- * The max depth.
- */
+ /** The max depth. */
@AlgoParam(name = "maxDepth", help = "Max depth of the decision tree. Integer values expected.",
defaultValue = "100")
private int maxDepth;
@@ -75,10 +68,6 @@ public class DecisionTreeAlgo extends BaseSparkAlgo {
super(name, description);
}
- public DecisionTreeAlgo() {
- super(NAME, DESCRIPTION);
- }
-
/*
* (non-Javadoc)
*
@@ -111,10 +100,9 @@ public class DecisionTreeAlgo extends BaseSparkAlgo {
* @see org.apache.lens.ml.spark.algos.BaseSparkAlgo#trainInternal(java.lang.String, org.apache.spark.rdd.RDD)
*/
@Override
- protected BaseSparkClassificationModel trainInternal(RDD<LabeledPoint> trainingRDD)
+ protected BaseSparkClassificationModel trainInternal(String modelId, RDD<LabeledPoint> trainingRDD)
throws LensException {
DecisionTreeModel model = DecisionTree$.MODULE$.train(trainingRDD, algo, decisionTreeImpurity, maxDepth);
- return new DecisionTreeClassificationModel(features, new SparkDecisionTreeModel(model));
+ return new DecisionTreeClassificationModel(modelId, new SparkDecisionTreeModel(model));
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeClassificationModel.java
index 2797f9a..27c32f4 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeClassificationModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/DecisionTreeClassificationModel.java
@@ -18,10 +18,7 @@
*/
package org.apache.lens.ml.algo.spark.dt;
-import java.util.List;
-
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.Feature;
/**
* The Class DecisionTreeClassificationModel.
@@ -29,10 +26,12 @@ import org.apache.lens.ml.api.Feature;
public class DecisionTreeClassificationModel extends BaseSparkClassificationModel<SparkDecisionTreeModel> {
/**
- * @param featureList
- * @param model
+ * Instantiates a new decision tree classification model.
+ *
+ * @param modelId the model id
+ * @param model the model
*/
- public DecisionTreeClassificationModel(List<Feature> featureList, SparkDecisionTreeModel model) {
- super(featureList, model);
+ public DecisionTreeClassificationModel(String modelId, SparkDecisionTreeModel model) {
+ super(modelId, model);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/SparkDecisionTreeModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/SparkDecisionTreeModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/SparkDecisionTreeModel.java
index c3082c9..e561a8d 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/SparkDecisionTreeModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/dt/SparkDecisionTreeModel.java
@@ -31,9 +31,7 @@ import org.apache.spark.rdd.RDD;
*/
public class SparkDecisionTreeModel implements ClassificationModel {
- /**
- * The model.
- */
+ /** The model. */
private final DecisionTreeModel model;
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansAlgo.java
index d29440c..be9af18 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansAlgo.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansAlgo.java
@@ -18,18 +18,15 @@
*/
package org.apache.lens.ml.algo.spark.kmeans;
-import java.util.ArrayList;
import java.util.List;
import org.apache.lens.api.LensConf;
+import org.apache.lens.ml.algo.api.AlgoParam;
import org.apache.lens.ml.algo.api.Algorithm;
-import org.apache.lens.ml.algo.api.TrainedModel;
+import org.apache.lens.ml.algo.api.MLAlgo;
+import org.apache.lens.ml.algo.api.MLModel;
import org.apache.lens.ml.algo.lib.AlgoArgParser;
import org.apache.lens.ml.algo.spark.HiveTableRDD;
-import org.apache.lens.ml.api.AlgoParam;
-import org.apache.lens.ml.api.DataSet;
-import org.apache.lens.ml.api.Feature;
-import org.apache.lens.ml.api.Model;
import org.apache.lens.server.api.error.LensException;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -49,71 +46,54 @@ import org.apache.spark.mllib.linalg.Vectors;
import scala.Tuple2;
-
/**
* The Class KMeansAlgo.
*/
-public class KMeansAlgo implements Algorithm {
-
- static String description = "Spark K means algo";
- static String name = "spark_k_means";
+@Algorithm(name = "spark_kmeans_algo", description = "Spark MLLib KMeans algo")
+public class KMeansAlgo implements MLAlgo {
- /**
- * The conf.
- */
+ /** The conf. */
private transient LensConf conf;
- /**
- * The spark context.
- */
+ /** The spark context. */
private JavaSparkContext sparkContext;
- /**
- * The part filter.
- */
+ /** The part filter. */
@AlgoParam(name = "partition", help = "Partition filter to be used while constructing table RDD")
private String partFilter = null;
- /**
- * The k.
- */
+ /** The k. */
@AlgoParam(name = "k", help = "Number of cluster")
private int k;
- /**
- * The max iterations.
- */
+ /** The max iterations. */
@AlgoParam(name = "maxIterations", help = "Maximum number of iterations", defaultValue = "100")
private int maxIterations = 100;
- /**
- * The runs.
- */
+ /** The runs. */
@AlgoParam(name = "runs", help = "Number of parallel run", defaultValue = "1")
private int runs = 1;
- /**
- * The initialization mode.
- */
+ /** The initialization mode. */
@AlgoParam(name = "initializationMode",
help = "initialization model, either \"random\" or \"k-means||\" (default).", defaultValue = "k-means||")
private String initializationMode = "k-means||";
@Override
public String getName() {
- return name;
+ return getClass().getAnnotation(Algorithm.class).name();
}
@Override
public String getDescription() {
- return description;
+ return getClass().getAnnotation(Algorithm.class).description();
}
/*
- * (non-Javadoc)
- *
- * @see org.apache.lens.ml.MLAlgo#configure(org.apache.lens.api.LensConf)
- */
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.MLAlgo#configure(org.apache.lens.api.LensConf)
+ */
@Override
public void configure(LensConf configuration) {
this.conf = configuration;
@@ -124,42 +104,22 @@ public class KMeansAlgo implements Algorithm {
return conf;
}
-
- /**
- * To hive conf.
+ /*
+ * (non-Javadoc)
*
- * @param conf the conf
- * @return the hive conf
+ * @see org.apache.lens.ml.MLAlgo#train(org.apache.lens.api.LensConf, java.lang.String, java.lang.String,
+ * java.lang.String, java.lang.String[])
*/
- private HiveConf toHiveConf(LensConf conf) {
- HiveConf hiveConf = new HiveConf();
- for (String key : conf.getProperties().keySet()) {
- hiveConf.set(key, conf.getProperties().get(key));
- }
- return hiveConf;
- }
-
@Override
- public List<AlgoParam> getParams() {
- return null;
- }
-
- @Override
- public TrainedModel train(Model model, DataSet trainingDataSet) throws LensException {
- AlgoArgParser.parseArgs(this, model.getAlgoSpec().getAlgoParams());
- List<String> features = new ArrayList<>();
- for (Feature feature : model.getFeatureSpec()) {
- features.add(feature.getDataColumn());
- }
-
+ public MLModel train(LensConf conf, String db, String table, String modelId, String... params) throws LensException {
+ List<String> features = AlgoArgParser.parseArgs(this, params);
final int[] featurePositions = new int[features.size()];
final int NUM_FEATURES = features.size();
JavaPairRDD<WritableComparable, HCatRecord> rdd = null;
-
try {
// Map feature names to positions
- Table tbl = Hive.get(toHiveConf(conf)).getTable(trainingDataSet.getDatabase(), trainingDataSet.getTableName());
+ Table tbl = Hive.get(toHiveConf(conf)).getTable(db, table);
List<FieldSchema> allCols = tbl.getAllCols();
int f = 0;
for (int i = 0; i < tbl.getAllCols().size(); i++) {
@@ -169,8 +129,7 @@ public class KMeansAlgo implements Algorithm {
}
}
- rdd = HiveTableRDD.createHiveTableRDD(sparkContext, toHiveConf(conf), trainingDataSet.getDatabase(),
- trainingDataSet.getTableName(), partFilter);
+ rdd = HiveTableRDD.createHiveTableRDD(sparkContext, toHiveConf(conf), db, table, partFilter);
JavaRDD<Vector> trainableRDD = rdd.map(new Function<Tuple2<WritableComparable, HCatRecord>, Vector>() {
@Override
public Vector call(Tuple2<WritableComparable, HCatRecord> v1) throws Exception {
@@ -184,11 +143,24 @@ public class KMeansAlgo implements Algorithm {
}
});
- KMeansModel kMeansModel = KMeans.train(trainableRDD.rdd(), k, maxIterations, runs, initializationMode);
- return new KMeansClusteringModel(model.getFeatureSpec(), kMeansModel);
+ KMeansModel model = KMeans.train(trainableRDD.rdd(), k, maxIterations, runs, initializationMode);
+ return new KMeansClusteringModel(modelId, model);
} catch (Exception e) {
- throw new LensException(
- "KMeans algo failed for " + trainingDataSet.getDatabase() + "." + trainingDataSet.getTableName(), e);
+ throw new LensException("KMeans algo failed for " + db + "." + table, e);
+ }
+ }
+
+ /**
+ * To hive conf.
+ *
+ * @param conf the conf
+ * @return the hive conf
+ */
+ private HiveConf toHiveConf(LensConf conf) {
+ HiveConf hiveConf = new HiveConf();
+ for (String key : conf.getProperties().keySet()) {
+ hiveConf.set(key, conf.getProperties().get(key));
}
+ return hiveConf;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansClusteringModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansClusteringModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansClusteringModel.java
index 6341268..62dc536 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansClusteringModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/kmeans/KMeansClusteringModel.java
@@ -18,12 +18,7 @@
*/
package org.apache.lens.ml.algo.spark.kmeans;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.lens.ml.algo.api.TrainedModel;
-import org.apache.lens.ml.api.Feature;
-import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.ml.algo.api.MLModel;
import org.apache.spark.mllib.clustering.KMeansModel;
import org.apache.spark.mllib.linalg.Vectors;
@@ -31,39 +26,42 @@ import org.apache.spark.mllib.linalg.Vectors;
/**
* The Class KMeansClusteringModel.
*/
-public class KMeansClusteringModel implements TrainedModel<Integer> {
+public class KMeansClusteringModel extends MLModel<Integer> {
- /**
- * The model.
- */
- private final KMeansModel kMeansModel;
+ /** The model. */
+ private final KMeansModel model;
- private List<Feature> featureList;
+ /** The model id. */
+ private final String modelId;
/**
- * Instantiates a new k means clustering model.
+ * Instantiates a new k means clustering model.
*
- * @param model the model
+ * @param modelId the model id
+ * @param model the model
*/
- public KMeansClusteringModel(List<Feature> featureList, KMeansModel model) {
- this.kMeansModel = model;
- this.featureList = featureList;
+ public KMeansClusteringModel(String modelId, KMeansModel model) {
+ this.model = model;
+ this.modelId = modelId;
}
-
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.ml.MLModel#predict(java.lang.Object[])
+ */
@Override
- public Integer predict(Map<String, String> featureVector) throws LensException {
- double[] featureArray = new double[featureList.size()];
- int i = 0;
- for (Feature feature : featureList) {
- String featureValue = featureVector.get(feature.getName());
- if (featureValue == null || featureValue.isEmpty()) {
- throw new LensException("Error while predicting: input featureVector doesn't contain all required features : "
- + "Feature Name: " + feature.getName());
+ public Integer predict(Object... args) {
+ // Convert the params to array of double
+ double[] arr = new double[args.length];
+ for (int i = 0; i < args.length; i++) {
+ if (args[i] != null) {
+ arr[i] = (Double) args[i];
} else {
- featureArray[i++] = Double.parseDouble(featureValue);
+ arr[i] = 0d;
}
}
- return kMeansModel.predict(Vectors.dense(featureArray));
+
+ return model.predict(Vectors.dense(arr));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogisticRegressionAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogisticRegressionAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogisticRegressionAlgo.java
index 7e21871..c2f97af 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogisticRegressionAlgo.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogisticRegressionAlgo.java
@@ -20,9 +20,10 @@ package org.apache.lens.ml.algo.spark.lr;
import java.util.Map;
+import org.apache.lens.ml.algo.api.AlgoParam;
+import org.apache.lens.ml.algo.api.Algorithm;
import org.apache.lens.ml.algo.spark.BaseSparkAlgo;
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.AlgoParam;
import org.apache.lens.server.api.error.LensException;
import org.apache.spark.mllib.classification.LogisticRegressionModel;
@@ -33,24 +34,18 @@ import org.apache.spark.rdd.RDD;
/**
* The Class LogisticRegressionAlgo.
*/
+@Algorithm(name = "spark_logistic_regression", description = "Spark logistic regression algo")
public class LogisticRegressionAlgo extends BaseSparkAlgo {
- static final String DESCRIPTION = "Spark logistic regression algo";
- static final String NAME = "spark_logistic_regression";
-
- /**
- * The iterations.
- */
+ /** The iterations. */
@AlgoParam(name = "iterations", help = "Max number of iterations", defaultValue = "100")
private int iterations;
- /**
- * The step size.
- */
+
+ /** The step size. */
@AlgoParam(name = "stepSize", help = "Step size", defaultValue = "1.0d")
private double stepSize;
- /**
- * The min batch fraction.
- */
+
+ /** The min batch fraction. */
@AlgoParam(name = "minBatchFraction", help = "Fraction for batched learning", defaultValue = "1.0d")
private double minBatchFraction;
@@ -64,14 +59,6 @@ public class LogisticRegressionAlgo extends BaseSparkAlgo {
super(name, description);
}
- public LogisticRegressionAlgo(String name) {
- super(name, DESCRIPTION);
- }
-
- public LogisticRegressionAlgo() {
- super(NAME, DESCRIPTION);
- }
-
/*
* (non-Javadoc)
*
@@ -90,11 +77,10 @@ public class LogisticRegressionAlgo extends BaseSparkAlgo {
* @see org.apache.lens.ml.spark.algos.BaseSparkAlgo#trainInternal(java.lang.String, org.apache.spark.rdd.RDD)
*/
@Override
- protected BaseSparkClassificationModel trainInternal(RDD<LabeledPoint> trainingRDD)
+ protected BaseSparkClassificationModel trainInternal(String modelId, RDD<LabeledPoint> trainingRDD)
throws LensException {
-
LogisticRegressionModel lrModel = LogisticRegressionWithSGD.train(trainingRDD, iterations, stepSize,
minBatchFraction);
- return new LogitRegressionClassificationModel(features, lrModel);
+ return new LogitRegressionClassificationModel(modelId, lrModel);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogitRegressionClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogitRegressionClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogitRegressionClassificationModel.java
index d6e07a5..a4206e5 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogitRegressionClassificationModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/lr/LogitRegressionClassificationModel.java
@@ -18,10 +18,7 @@
*/
package org.apache.lens.ml.algo.spark.lr;
-import java.util.List;
-
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.Feature;
import org.apache.spark.mllib.classification.LogisticRegressionModel;
@@ -31,10 +28,12 @@ import org.apache.spark.mllib.classification.LogisticRegressionModel;
public class LogitRegressionClassificationModel extends BaseSparkClassificationModel<LogisticRegressionModel> {
/**
- * @param featureList
- * @param model
+ * Instantiates a new logit regression classification model.
+ *
+ * @param modelId the model id
+ * @param model the model
*/
- public LogitRegressionClassificationModel(List<Feature> featureList, LogisticRegressionModel model) {
- super(featureList, model);
+ public LogitRegressionClassificationModel(String modelId, LogisticRegressionModel model) {
+ super(modelId, model);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesAlgo.java
index e9709d3..c484dfe 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesAlgo.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesAlgo.java
@@ -20,27 +20,23 @@ package org.apache.lens.ml.algo.spark.nb;
import java.util.Map;
+import org.apache.lens.ml.algo.api.AlgoParam;
+import org.apache.lens.ml.algo.api.Algorithm;
import org.apache.lens.ml.algo.spark.BaseSparkAlgo;
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.AlgoParam;
import org.apache.lens.server.api.error.LensException;
import org.apache.spark.mllib.classification.NaiveBayes;
-import org.apache.spark.mllib.classification.NaiveBayesModel;
import org.apache.spark.mllib.regression.LabeledPoint;
import org.apache.spark.rdd.RDD;
/**
* The Class NaiveBayesAlgo.
*/
+@Algorithm(name = "spark_naive_bayes", description = "Spark Naive Bayes classifier algo")
public class NaiveBayesAlgo extends BaseSparkAlgo {
- static final String DESCRIPTION = "Spark naive bayes algo";
- static final String NAME = "spark_naive_bayes";
-
- /**
- * The lambda.
- */
+ /** The lambda. */
@AlgoParam(name = "lambda", help = "Lambda parameter for naive bayes learner", defaultValue = "1.0d")
private double lambda = 1.0;
@@ -54,10 +50,6 @@ public class NaiveBayesAlgo extends BaseSparkAlgo {
super(name, description);
}
- public NaiveBayesAlgo() {
- super(NAME, DESCRIPTION);
- }
-
/*
* (non-Javadoc)
*
@@ -74,9 +66,8 @@ public class NaiveBayesAlgo extends BaseSparkAlgo {
* @see org.apache.lens.ml.spark.algos.BaseSparkAlgo#trainInternal(java.lang.String, org.apache.spark.rdd.RDD)
*/
@Override
- protected BaseSparkClassificationModel trainInternal(RDD<LabeledPoint> trainingRDD)
+ protected BaseSparkClassificationModel trainInternal(String modelId, RDD<LabeledPoint> trainingRDD)
throws LensException {
- NaiveBayesModel naiveBayesModel = NaiveBayes.train(trainingRDD, lambda);
- return new NaiveBayesClassificationModel(features, NaiveBayes.train(trainingRDD, lambda));
+ return new NaiveBayesClassificationModel(modelId, NaiveBayes.train(trainingRDD, lambda));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesClassificationModel.java
index edbcba4..26d39df 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesClassificationModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/nb/NaiveBayesClassificationModel.java
@@ -18,10 +18,7 @@
*/
package org.apache.lens.ml.algo.spark.nb;
-import java.util.List;
-
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.Feature;
import org.apache.spark.mllib.classification.NaiveBayesModel;
@@ -31,10 +28,12 @@ import org.apache.spark.mllib.classification.NaiveBayesModel;
public class NaiveBayesClassificationModel extends BaseSparkClassificationModel<NaiveBayesModel> {
/**
- * @param featureList
- * @param model
+ * Instantiates a new naive bayes classification model.
+ *
+ * @param modelId the model id
+ * @param model the model
*/
- public NaiveBayesClassificationModel(List<Feature> featureList, NaiveBayesModel model) {
- super(featureList, model);
+ public NaiveBayesClassificationModel(String modelId, NaiveBayesModel model) {
+ super(modelId, model);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMAlgo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMAlgo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMAlgo.java
index 8e3083d..4b14d66 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMAlgo.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMAlgo.java
@@ -20,9 +20,10 @@ package org.apache.lens.ml.algo.spark.svm;
import java.util.Map;
+import org.apache.lens.ml.algo.api.AlgoParam;
+import org.apache.lens.ml.algo.api.Algorithm;
import org.apache.lens.ml.algo.spark.BaseSparkAlgo;
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.AlgoParam;
import org.apache.lens.server.api.error.LensException;
import org.apache.spark.mllib.classification.SVMModel;
@@ -33,31 +34,22 @@ import org.apache.spark.rdd.RDD;
/**
* The Class SVMAlgo.
*/
+@Algorithm(name = "spark_svm", description = "Spark SVML classifier algo")
public class SVMAlgo extends BaseSparkAlgo {
- static final String DESCRIPTION = "Spark SVM algo";
- static final String NAME = "spark_svm";
- /**
- * The min batch fraction.
- */
+ /** The min batch fraction. */
@AlgoParam(name = "minBatchFraction", help = "Fraction for batched learning", defaultValue = "1.0d")
private double minBatchFraction;
- /**
- * The reg param.
- */
+ /** The reg param. */
@AlgoParam(name = "regParam", help = "regularization parameter for gradient descent", defaultValue = "1.0d")
private double regParam;
- /**
- * The step size.
- */
+ /** The step size. */
@AlgoParam(name = "stepSize", help = "Iteration step size", defaultValue = "1.0d")
private double stepSize;
- /**
- * The iterations.
- */
+ /** The iterations. */
@AlgoParam(name = "iterations", help = "Number of iterations", defaultValue = "100")
private int iterations;
@@ -71,10 +63,6 @@ public class SVMAlgo extends BaseSparkAlgo {
super(name, description);
}
- public SVMAlgo() {
- super(NAME, DESCRIPTION);
- }
-
/*
* (non-Javadoc)
*
@@ -94,9 +82,9 @@ public class SVMAlgo extends BaseSparkAlgo {
* @see org.apache.lens.ml.spark.algos.BaseSparkAlgo#trainInternal(java.lang.String, org.apache.spark.rdd.RDD)
*/
@Override
- protected BaseSparkClassificationModel trainInternal(RDD<LabeledPoint> trainingRDD)
+ protected BaseSparkClassificationModel trainInternal(String modelId, RDD<LabeledPoint> trainingRDD)
throws LensException {
SVMModel svmModel = SVMWithSGD.train(trainingRDD, iterations, stepSize, regParam, minBatchFraction);
- return new SVMClassificationModel(features, svmModel);
+ return new SVMClassificationModel(modelId, svmModel);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMClassificationModel.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMClassificationModel.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMClassificationModel.java
index d793d4a..433c0f9 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMClassificationModel.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/svm/SVMClassificationModel.java
@@ -18,10 +18,7 @@
*/
package org.apache.lens.ml.algo.spark.svm;
-import java.util.List;
-
import org.apache.lens.ml.algo.spark.BaseSparkClassificationModel;
-import org.apache.lens.ml.api.Feature;
import org.apache.spark.mllib.classification.SVMModel;
@@ -31,10 +28,12 @@ import org.apache.spark.mllib.classification.SVMModel;
public class SVMClassificationModel extends BaseSparkClassificationModel<SVMModel> {
/**
- * @param featureList
- * @param model
+ * Instantiates a new SVM classification model.
+ *
+ * @param modelId the model id
+ * @param model the model
*/
- public SVMClassificationModel(List<Feature> featureList, SVMModel model) {
- super(featureList, model);
+ public SVMClassificationModel(String modelId, SVMModel model) {
+ super(modelId, model);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Algo.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Algo.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Algo.java
deleted file mode 100644
index 1bd0eb3..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Algo.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlElementWrapper;
-
-
-public interface Algo {
-
- @XmlElement
- String getName();
-
- @XmlElement
- String getDescription();
-
- @XmlElementWrapper
- List<AlgoParam> getParams();
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoParam.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoParam.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoParam.java
deleted file mode 100644
index 4186318..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoParam.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-import javax.xml.bind.annotation.XmlElement;
-
-/**
- * The Interface AlgoParam.
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.FIELD)
-public @interface AlgoParam {
-
- /**
- * Name.
- *
- * @return the string
- */
- @XmlElement String name();
-
- /**
- * Help.
- *
- * @return the string
- */
- @XmlElement String help();
-
- /**
- * Default value.
- *
- * @return the string
- */
- @XmlElement String defaultValue() default "None";
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoSpec.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoSpec.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoSpec.java
deleted file mode 100644
index e96a350..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/AlgoSpec.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.util.Map;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.Setter;
-
-/**
- * The Algo Spec class. This class works as a particular instance of an Algorithm run. since it contains the exact
- * algoParams with which an Algorithm was run.
- */
-@AllArgsConstructor
-public class AlgoSpec {
- @Getter
- @Setter
- private Algo algo;
- @Getter
- @Setter
- private Map<String, String> algoParams;
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/DataSet.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/DataSet.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/DataSet.java
deleted file mode 100644
index 892aec0..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/DataSet.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.Setter;
-
-/**
- * Contains meta data for a data set. A data set is identified by name, tableName and database.
- */
-@AllArgsConstructor
-@XmlRootElement
-public class DataSet {
- @Getter
- @Setter
- @XmlElement
- String name;
-
- @Getter
- @Setter
- @XmlElement
- String tableName;
-
- @Getter
- @Setter
- @XmlElement
- String database;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Evaluation.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Evaluation.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Evaluation.java
deleted file mode 100644
index 8022b9f..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Evaluation.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.util.Date;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.lens.api.LensSessionHandle;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-
-/**
- * Contains meta data for an Evaluation. Evaluation captures metadata of the process of evaluating data contained in
- * inputDataSetName against modelInstanceId.
- */
-@NoArgsConstructor
-@AllArgsConstructor
-@XmlRootElement
-public class Evaluation implements MLProcess {
-
- @XmlElement
- @Getter
- @Setter
- String id;
-
- @XmlElement
- @Getter
- @Setter
- Date startTime;
-
- @XmlElement
- @Getter
- @Setter
- Date finishTime;
-
- @XmlElement
- @Getter
- @Setter
- Status status;
-
- @Getter
- @Setter
- @XmlElement
- LensSessionHandle lensSessionHandle;
-
- @Getter
- @Setter
- @XmlElement
- String modeInstanceId;
-
- @Getter
- @Setter
- @XmlElement
- String inputDataSetName;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Feature.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Feature.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Feature.java
deleted file mode 100644
index 5dc8e87..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Feature.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.io.Serializable;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.Setter;
-
-/**
- * Feature class. Equivalent of a feature of a Machine Learning model.
- */
-
-@AllArgsConstructor
-@XmlRootElement
-public class Feature implements Serializable {
-
- @Getter
- @Setter
- @XmlElement
- private String name;
- @Getter
- @Setter
- @XmlElement
- private String description;
- @Getter
- @Setter
- @XmlElement
- private Type type;
- /**
- * Name of the column of the table to which this feature is mapped.
- */
- @Getter
- @Setter
- @XmlElement
- private String dataColumn;
-
- @XmlRootElement
- public enum Type {
- Categorical,
- Continuous
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/LensML.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/LensML.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/LensML.java
index 191125d..23b5437 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/LensML.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/LensML.java
@@ -22,171 +22,140 @@ import java.util.List;
import java.util.Map;
import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.ml.algo.api.MLAlgo;
+import org.apache.lens.ml.algo.api.MLModel;
import org.apache.lens.server.api.error.LensException;
/**
- * LensML interface to train and evaluate Machine learning models.
+ * Lens's machine learning interface used by client code as well as Lens ML service.
*/
public interface LensML {
- /**
- * Name of ML service
- */
+ /** Name of ML service */
String NAME = "ml";
/**
- * @return the list of supported Algos.
- */
- List<Algo> getAlgos();
-
- /**
- * Get Algo given an algo name.
- *
- * @param name of the Algo
- * @return Algo definition
- */
- Algo getAlgo(String name) throws LensException;
-
- /**
- * Creates a data set from an existing table
+ * Get list of available machine learning algorithms
*
- * @param name the name of the data set
- * @param dataTable the data table
* @return
*/
- String createDataSet(String name, String dataTable, String dataBase) throws LensException;
+ List<String> getAlgorithms();
/**
- * Creates a data set from a query
+ * Get user friendly information about parameters accepted by the algorithm.
*
- * @param name the name of the data set
- * @param query query
- * @return
+ * @param algorithm the algorithm
+ * @return map of param key to its help message
*/
- String createDataSetFromQuery(String name, String query);
+ Map<String, String> getAlgoParamDescription(String algorithm);
/**
- * Returns the data set given the name
+ * Get a algo object instance which could be used to generate a model of the given algorithm.
*
- * @param name the name of the data set
- * @return
+ * @param algorithm the algorithm
+ * @return the algo for name
+ * @throws LensException the lens exception
*/
- DataSet getDataSet(String name) throws LensException;
+ MLAlgo getAlgoForName(String algorithm) throws LensException;
/**
- * Creates a Model with a chosen Algo and its parameters, feature list and the label.
+ * Create a model using the given HCatalog table as input. The arguments should contain information needeed to
+ * generate the model.
*
- * @param name the name of Model
- * @param algo the name of the Alog
- * @param algoParams the algo parameters
- * @param features list of features
- * @param label the label to use
- * @returns Model id of the created model
+ * @param table the table
+ * @param algorithm the algorithm
+ * @param args the args
+ * @return Unique ID of the model created after training is complete
+ * @throws LensException the lens exception
*/
- String createModel(String name, String algo, Map<String, String> algoParams,
- List<Feature> features, Feature label, LensSessionHandle lensSessionHandle) throws LensException;
+ String train(String table, String algorithm, String[] args) throws LensException;
/**
- * Get Model given a modelId
+ * Get model IDs for the given algorithm.
*
- * @param modelId the id of the model
- * @return the model
+ * @param algorithm the algorithm
+ * @return the models
+ * @throws LensException the lens exception
*/
- Model getModel(String modelId) throws LensException;
+ List<String> getModels(String algorithm) throws LensException;
/**
- * Train a model. This calls returns immediately after triggering the training
- * asynchronously, the readiness of the ModelInstance should be checked based on its status.
+ * Get a model instance given the algorithm name and model ID.
*
- * @param modelId the model id
- * @param dataSetName data set name to use
- * @return ModelInstance id the handle to the ModelInstance instance
- */
- String trainModel(String modelId, String dataSetName, LensSessionHandle lensSessionHandle) throws LensException;
-
- /**
- * Get Trained Model
- *
- * @param modelInstanceId the id of the ModelInstance
- * @return Trained model
- */
- ModelInstance getModelInstance(String modelInstanceId) throws LensException;
-
- /**
- * Cancels the creation of modelInstance.
- *
- * @param modelInstanceId
- * @return true on successful cancellation false otherwise.
- */
- boolean cancelModelInstance(String modelInstanceId);
-
- /**
- * Get the list of ModelInstance for a given model
- *
- * @param modelId the model id
- * @return List of trained models
+ * @param algorithm the algorithm
+ * @param modelId the model id
+ * @return the model
+ * @throws LensException the lens exception
*/
- List<ModelInstance> getAllModelInstances(String modelId);
+ MLModel getModel(String algorithm, String modelId) throws LensException;
/**
- * Evaluate a ModelInstance. This calls returns immediately after triggering the training
- * asynchronously, the readiness of the Evaluation should be checked based on its status.
+ * Get the FS location where model instance is saved.
*
- * @param modelInstanceId the trained model id
- * @param dataSetName the data to use to evaluate
- * @return the evaluationId
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @return the model path
*/
- String evaluate(String modelInstanceId, String dataSetName, LensSessionHandle lensSessionHandle) throws LensException;
+ String getModelPath(String algorithm, String modelID);
/**
- * Get Evaluation
+ * Evaluate model by running it against test data contained in the given table.
*
- * @param evalId
- * @return the evaluation
+ * @param session the session
+ * @param table the table
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @return Test report object containing test output table, and various evaluation metrics
+ * @throws LensException the lens exception
*/
- Evaluation getEvaluation(String evalId) throws LensException;
+ MLTestReport testModel(LensSessionHandle session, String table, String algorithm, String modelID,
+ String outputTable) throws LensException;
/**
- * Cancels the Evaluation
+ * Get test reports for an algorithm.
*
- * @param evalId
- * @return true on successful cancellation false otherwise.
+ * @param algorithm the algorithm
+ * @return the test reports
+ * @throws LensException the lens exception
*/
- boolean cancelEvaluation(String evalId);
+ List<String> getTestReports(String algorithm) throws LensException;
/**
- * Batch predicts for a given data set. This calls returns immediately after triggering the prediction
- * asynchronously, the readiness of the prediction should be checked based on its status.
+ * Get a test report by ID.
*
- * @param modelInstanceId
- * @param dataSetName
- * @return prediction id
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @return the test report
+ * @throws LensException the lens exception
*/
- String predict(String modelInstanceId, String dataSetName, LensSessionHandle lensSessionHandle) throws LensException;
+ MLTestReport getTestReport(String algorithm, String reportID) throws LensException;
/**
- * Get BatchPrediction information
+ * Online predict call given a model ID, algorithm name and sample feature values.
*
- * @param predictionId
- * @return
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @param features the features
+ * @return prediction result
+ * @throws LensException the lens exception
*/
- Prediction getPrediction(String predictionId) throws LensException;
+ Object predict(String algorithm, String modelID, Object[] features) throws LensException;
/**
- * Cancels the Prediction
+ * Permanently delete a model instance.
*
- * @param predictionId
- * @return true on successful cancellation false otherwise.
+ * @param algorithm the algorithm
+ * @param modelID the model id
+ * @throws LensException the lens exception
*/
- boolean cancelPrediction(String predictionId);
+ void deleteModel(String algorithm, String modelID) throws LensException;
/**
- * Predict for a given feature vector
+ * Permanently delete a test report instance.
*
- * @param modelInstanceId
- * @param featureVector the key is feature name.
- * @return predicted value
+ * @param algorithm the algorithm
+ * @param reportID the report id
+ * @throws LensException the lens exception
*/
- String predict(String modelInstanceId, Map<String, String> featureVector) throws LensException;
-
+ void deleteTestReport(String algorithm, String reportID) throws LensException;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLConfConstants.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLConfConstants.java
deleted file mode 100644
index ddc12a6..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLConfConstants.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import org.apache.lens.server.api.error.LensException;
-
-/**
- * The class MLConfConstants.
- */
-public class MLConfConstants {
-
- private MLConfConstants() throws LensException {
- throw new LensException("Can't instantiate MLConfConstants");
- }
-
- public static final String SERVER_PFX = "lens.server.ml.";
- /**
- * Minimum thread pool size for worked threads who performing ML process i.e. running MLEvaluation,
- * prediction or evaluation
- */
- public static final int DEFAULT_EXECUTOR_POOL_MIN_THREADS = 3;
- public static final String EXECUTOR_POOL_MIN_THREADS = SERVER_PFX + "executor.pool.min.threads";
-
- /**
- * Maximum thread pool size for worked threads who performing ML process i.e. running MLEvaluation,
- * prediction or evaluation
- */
- public static final int DEFAULT_EXECUTOR_POOL_MAX_THREADS = 100;
- public static final String EXECUTOR_POOL_MAX_THREADS = SERVER_PFX + "executor.pool.max.threads";
-
- /**
- * Minimum thread pool size for worked threads who performing ML process i.e. running MLEvaluation,
- * prediction or evaluation
- */
-
-
- /**
- * keep alive time for threads in the MLProcess thread pool
- */
- public static final int DEFAULT_CREATOR_POOL_KEEP_ALIVE_MILLIS = 60000;
-
- /**
- * This is the time a MLprocess will be in cache after it is finished. After which request for that MLprocess will
- * be server from the Meta store.
- */
- public static final long DEFAULT_ML_PROCESS_CACHE_LIFE = 1000 * 60 * 10; //10 min.
- public static final String ML_PROCESS_CACHE_LIFE = SERVER_PFX + "mlprocess.cache.life";
-
- /**
- * This is the maximum time allowed for a MLProcess to run. After which it will be killed by the MLProcesPurger
- * thread.
- */
- public static final long DEFAULT_ML_PROCESS_MAX_LIFE = 1000 * 60 * 60 * 10; // 10 hours.
- public static final String ML_PROCESS_MAX_LIFE = SERVER_PFX + "mlprocess.max.life";
-
- /**
- * The Constant UDF_NAME.
- */
- public static final String UDF_NAME = "predict";
-
- /**
- * prefix for output table which gets created for any prediction.
- */
- public static final String PREDICTION_OUTPUT_TABLE_PREFIX = "prediction_";
-
- /**
- * prefix for output table which gets created for any evaluation.
- */
- public static final String EVALUATION_OUTPUT_TABLE_PREFIX = "evaluation_";
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLProcess.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLProcess.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLProcess.java
deleted file mode 100644
index 8f76435..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLProcess.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.util.Date;
-
-import org.apache.lens.api.LensSessionHandle;
-
-/**
- * Interface MLProcess for Process which go through ML LifeCycle of Submitting, Polling and Completion.
- */
-
-public interface MLProcess {
-
- String getId();
-
- void setId(String id);
-
- Date getStartTime();
-
- void setStartTime(Date time);
-
- Date getFinishTime();
-
- void setFinishTime(Date time);
-
- Status getStatus();
-
- void setStatus(Status status);
-
- LensSessionHandle getLensSessionHandle();
-
- void setLensSessionHandle(LensSessionHandle lensSessionHandle);
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLTestReport.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLTestReport.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLTestReport.java
new file mode 100644
index 0000000..965161a
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/MLTestReport.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.api;
+
+import java.io.Serializable;
+import java.util.List;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+
+/**
+ * Instantiates a new ML test report.
+ */
+@NoArgsConstructor
+@ToString
+public class MLTestReport implements Serializable {
+
+ /** The test table. */
+ @Getter
+ @Setter
+ private String testTable;
+
+ /** The output table. */
+ @Getter
+ @Setter
+ private String outputTable;
+
+ /** The output column. */
+ @Getter
+ @Setter
+ private String outputColumn;
+
+ /** The label column. */
+ @Getter
+ @Setter
+ private String labelColumn;
+
+ /** The feature columns. */
+ @Getter
+ @Setter
+ private List<String> featureColumns;
+
+ /** The algorithm. */
+ @Getter
+ @Setter
+ private String algorithm;
+
+ /** The model id. */
+ @Getter
+ @Setter
+ private String modelID;
+
+ /** The report id. */
+ @Getter
+ @Setter
+ private String reportID;
+
+ /** The query id. */
+ @Getter
+ @Setter
+ private String queryID;
+
+ /** The test output path. */
+ @Getter
+ @Setter
+ private String testOutputPath;
+
+ /** The prediction result column. */
+ @Getter
+ @Setter
+ private String predictionResultColumn;
+
+ /** The lens query id. */
+ @Getter
+ @Setter
+ private String lensQueryID;
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Model.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Model.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Model.java
deleted file mode 100644
index 8cb2193..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Model.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlElementWrapper;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.Setter;
-
-/**
- * The Model class. Contains meta data for a model creation. Algorithm to use, list of features and label. This doesn't
- * contains the actual data for training the model (which is separated and stored in model instance class).
- */
-@AllArgsConstructor
-@XmlRootElement
-public class Model {
-
- @Getter
- @Setter
- @XmlElement
- private String id;
-
- @Getter
- @Setter
- @XmlElement
- private String name;
-
- @Getter
- @Setter
- @XmlElement
- private AlgoSpec algoSpec;
-
- @Getter
- @Setter
- @XmlElementWrapper
- private List<Feature> featureSpec;
-
- @Getter
- @Setter
- @XmlElementWrapper
- private Feature labelSpec;
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelInstance.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelInstance.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelInstance.java
deleted file mode 100644
index 11c53a0..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelInstance.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.util.Date;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.lens.api.LensSessionHandle;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.Setter;
-
-/**
- * The Model Instance Class. Contains meta data for TrainedModel i.e. information about the algorithm used for
- * training and specification about features and label. Model instance captures meta data for the process of training
- * Model modelId using test data from dataSet.
- */
-@AllArgsConstructor
-@XmlRootElement
-public class ModelInstance implements MLProcess {
-
- @XmlElement
- @Getter
- @Setter
- String id;
-
- @XmlElement
- @Getter
- @Setter
- Date startTime;
-
- @XmlElement
- @Getter
- @Setter
- Date finishTime;
-
- @XmlElement
- @Getter
- @Setter
- Status status;
-
- @Getter
- @Setter
- @XmlElement
- LensSessionHandle lensSessionHandle;
-
- @Getter
- @Setter
- @XmlElement
- String modelId;
-
- @Getter
- @Setter
- @XmlElement
- String dataSet;
-
- @Getter
- @Setter
- @XmlElement
- String path;
-
- @Getter
- @Setter
- @XmlElement
- String evaluationId;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelMetadata.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelMetadata.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelMetadata.java
new file mode 100644
index 0000000..3f7dff1
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/ModelMetadata.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.api;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * The Class ModelMetadata.
+ */
+@XmlRootElement
+/**
+ * Instantiates a new model metadata.
+ *
+ * @param modelID
+ * the model id
+ * @param table
+ * the table
+ * @param algorithm
+ * the algorithm
+ * @param params
+ * the params
+ * @param createdAt
+ * the created at
+ * @param modelPath
+ * the model path
+ * @param labelColumn
+ * the label column
+ * @param features
+ * the features
+ */
+@AllArgsConstructor
+/**
+ * Instantiates a new model metadata.
+ */
+@NoArgsConstructor
+public class ModelMetadata {
+
+ /** The model id. */
+ @XmlElement
+ @Getter
+ private String modelID;
+
+ /** The table. */
+ @XmlElement
+ @Getter
+ private String table;
+
+ /** The algorithm. */
+ @XmlElement
+ @Getter
+ private String algorithm;
+
+ /** The params. */
+ @XmlElement
+ @Getter
+ private String params;
+
+ /** The created at. */
+ @XmlElement
+ @Getter
+ private String createdAt;
+
+ /** The model path. */
+ @XmlElement
+ @Getter
+ private String modelPath;
+
+ /** The label column. */
+ @XmlElement
+ @Getter
+ private String labelColumn;
+
+ /** The features. */
+ @XmlElement
+ @Getter
+ private String features;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+
+ builder.append("Algorithm: ").append(algorithm).append('\n');
+ builder.append("Model ID: ").append(modelID).append('\n');
+ builder.append("Training table: ").append(table).append('\n');
+ builder.append("Features: ").append(features).append('\n');
+ builder.append("Labelled Column: ").append(labelColumn).append('\n');
+ builder.append("Training params: ").append(params).append('\n');
+ builder.append("Created on: ").append(createdAt).append('\n');
+ builder.append("Model saved at: ").append(modelPath).append('\n');
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Prediction.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Prediction.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Prediction.java
deleted file mode 100644
index 204becf..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Prediction.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import java.util.Date;
-
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.lens.api.LensSessionHandle;
-
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.Setter;
-
-/*
- * Batch prediction Instance
- */
-
-/**
- * Contains meta data for an Prediction. Prediction captures the meta data of process of batch predicting data of
- * inputDataSet against the ModelInstance modelInstanceId and populating results in outputDataSet.
- */
-@AllArgsConstructor
-@XmlRootElement
-public class Prediction implements MLProcess {
- @XmlElement
- @Getter
- @Setter
- String id;
-
- @XmlElement
- @Getter
- @Setter
- Date startTime;
-
- @XmlElement
- @Getter
- @Setter
- Date finishTime;
-
- @XmlElement
- @Getter
- @Setter
- Status status;
-
- @Getter
- @Setter
- @XmlElement
- LensSessionHandle lensSessionHandle;
-
- @Getter
- @Setter
- @XmlElement
- String modelInstanceId;
-
- @Getter
- @Setter
- @XmlElement
- String inputDataSet;
-
- @Getter
- @Setter
- @XmlElement
- String outputDataSet;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Status.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Status.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Status.java
deleted file mode 100644
index 8436e28..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/Status.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.api;
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-/**
- * Status of the batch jobs for training, evaluation and prediction
- */
-
-@XmlRootElement
-public enum Status {
- SUBMITTED,
- RUNNING,
- FAILED,
- CANCELLED,
- COMPLETED
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/api/TestReport.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/api/TestReport.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/TestReport.java
new file mode 100644
index 0000000..294fef3
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/api/TestReport.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.api;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/**
+ * The Class TestReport.
+ */
+@XmlRootElement
+/**
+ * Instantiates a new test report.
+ *
+ * @param testTable
+ * the test table
+ * @param outputTable
+ * the output table
+ * @param outputColumn
+ * the output column
+ * @param labelColumn
+ * the label column
+ * @param featureColumns
+ * the feature columns
+ * @param algorithm
+ * the algorithm
+ * @param modelID
+ * the model id
+ * @param reportID
+ * the report id
+ * @param queryID
+ * the query id
+ */
+@AllArgsConstructor
+/**
+ * Instantiates a new test report.
+ */
+@NoArgsConstructor
+public class TestReport {
+
+ /** The test table. */
+ @XmlElement
+ @Getter
+ private String testTable;
+
+ /** The output table. */
+ @XmlElement
+ @Getter
+ private String outputTable;
+
+ /** The output column. */
+ @XmlElement
+ @Getter
+ private String outputColumn;
+
+ /** The label column. */
+ @XmlElement
+ @Getter
+ private String labelColumn;
+
+ /** The feature columns. */
+ @XmlElement
+ @Getter
+ private String featureColumns;
+
+ /** The algorithm. */
+ @XmlElement
+ @Getter
+ private String algorithm;
+
+ /** The model id. */
+ @XmlElement
+ @Getter
+ private String modelID;
+
+ /** The report id. */
+ @XmlElement
+ @Getter
+ private String reportID;
+
+ /** The query id. */
+ @XmlElement
+ @Getter
+ private String queryID;
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("Input test table: ").append(testTable).append('\n');
+ builder.append("Algorithm: ").append(algorithm).append('\n');
+ builder.append("Report id: ").append(reportID).append('\n');
+ builder.append("Model id: ").append(modelID).append('\n');
+ builder.append("Lens Query id: ").append(queryID).append('\n');
+ builder.append("Feature columns: ").append(featureColumns).append('\n');
+ builder.append("Labelled column: ").append(labelColumn).append('\n');
+ builder.append("Predicted column: ").append(outputColumn).append('\n');
+ builder.append("Test output table: ").append(outputTable).append('\n');
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/InMemoryMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/InMemoryMetaStoreClient.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/InMemoryMetaStoreClient.java
deleted file mode 100644
index e4af3dc..0000000
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/InMemoryMetaStoreClient.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.ml.dao;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.ml.api.*;
-import org.apache.lens.server.api.error.LensException;
-
-public class InMemoryMetaStoreClient implements MetaStoreClient {
-
- ConcurrentMap<String, Model> modelConcurrentMap = new ConcurrentHashMap<String, Model>();
- ConcurrentMap<String, ModelInstance> modelInstanceConcurrentMap = new ConcurrentHashMap<String, ModelInstance>();
- ConcurrentMap<String, Evaluation> evaluationConcurrentMap = new ConcurrentHashMap<String, Evaluation>();
- ConcurrentMap<String, Prediction> predictionConcurrentMap = new ConcurrentHashMap<String, Prediction>();
- ConcurrentMap<String, DataSet> dataSetConcurrentMap = new ConcurrentHashMap();
-
- @Override
- public DataSet createDataSet(String name, String dataTable, String dataBase) throws LensException {
- DataSet dataSet = new DataSet(name, dataTable, dataBase);
- dataSetConcurrentMap.put(name, dataSet);
- return dataSet;
- }
-
- @Override
- public DataSet getDataSet(String name) {
- return dataSetConcurrentMap.get(name);
- }
-
- @Override
- public Model createModel(String name, String algo, AlgoSpec algoSpec, List<Feature> features, Feature label) throws
- LensException {
- String id = UUID.randomUUID().toString();
- while (modelConcurrentMap.containsKey(id)) {
- id = UUID.randomUUID().toString();
- }
-
- Model model = new Model(id, name, algoSpec, features, label);
- modelConcurrentMap.put(id, model);
- return model;
- }
-
- @Override
- public Model getModel(String modelId) throws LensException {
- return modelConcurrentMap.get(modelId);
- }
-
- @Override
- public ModelInstance createModelInstance(Date startTime, Date finishTime, Status status,
- LensSessionHandle lensSessionHandle,
- String modelId, String dataSet, String path, String evaluationId)
- throws LensException {
- String id = UUID.randomUUID().toString();
- while (modelInstanceConcurrentMap.containsKey(id)) {
- id = UUID.randomUUID().toString();
- }
- ModelInstance modelInstance = new ModelInstance(id, startTime, finishTime, status, lensSessionHandle, modelId,
- dataSet, path, evaluationId);
- modelInstanceConcurrentMap.put(modelInstance.getId(), modelInstance);
- return modelInstance;
- }
-
- @Override
- public void updateModelInstance(ModelInstance modelInstance) throws LensException {
- if (modelConcurrentMap.keySet().contains(modelInstance.getId())) {
- ModelInstance modelInstanceDb = modelInstanceConcurrentMap.get(modelInstance.getId());
- modelInstanceDb.setStatus(modelInstance.getStatus());
- modelInstanceDb.setFinishTime(modelInstance.getFinishTime());
- }
- }
-
- @Override
- public ModelInstance getModelInstance(String modelInstanceId) throws LensException {
- ModelInstance modelInstance = modelInstanceConcurrentMap.get(modelInstanceId);
- return modelInstance;
- }
-
-
- public Evaluation getEvaluation(String evaluationId) throws LensException {
- return evaluationConcurrentMap.get(evaluationId);
- }
-
- @Override
- public void updateEvaluation(Evaluation evaluation) throws LensException {
- if (modelConcurrentMap.keySet().contains(evaluation.getId())) {
- Evaluation evaluationDb = evaluationConcurrentMap.get(evaluation.getId());
- evaluationDb.setStatus(evaluation.getStatus());
- evaluationDb.setFinishTime(evaluation.getFinishTime());
- }
- }
-
- @Override
- public List<ModelInstance> getIncompleteModelInstances() {
- List<ModelInstance> incompleteModelInstances = new ArrayList<>();
- for (ModelInstance modelInstance : modelInstanceConcurrentMap.values()) {
- if (!(modelInstance.getStatus() == Status.COMPLETED || modelInstance.getStatus() == Status.FAILED || modelInstance
- .getStatus() == Status.CANCELLED)) {
- incompleteModelInstances.add(modelInstance);
- }
- }
- return incompleteModelInstances;
- }
-
- @Override
- public List<Evaluation> getIncompleteEvaluations() {
- List<Evaluation> incompleteEvaluations = new ArrayList<>();
- for (Evaluation evaluation : evaluationConcurrentMap.values()) {
- if (!(evaluation.getStatus() == Status.COMPLETED || evaluation.getStatus() == Status.FAILED || evaluation
- .getStatus() == Status.CANCELLED)) {
- incompleteEvaluations.add(evaluation);
- }
- }
- return incompleteEvaluations;
- }
-
- @Override
- public Prediction createPrediction(Date startTime, Date finishTime, Status status,
- LensSessionHandle lensSessionHandle,
- String modelInstanceId, String inputDataSet, String outputDataSet)
- throws LensException {
- String id = UUID.randomUUID().toString();
- while (modelInstanceConcurrentMap.containsKey(id)) {
- id = UUID.randomUUID().toString();
- }
- Prediction prediction = new Prediction(id, startTime, finishTime, status, lensSessionHandle, modelInstanceId,
- inputDataSet, outputDataSet);
- predictionConcurrentMap.put(prediction.getId(), prediction);
- return prediction;
- }
-
- @Override
- public Prediction getPrediction(String predictionId) throws LensException {
- return predictionConcurrentMap.get(predictionId);
- }
-
- @Override
- public void updatePrediction(Prediction prediction) throws LensException {
- if (modelConcurrentMap.keySet().contains(prediction.getId())) {
- Prediction predictionDb = predictionConcurrentMap.get(prediction.getId());
- predictionDb.setStatus(prediction.getStatus());
- predictionDb.setFinishTime(prediction.getFinishTime());
- }
- }
-
- @Override
- public Evaluation createEvaluation(Date startTime, Date finishTime, Status status,
- LensSessionHandle lensSessionHandle,
- String modelInstanceId, String inputDataSetName)
- throws LensException {
- String id = UUID.randomUUID().toString();
- while (modelInstanceConcurrentMap.containsKey(id)) {
- id = UUID.randomUUID().toString();
- }
- Evaluation evaluation = new Evaluation(id, startTime, finishTime, status, lensSessionHandle, modelInstanceId,
- inputDataSetName);
- evaluationConcurrentMap.put(evaluation.getId(), evaluation);
- return evaluation;
- }
-
- @Override
- public List<Prediction> getIncompletePredictions() throws LensException {
- List<Prediction> incompletePrediction = new ArrayList<>();
- for (Prediction prediction : predictionConcurrentMap.values()) {
- if (!(prediction.getStatus() == Status.COMPLETED || prediction.getStatus() == Status.FAILED || prediction
- .getStatus() == Status.CANCELLED)) {
- incompletePrediction.add(prediction);
- }
- }
- return incompletePrediction;
- }
-
- @Override
- public List<ModelInstance> getModelInstances(String modelId) throws LensException {
- List<ModelInstance> modelInstances = new ArrayList<>();
- for (ModelInstance modelInstance : modelInstanceConcurrentMap.values()) {
- if (modelInstance.getModelId().compareTo(modelId) == 0) {
- modelInstances.add(modelInstance);
- }
- }
- return modelInstances;
- }
-
- @Override
- public List<Evaluation> getEvaluations(String modelInstanceId) throws LensException {
- List<Evaluation> evaluations = new ArrayList<>();
- for (Evaluation evaluation : evaluationConcurrentMap.values()) {
- if (evaluation.getModeInstanceId().compareTo(modelInstanceId) == 0) {
- evaluations.add(evaluation);
- }
- }
- return evaluations;
- }
-
- @Override
- public List<Prediction> getPredictions(String modelInstanceId) throws LensException {
- List<Prediction> predictions = new ArrayList<>();
- for (Prediction prediction : predictionConcurrentMap.values()) {
- if (prediction.getModelInstanceId().compareTo(modelInstanceId) == 0) {
- predictions.add(prediction);
- }
- }
- return predictions;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/ace0dd24/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java
new file mode 100644
index 0000000..d444a32
--- /dev/null
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/dao/MLDBUtils.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.ml.dao;
+
+import org.apache.lens.ml.algo.api.MLModel;
+import org.apache.lens.ml.api.MLTestReport;
+import org.apache.lens.ml.impl.MLTask;
+
+public class MLDBUtils {
+
+ /**
+ * Create table to store test report data
+ */
+ public void createTestReportTable() {
+
+ }
+
+ /**
+ * Create table to store ML task workflow data
+ */
+ public void createMLTaskTable() {
+
+ }
+
+ /**
+ * Create table to save ML Models
+ */
+ public void createMLModelTable() {
+
+ }
+
+ /**
+ * Insert an ML Task into ml task table
+ *
+ * @param task
+ */
+ public void saveMLTask(MLTask task) {
+
+ }
+
+ /**
+ * Get ML Task given its id
+ *
+ * @param taskID
+ * @return
+ */
+ public MLTask getMLTask(String taskID) {
+ return null;
+ }
+
+ /**
+ * Insert test report into test report table
+ *
+ * @param testReport
+ */
+ public void saveTestReport(MLTestReport testReport) {
+
+ }
+
+ /**
+ * Get test report given its ID
+ *
+ * @param testReportID
+ * @return
+ */
+ public MLTestReport getTestReport(String testReportID) {
+ return null;
+ }
+
+ /**
+ * Insert model metadata into model table
+ *
+ * @param mlModel
+ */
+ public void saveMLModel(MLModel<?> mlModel) {
+
+ }
+
+ /**
+ * Get model metadata given ID
+ *
+ * @param modelID
+ * @return
+ */
+ public MLModel<?> getMLModel(String modelID) {
+ return null;
+ }
+
+}
[38/50] [abbrv] incubator-lens git commit: LENS-477 : Add Lens server
health check
Posted by sh...@apache.org.
LENS-477 : Add Lens server health check
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/651a9b6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/651a9b6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/651a9b6a
Branch: refs/heads/LENS-581
Commit: 651a9b6a41401fb637bc8130489f18654c0bca1f
Parents: ad8f4a8
Author: Yash Sharma <ya...@gmail.com>
Authored: Mon Jul 27 10:21:35 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Mon Jul 27 10:21:35 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/api/LensService.java | 25 ++
.../server/api/events/LensEventService.java | 4 +-
.../lens/server/api/health/HealthCheckable.java | 29 ++
.../lens/server/api/health/HealthStatus.java | 32 ++
.../api/metastore/CubeMetastoreService.java | 3 +-
.../lens/server/api/metrics/MetricsService.java | 5 +-
.../org/apache/lens/server/BaseLensService.java | 433 +++++++++++++++++++
.../apache/lens/server/EventServiceImpl.java | 10 +
.../org/apache/lens/server/LensService.java | 424 ------------------
.../org/apache/lens/server/LensServices.java | 19 +-
.../healthcheck/LensServiceHealthCheck.java | 45 ++
.../metastore/CubeMetastoreServiceImpl.java | 36 +-
.../lens/server/metrics/MetricsServiceImpl.java | 44 +-
.../server/query/QueryExecutionServiceImpl.java | 57 ++-
.../lens/server/quota/QuotaServiceImpl.java | 21 +-
.../scheduler/QuerySchedulerServiceImpl.java | 22 +-
.../lens/server/session/HiveSessionService.java | 20 +-
.../lens/server/session/SessionResource.java | 4 +-
.../apache/lens/server/TestServerRestart.java | 10 +
.../server/healthcheck/TestHealthChecks.java | 127 ++++++
src/site/apt/user/cli.apt | 8 +-
21 files changed, 920 insertions(+), 458 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server-api/src/main/java/org/apache/lens/server/api/LensService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensService.java
new file mode 100644
index 0000000..2098c3e
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensService.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api;
+
+import org.apache.lens.server.api.health.HealthCheckable;
+
+public interface LensService extends HealthCheckable {
+ //TODO: Pull out method definitions from individual services
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEventService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEventService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEventService.java
index 74cc00c..4536a18 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEventService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/events/LensEventService.java
@@ -20,12 +20,13 @@ package org.apache.lens.server.api.events;
import java.util.Collection;
+import org.apache.lens.server.api.LensService;
import org.apache.lens.server.api.error.LensException;
/**
* Singleton which is responsible for managing the event-listener mapping as well as processing events.
*/
-public interface LensEventService {
+public interface LensEventService extends LensService {
/**
* The Constant NAME.
@@ -71,4 +72,5 @@ public interface LensEventService {
* @return all the listeners
*/
<T extends LensEvent> Collection<LensEventListener> getListeners(Class<T> changeType);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthCheckable.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthCheckable.java b/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthCheckable.java
new file mode 100644
index 0000000..667165b
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthCheckable.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.health;
+
+public interface HealthCheckable {
+
+ /**
+ * Returns the health status of the service.
+ *
+ * @return
+ */
+ HealthStatus getHealthStatus();
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthStatus.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthStatus.java b/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthStatus.java
new file mode 100644
index 0000000..21ce9da
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/health/HealthStatus.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.health;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class HealthStatus {
+
+ @Getter
+ private final boolean isHealthy;
+ @Getter
+ private final String details;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
index bd31f82..d7a033e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.metastore.*;
+import org.apache.lens.server.api.LensService;
import org.apache.lens.server.api.error.LensException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -30,7 +31,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
/**
* Server api for OLAP Cube Metastore.
*/
-public interface CubeMetastoreService {
+public interface CubeMetastoreService extends LensService {
/** The constant NAME */
String NAME = "metastore";
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
index 72fe455..7fd2d81 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MetricsService.java
@@ -18,13 +18,15 @@
*/
package org.apache.lens.server.api.metrics;
+import org.apache.lens.server.api.LensService;
+
import org.glassfish.jersey.server.ContainerRequest;
import org.glassfish.jersey.server.model.ResourceMethod;
/**
* The Interface MetricsService.
*/
-public interface MetricsService {
+public interface MetricsService extends LensService {
/**
* The Constant NAME.
@@ -193,4 +195,5 @@ public interface MetricsService {
* Specifies the count of all opened sessions which are not closed after restarted and the restored sessions
*/
int getActiveSessions();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
new file mode 100644
index 0000000..da11069
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/BaseLensService.java
@@ -0,0 +1,433 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.ClientErrorException;
+import javax.ws.rs.NotAuthorizedException;
+import javax.ws.rs.NotFoundException;
+
+import org.apache.lens.api.LensConf;
+import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.LensService;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.events.LensEvent;
+import org.apache.lens.server.api.events.LensEventService;
+import org.apache.lens.server.api.health.HealthStatus;
+import org.apache.lens.server.session.LensSessionImpl;
+import org.apache.lens.server.user.UserConfigLoaderFactory;
+import org.apache.lens.server.util.UtilityMethods;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
+import org.apache.hive.service.CompositeService;
+import org.apache.hive.service.auth.AuthenticationProviderFactory;
+import org.apache.hive.service.auth.HiveAuthFactory;
+import org.apache.hive.service.auth.PasswdAuthenticationProvider;
+import org.apache.hive.service.cli.CLIService;
+import org.apache.hive.service.cli.HandleIdentifier;
+import org.apache.hive.service.cli.HiveSQLException;
+import org.apache.hive.service.cli.SessionHandle;
+import org.apache.hive.service.cli.session.SessionManager;
+import org.apache.hive.service.cli.thrift.TSessionHandle;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * The Class LensService.
+ */
+@Slf4j
+public abstract class BaseLensService extends CompositeService implements Externalizable, LensService {
+
+ /** The cli service. */
+ private final CLIService cliService;
+
+ /** The stopped. */
+ protected boolean stopped = false;
+
+ // Static session map which is used by query submission thread to get the
+ // lens session before submitting a query to hive server
+ /** The session map. */
+ protected static final ConcurrentHashMap<String, LensSessionHandle> SESSION_MAP
+ = new ConcurrentHashMap<String, LensSessionHandle>();
+
+ /**
+ * Instantiates a new lens service.
+ *
+ * @param name the name
+ * @param cliService the cli service
+ */
+ protected BaseLensService(String name, CLIService cliService) {
+ super(name);
+ this.cliService = cliService;
+ }
+
+ /**
+ * @return the cliService
+ */
+ public CLIService getCliService() {
+ return cliService;
+ }
+
+ public String getServerDomain() {
+ return cliService.getHiveConf().get(LensConfConstants.SERVER_DOMAIN);
+ }
+
+ public static int getNumberOfSessions() {
+ return BaseLensService.SESSION_MAP.size();
+ }
+
+ /**
+ * Open session.
+ *
+ * @param username the username
+ * @param password the password
+ * @param configuration the configuration
+ * @return the lens session handle
+ * @throws LensException the lens exception
+ */
+ public LensSessionHandle openSession(String username, String password, Map<String, String> configuration)
+ throws LensException {
+ if (StringUtils.isBlank(username)) {
+ throw new BadRequestException("User name cannot be null or empty");
+ }
+ SessionHandle sessionHandle;
+ username = UtilityMethods.removeDomain(username);
+ doPasswdAuth(username, password);
+ try {
+ Map<String, String> sessionConf = new HashMap<String, String>();
+ sessionConf.putAll(LensSessionImpl.DEFAULT_HIVE_SESSION_CONF);
+ if (configuration != null) {
+ sessionConf.putAll(configuration);
+ }
+ Map<String, String> userConfig = UserConfigLoaderFactory.getUserConfig(username);
+ log.info("Got user config: {}", userConfig);
+ UtilityMethods.mergeMaps(sessionConf, userConfig, false);
+ sessionConf.put(LensConfConstants.SESSION_LOGGEDIN_USER, username);
+ if (sessionConf.get(LensConfConstants.SESSION_CLUSTER_USER) == null) {
+ log.info("Didn't get cluster user from user config loader. Setting same as logged in user: {}", username);
+ sessionConf.put(LensConfConstants.SESSION_CLUSTER_USER, username);
+ }
+ String clusterUser = sessionConf.get(LensConfConstants.SESSION_CLUSTER_USER);
+ password = "useless";
+ if (cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
+ .equals(HiveAuthFactory.AuthTypes.KERBEROS.toString())
+ && cliService.getHiveConf().getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS)) {
+ String delegationTokenStr = null;
+ try {
+ delegationTokenStr = cliService.getDelegationTokenFromMetaStore(username);
+ } catch (UnsupportedOperationException e) {
+ // The delegation token is not applicable in the given deployment mode
+ }
+ sessionHandle = cliService.openSessionWithImpersonation(clusterUser, password, sessionConf, delegationTokenStr);
+ } else {
+ sessionHandle = cliService.openSession(clusterUser, password, sessionConf);
+ }
+ } catch (Exception e) {
+ throw new LensException(e);
+ }
+ LensSessionHandle lensSession = new LensSessionHandle(sessionHandle.getHandleIdentifier().getPublicId(),
+ sessionHandle.getHandleIdentifier().getSecretId());
+ SESSION_MAP.put(lensSession.getPublicId().toString(), lensSession);
+ return lensSession;
+ }
+
+ protected LensEventService getEventService() {
+ LensEventService eventService = (LensEventService) LensServices.get().getService(LensEventService.NAME);
+ if (eventService == null) {
+ throw new NullPointerException("Could not get event service");
+ }
+ return eventService;
+ }
+
+ protected void notifyEvent(LensEvent event) throws LensException {
+ getEventService().notifyEvent(event);
+ }
+
+ /**
+ * Restore session from previous instance of lens server.
+ *
+ * @param sessionHandle the session handle
+ * @param userName the user name
+ * @param password the password
+ * @throws LensException the lens exception
+ */
+ public void restoreSession(LensSessionHandle sessionHandle, String userName, String password) throws LensException {
+ HandleIdentifier handleIdentifier = new HandleIdentifier(sessionHandle.getPublicId(), sessionHandle.getSecretId());
+ SessionHandle hiveSessionHandle = new SessionHandle(new TSessionHandle(handleIdentifier.toTHandleIdentifier()));
+ try {
+ SessionHandle restoredHandle = cliService.restoreSession(hiveSessionHandle, userName, password,
+ new HashMap<String, String>());
+ LensSessionHandle restoredSession = new LensSessionHandle(restoredHandle.getHandleIdentifier().getPublicId(),
+ restoredHandle.getHandleIdentifier().getSecretId());
+ SESSION_MAP.put(restoredSession.getPublicId().toString(), restoredSession);
+ } catch (HiveSQLException e) {
+ throw new LensException("Error restoring session " + sessionHandle, e);
+ }
+ }
+
+ /**
+ * Do passwd auth.
+ *
+ * @param userName the user name
+ * @param password the password
+ */
+ private void doPasswdAuth(String userName, String password) {
+ // Lens confs to Hive Confs.
+ for (ConfVars var : new ConfVars[]{ConfVars.HIVE_SERVER2_PLAIN_LDAP_DOMAIN}) {
+ if (cliService.getHiveConf().getVar(var) == null) {
+ cliService.getHiveConf().setVar(var, cliService.getHiveConf().get(LensConfConstants.SERVER_DOMAIN));
+ }
+ }
+ String authType = cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION);
+ // No-op when authType is NOSASL
+ if (!authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.NOSASL.toString())) {
+ try {
+ AuthenticationProviderFactory.AuthMethods authMethod = AuthenticationProviderFactory.AuthMethods
+ .getValidAuthMethod(authType);
+ PasswdAuthenticationProvider provider = AuthenticationProviderFactory.getAuthenticationProvider(authMethod,
+ cliService.getHiveConf());
+ provider.Authenticate(userName, password);
+ } catch (Exception e) {
+ log.error("Auth error: ", e);
+ throw new NotAuthorizedException(e);
+ }
+ }
+ }
+
+ /**
+ * Close session.
+ *
+ * @param sessionHandle the session handle
+ * @throws LensException the lens exception
+ */
+ public void closeSession(LensSessionHandle sessionHandle) throws LensException {
+ try {
+ cliService.closeSession(getHiveSessionHandle(sessionHandle));
+ SESSION_MAP.remove(sessionHandle.getPublicId().toString());
+ } catch (Exception e) {
+ throw new LensException(e);
+ }
+ }
+
+ public SessionManager getSessionManager() {
+ return cliService.getSessionManager();
+ }
+
+ /**
+ * Gets the session.
+ *
+ * @param sessionHandle the session handle
+ * @return the session
+ */
+ public LensSessionImpl getSession(LensSessionHandle sessionHandle) {
+ if (sessionHandle == null) {
+ throw new ClientErrorException("Session is null " + sessionHandle, 400);
+ }
+
+ try {
+ return ((LensSessionImpl) getSessionManager().getSession(getHiveSessionHandle(sessionHandle)));
+ } catch (HiveSQLException exc) {
+ log.warn("Session {} not found", sessionHandle.getPublicId(), exc);
+ // throw resource gone exception (410)
+ throw new ClientErrorException("Session " + sessionHandle.getPublicId() + " is invalid " + sessionHandle, 410);
+ }
+ }
+
+ /**
+ * Acquire.
+ *
+ * @param sessionHandle the session handle
+ */
+ public void acquire(LensSessionHandle sessionHandle) {
+ if (sessionHandle != null) {
+ log.debug("Acquiring lens session:{}", sessionHandle.getPublicId());
+ getSession(sessionHandle).acquire();
+ }
+ }
+
+ /**
+ * Acquire a lens session specified by the public UUID.
+ *
+ * @param sessionHandle public UUID of the session
+ */
+ public void acquire(String sessionHandle) {
+ LensSessionHandle handle = SESSION_MAP.get(sessionHandle);
+
+ if (handle == null) {
+ throw new NotFoundException("Session handle not found " + sessionHandle);
+ }
+
+ acquire(handle);
+ }
+
+ /**
+ * Release.
+ *
+ * @param sessionHandle the session handle
+ */
+ public void release(LensSessionHandle sessionHandle) {
+ if (sessionHandle != null) {
+ getSession(sessionHandle).release();
+ log.debug("Released lens session:{}", sessionHandle.getPublicId());
+ }
+ }
+
+ /**
+ * Releases a lens session specified by the public UUID.
+ *
+ * @param sessionHandle the session handle
+ * @throws LensException if session cannot be released
+ */
+ public void release(String sessionHandle) throws LensException {
+ LensSessionHandle handle = SESSION_MAP.get(sessionHandle);
+ if (handle != null) {
+ getSession(handle).release();
+ }
+ }
+
+ /**
+ * Gets the session handle.
+ *
+ * @param sessionid the sessionid
+ * @return the session handle
+ */
+ protected LensSessionHandle getSessionHandle(String sessionid) {
+ return SESSION_MAP.get(sessionid);
+ }
+
+ /**
+ * Adds the resource.
+ *
+ * @param sessionHandle the session handle
+ * @param type the type
+ * @param path the path
+ * @throws LensException the lens exception
+ */
+ public void addResource(LensSessionHandle sessionHandle, String type, String path) throws LensException {
+ }
+
+ /**
+ * Delete resource.
+ *
+ * @param sessionHandle the session handle
+ * @param type the type
+ * @param path the path
+ * @throws LensException the lens exception
+ */
+ public void deleteResource(LensSessionHandle sessionHandle, String type, String path) throws LensException {
+ }
+
+ /**
+ * Gets the hive session handle.
+ *
+ * @param lensHandle the lens handle
+ * @return the hive session handle
+ */
+ public static SessionHandle getHiveSessionHandle(LensSessionHandle lensHandle) {
+ return new SessionHandle(new HandleIdentifier(lensHandle.getPublicId(), lensHandle.getSecretId()),
+ CLIService.SERVER_VERSION);
+ }
+
+ /**
+ * Gets the lens conf.
+ *
+ * @param sessionHandle the session handle
+ * @param conf the conf
+ * @return the lens conf
+ * @throws LensException the lens exception
+ */
+ public Configuration getLensConf(LensSessionHandle sessionHandle, LensConf conf) throws LensException {
+ Configuration qconf = new Configuration(false);
+ for (Map.Entry<String, String> entry : getSession(sessionHandle).getSessionConf()) {
+ qconf.set(entry.getKey(), entry.getValue());
+ }
+
+ if (conf != null && !conf.getProperties().isEmpty()) {
+ for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
+ qconf.set(entry.getKey(), entry.getValue());
+ }
+ }
+ qconf.setClassLoader(getSession(sessionHandle).getClassLoader());
+ return qconf;
+ }
+
+ /**
+ * Gets the lens conf.
+ *
+ * @param conf the conf
+ * @return the lens conf
+ * @throws LensException the lens exception
+ */
+ public Configuration getLensConf(LensConf conf) throws LensException {
+ Configuration qconf = LensSessionImpl.createDefaultConf();
+
+ if (conf != null && !conf.getProperties().isEmpty()) {
+ for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
+ qconf.set(entry.getKey(), entry.getValue());
+ }
+ }
+ return qconf;
+ }
+
+ /**
+ * Prepare stopping.
+ */
+ public void prepareStopping() {
+ this.stopped = true;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.io.Externalizable#readExternal(java.io.ObjectInput)
+ */
+ @Override
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput)
+ */
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ }
+
+ /**
+ * Returns the health status of the service.
+ *
+ * @return
+ */
+ public abstract HealthStatus getHealthStatus();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
index 2c6f912..a276828 100644
--- a/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/EventServiceImpl.java
@@ -27,6 +27,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEvent;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.events.LensEventService;
+import org.apache.lens.server.api.health.HealthStatus;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.AbstractService;
@@ -171,6 +172,15 @@ public class EventServiceImpl extends AbstractService implements LensEventServic
return Collections.unmodifiableList(eventListeners.get(eventType));
}
+ @Override
+ public HealthStatus getHealthStatus() {
+ return (this.getServiceState().equals(STATE.STARTED)
+ && !eventHandlerPool.isShutdown()
+ && !eventHandlerPool.isTerminated())
+ ? new HealthStatus(true, "Event service is healthy.")
+ : new HealthStatus(false, "Event service is unhealthy.");
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/LensService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensService.java b/lens-server/src/main/java/org/apache/lens/server/LensService.java
deleted file mode 100644
index 0e564a9..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/LensService.java
+++ /dev/null
@@ -1,424 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import javax.ws.rs.BadRequestException;
-import javax.ws.rs.ClientErrorException;
-import javax.ws.rs.NotAuthorizedException;
-import javax.ws.rs.NotFoundException;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.events.LensEvent;
-import org.apache.lens.server.api.events.LensEventService;
-import org.apache.lens.server.session.LensSessionImpl;
-import org.apache.lens.server.user.UserConfigLoaderFactory;
-import org.apache.lens.server.util.UtilityMethods;
-
-import org.apache.commons.lang3.StringUtils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-
-import org.apache.hive.service.CompositeService;
-import org.apache.hive.service.auth.AuthenticationProviderFactory;
-import org.apache.hive.service.auth.HiveAuthFactory;
-import org.apache.hive.service.auth.PasswdAuthenticationProvider;
-import org.apache.hive.service.cli.CLIService;
-import org.apache.hive.service.cli.HandleIdentifier;
-import org.apache.hive.service.cli.HiveSQLException;
-import org.apache.hive.service.cli.SessionHandle;
-import org.apache.hive.service.cli.session.SessionManager;
-import org.apache.hive.service.cli.thrift.TSessionHandle;
-
-import lombok.extern.slf4j.Slf4j;
-
-/**
- * The Class LensService.
- */
-@Slf4j
-public abstract class LensService extends CompositeService implements Externalizable {
-
- /** The cli service. */
- private final CLIService cliService;
-
- /** The stopped. */
- protected boolean stopped = false;
-
- // Static session map which is used by query submission thread to get the
- // lens session before submitting a query to hive server
- /** The session map. */
- protected static final ConcurrentHashMap<String, LensSessionHandle> SESSION_MAP
- = new ConcurrentHashMap<String, LensSessionHandle>();
-
- /**
- * Instantiates a new lens service.
- *
- * @param name the name
- * @param cliService the cli service
- */
- protected LensService(String name, CLIService cliService) {
- super(name);
- this.cliService = cliService;
- }
-
- /**
- * @return the cliService
- */
- public CLIService getCliService() {
- return cliService;
- }
-
- public String getServerDomain() {
- return cliService.getHiveConf().get(LensConfConstants.SERVER_DOMAIN);
- }
-
- public static int getNumberOfSessions() {
- return LensService.SESSION_MAP.size();
- }
-
- /**
- * Open session.
- *
- * @param username the username
- * @param password the password
- * @param configuration the configuration
- * @return the lens session handle
- * @throws LensException the lens exception
- */
- public LensSessionHandle openSession(String username, String password, Map<String, String> configuration)
- throws LensException {
- if (StringUtils.isBlank(username)) {
- throw new BadRequestException("User name cannot be null or empty");
- }
- SessionHandle sessionHandle;
- username = UtilityMethods.removeDomain(username);
- doPasswdAuth(username, password);
- try {
- Map<String, String> sessionConf = new HashMap<String, String>();
- sessionConf.putAll(LensSessionImpl.DEFAULT_HIVE_SESSION_CONF);
- if (configuration != null) {
- sessionConf.putAll(configuration);
- }
- Map<String, String> userConfig = UserConfigLoaderFactory.getUserConfig(username);
- log.info("Got user config: {}", userConfig);
- UtilityMethods.mergeMaps(sessionConf, userConfig, false);
- sessionConf.put(LensConfConstants.SESSION_LOGGEDIN_USER, username);
- if (sessionConf.get(LensConfConstants.SESSION_CLUSTER_USER) == null) {
- log.info("Didn't get cluster user from user config loader. Setting same as logged in user: {}", username);
- sessionConf.put(LensConfConstants.SESSION_CLUSTER_USER, username);
- }
- String clusterUser = sessionConf.get(LensConfConstants.SESSION_CLUSTER_USER);
- password = "useless";
- if (cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION)
- .equals(HiveAuthFactory.AuthTypes.KERBEROS.toString())
- && cliService.getHiveConf().getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS)) {
- String delegationTokenStr = null;
- try {
- delegationTokenStr = cliService.getDelegationTokenFromMetaStore(username);
- } catch (UnsupportedOperationException e) {
- // The delegation token is not applicable in the given deployment mode
- }
- sessionHandle = cliService.openSessionWithImpersonation(clusterUser, password, sessionConf, delegationTokenStr);
- } else {
- sessionHandle = cliService.openSession(clusterUser, password, sessionConf);
- }
- } catch (Exception e) {
- throw new LensException(e);
- }
- LensSessionHandle lensSession = new LensSessionHandle(sessionHandle.getHandleIdentifier().getPublicId(),
- sessionHandle.getHandleIdentifier().getSecretId());
- SESSION_MAP.put(lensSession.getPublicId().toString(), lensSession);
- return lensSession;
- }
-
- protected LensEventService getEventService() {
- LensEventService eventService = (LensEventService) LensServices.get().getService(LensEventService.NAME);
- if (eventService == null) {
- throw new NullPointerException("Could not get event service");
- }
- return eventService;
- }
-
- protected void notifyEvent(LensEvent event) throws LensException {
- getEventService().notifyEvent(event);
- }
-
- /**
- * Restore session from previous instance of lens server.
- *
- * @param sessionHandle the session handle
- * @param userName the user name
- * @param password the password
- * @throws LensException the lens exception
- */
- public void restoreSession(LensSessionHandle sessionHandle, String userName, String password) throws LensException {
- HandleIdentifier handleIdentifier = new HandleIdentifier(sessionHandle.getPublicId(), sessionHandle.getSecretId());
- SessionHandle hiveSessionHandle = new SessionHandle(new TSessionHandle(handleIdentifier.toTHandleIdentifier()));
- try {
- SessionHandle restoredHandle = cliService.restoreSession(hiveSessionHandle, userName, password,
- new HashMap<String, String>());
- LensSessionHandle restoredSession = new LensSessionHandle(restoredHandle.getHandleIdentifier().getPublicId(),
- restoredHandle.getHandleIdentifier().getSecretId());
- SESSION_MAP.put(restoredSession.getPublicId().toString(), restoredSession);
- } catch (HiveSQLException e) {
- throw new LensException("Error restoring session " + sessionHandle, e);
- }
- }
-
- /**
- * Do passwd auth.
- *
- * @param userName the user name
- * @param password the password
- */
- private void doPasswdAuth(String userName, String password) {
- // Lens confs to Hive Confs.
- for (ConfVars var : new ConfVars[]{ConfVars.HIVE_SERVER2_PLAIN_LDAP_DOMAIN}) {
- if (cliService.getHiveConf().getVar(var) == null) {
- cliService.getHiveConf().setVar(var, cliService.getHiveConf().get(LensConfConstants.SERVER_DOMAIN));
- }
- }
- String authType = cliService.getHiveConf().getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION);
- // No-op when authType is NOSASL
- if (!authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.NOSASL.toString())) {
- try {
- AuthenticationProviderFactory.AuthMethods authMethod = AuthenticationProviderFactory.AuthMethods
- .getValidAuthMethod(authType);
- PasswdAuthenticationProvider provider = AuthenticationProviderFactory.getAuthenticationProvider(authMethod,
- cliService.getHiveConf());
- provider.Authenticate(userName, password);
- } catch (Exception e) {
- log.error("Auth error: ", e);
- throw new NotAuthorizedException(e);
- }
- }
- }
-
- /**
- * Close session.
- *
- * @param sessionHandle the session handle
- * @throws LensException the lens exception
- */
- public void closeSession(LensSessionHandle sessionHandle) throws LensException {
- try {
- cliService.closeSession(getHiveSessionHandle(sessionHandle));
- SESSION_MAP.remove(sessionHandle.getPublicId().toString());
- } catch (Exception e) {
- throw new LensException(e);
- }
- }
-
- public SessionManager getSessionManager() {
- return cliService.getSessionManager();
- }
-
- /**
- * Gets the session.
- *
- * @param sessionHandle the session handle
- * @return the session
- */
- public LensSessionImpl getSession(LensSessionHandle sessionHandle) {
- if (sessionHandle == null) {
- throw new ClientErrorException("Session is null " + sessionHandle, 400);
- }
-
- try {
- return ((LensSessionImpl) getSessionManager().getSession(getHiveSessionHandle(sessionHandle)));
- } catch (HiveSQLException exc) {
- log.warn("Session {} not found", sessionHandle.getPublicId(), exc);
- // throw resource gone exception (410)
- throw new ClientErrorException("Session " + sessionHandle.getPublicId() + " is invalid " + sessionHandle, 410);
- }
- }
-
- /**
- * Acquire.
- *
- * @param sessionHandle the session handle
- */
- public void acquire(LensSessionHandle sessionHandle) {
- if (sessionHandle != null) {
- log.debug("Acquiring lens session:{}", sessionHandle.getPublicId());
- getSession(sessionHandle).acquire();
- }
- }
-
- /**
- * Acquire a lens session specified by the public UUID.
- *
- * @param sessionHandle public UUID of the session
- */
- public void acquire(String sessionHandle) {
- LensSessionHandle handle = SESSION_MAP.get(sessionHandle);
-
- if (handle == null) {
- throw new NotFoundException("Session handle not found " + sessionHandle);
- }
-
- acquire(handle);
- }
-
- /**
- * Release.
- *
- * @param sessionHandle the session handle
- */
- public void release(LensSessionHandle sessionHandle) {
- if (sessionHandle != null) {
- getSession(sessionHandle).release();
- log.debug("Released lens session:{}", sessionHandle.getPublicId());
- }
- }
-
- /**
- * Releases a lens session specified by the public UUID.
- *
- * @param sessionHandle the session handle
- * @throws LensException if session cannot be released
- */
- public void release(String sessionHandle) throws LensException {
- LensSessionHandle handle = SESSION_MAP.get(sessionHandle);
- if (handle != null) {
- getSession(handle).release();
- }
- }
-
- /**
- * Gets the session handle.
- *
- * @param sessionid the sessionid
- * @return the session handle
- */
- protected LensSessionHandle getSessionHandle(String sessionid) {
- return SESSION_MAP.get(sessionid);
- }
-
- /**
- * Adds the resource.
- *
- * @param sessionHandle the session handle
- * @param type the type
- * @param path the path
- * @throws LensException the lens exception
- */
- public void addResource(LensSessionHandle sessionHandle, String type, String path) throws LensException {
- }
-
- /**
- * Delete resource.
- *
- * @param sessionHandle the session handle
- * @param type the type
- * @param path the path
- * @throws LensException the lens exception
- */
- public void deleteResource(LensSessionHandle sessionHandle, String type, String path) throws LensException {
- }
-
- /**
- * Gets the hive session handle.
- *
- * @param lensHandle the lens handle
- * @return the hive session handle
- */
- public static SessionHandle getHiveSessionHandle(LensSessionHandle lensHandle) {
- return new SessionHandle(new HandleIdentifier(lensHandle.getPublicId(), lensHandle.getSecretId()),
- CLIService.SERVER_VERSION);
- }
-
- /**
- * Gets the lens conf.
- *
- * @param sessionHandle the session handle
- * @param conf the conf
- * @return the lens conf
- * @throws LensException the lens exception
- */
- public Configuration getLensConf(LensSessionHandle sessionHandle, LensConf conf) throws LensException {
- Configuration qconf = new Configuration(false);
- for (Map.Entry<String, String> entry : getSession(sessionHandle).getSessionConf()) {
- qconf.set(entry.getKey(), entry.getValue());
- }
-
- if (conf != null && !conf.getProperties().isEmpty()) {
- for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
- qconf.set(entry.getKey(), entry.getValue());
- }
- }
- qconf.setClassLoader(getSession(sessionHandle).getClassLoader());
- return qconf;
- }
-
- /**
- * Gets the lens conf.
- *
- * @param conf the conf
- * @return the lens conf
- * @throws LensException the lens exception
- */
- public Configuration getLensConf(LensConf conf) throws LensException {
- Configuration qconf = LensSessionImpl.createDefaultConf();
-
- if (conf != null && !conf.getProperties().isEmpty()) {
- for (Map.Entry<String, String> entry : conf.getProperties().entrySet()) {
- qconf.set(entry.getKey(), entry.getValue());
- }
- }
- return qconf;
- }
-
- /**
- * Prepare stopping.
- */
- public void prepareStopping() {
- this.stopped = true;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.io.Externalizable#readExternal(java.io.ObjectInput)
- */
- @Override
- public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.io.Externalizable#writeExternal(java.io.ObjectOutput)
- */
- @Override
- public void writeExternal(ObjectOutput out) throws IOException {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index 1bcc5944..c4a61c8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -80,7 +80,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
private final Map<String, Service> services = new LinkedHashMap<String, Service>();
/** The lens services. */
- private final List<LensService> lensServices = new ArrayList<LensService>();
+ private final List<BaseLensService> lensServices = new ArrayList<BaseLensService>();
/** The persist dir. */
private Path persistDir;
@@ -209,11 +209,11 @@ public class LensServices extends CompositeService implements ServiceProvider {
Class<?> cls = Class.forName(serviceClassName);
- if (LensService.class.isAssignableFrom(cls)) {
- Class<? extends LensService> serviceClass = (Class<? extends LensService>) cls;
+ if (BaseLensService.class.isAssignableFrom(cls)) {
+ Class<? extends BaseLensService> serviceClass = (Class<? extends BaseLensService>) cls;
log.info("Adding {} service with {}", sName, serviceClass);
Constructor<?> constructor = serviceClass.getConstructor(CLIService.class);
- LensService service = (LensService) constructor.newInstance(new Object[]{cliService});
+ BaseLensService service = (BaseLensService) constructor.newInstance(new Object[]{cliService});
addService(service);
lensServices.add(service);
} else if (Service.class.isAssignableFrom(cls)) {
@@ -297,7 +297,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
if (conf.getBoolean(SERVER_RECOVER_ON_RESTART,
DEFAULT_SERVER_RECOVER_ON_RESTART)) {
- for (LensService service : lensServices) {
+ for (BaseLensService service : lensServices) {
ObjectInputStream in = null;
try {
try {
@@ -330,7 +330,8 @@ public class LensServices extends CompositeService implements ServiceProvider {
log.info("Persisting server state in {}", persistDir);
long now = System.currentTimeMillis();
- for (LensService service : lensServices) {
+
+ for (BaseLensService service : lensServices) {
log.info("Persisting state of service: {}", service.getName());
Path serviceWritePath = new Path(persistDir, service.getName() + ".out" + "." + now);
ObjectOutputStream out = null;
@@ -369,7 +370,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
* @param service the service
* @return the service persist path
*/
- private Path getServicePersistPath(LensService service) {
+ private Path getServicePersistPath(BaseLensService service) {
return new Path(persistDir, service.getName() + ".final");
}
@@ -382,7 +383,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
if (getServiceState() != STATE.STOPPED) {
log.info("Stopping lens server");
stopping = true;
- for (LensService service : lensServices) {
+ for (BaseLensService service : lensServices) {
service.prepareStopping();
}
@@ -434,7 +435,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
return (T) services.get(sName);
}
- public List<LensService> getLensServices() {
+ public List<BaseLensService> getLensServices() {
return lensServices;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/healthcheck/LensServiceHealthCheck.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/healthcheck/LensServiceHealthCheck.java b/lens-server/src/main/java/org/apache/lens/server/healthcheck/LensServiceHealthCheck.java
new file mode 100644
index 0000000..b811b66
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/healthcheck/LensServiceHealthCheck.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.healthcheck;
+
+import org.apache.lens.server.LensServices;
+import org.apache.lens.server.api.LensService;
+import org.apache.lens.server.api.health.HealthStatus;
+
+import com.codahale.metrics.health.HealthCheck;
+
+public class LensServiceHealthCheck extends HealthCheck {
+
+ private final LensService service;
+
+ public LensServiceHealthCheck(String serviceName) {
+ this.service = LensServices.get().getService(serviceName);
+ }
+
+ @Override
+ protected Result check() throws Exception {
+ HealthStatus status = service.getHealthStatus();
+
+ if (status.isHealthy()) {
+ return Result.healthy();
+ } else {
+ return Result.unhealthy(status.getDetails());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index 646de8c..cb7a530 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -28,8 +28,10 @@ import org.apache.lens.api.metastore.*;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.metadata.Dimension;
import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
-import org.apache.lens.server.LensService;
+import org.apache.lens.server.BaseLensService;
+import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.health.HealthStatus;
import org.apache.lens.server.api.metastore.CubeMetastoreService;
import org.apache.lens.server.session.LensSessionImpl;
@@ -47,7 +49,7 @@ import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class CubeMetastoreServiceImpl extends LensService implements CubeMetastoreService {
+public class CubeMetastoreServiceImpl extends BaseLensService implements CubeMetastoreService {
public CubeMetastoreServiceImpl(CLIService cliService) {
super(NAME, cliService);
@@ -1470,4 +1472,34 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
release(sessionHandle);
}
}
+
+ /**
+ * @inheritDoc
+ */
+ @Override
+ public HealthStatus getHealthStatus() {
+ boolean isHealthy = true;
+ StringBuilder details = new StringBuilder();
+
+ try{
+ /** Try to issue command on hive **/
+ Hive.get(LensServerConf.getHiveConf()).getAllDatabases();
+ } catch (HiveException e) {
+ isHealthy = false;
+ details.append("Could not connect to Hive.");
+ log.error("Could not connect to Hive.", e);
+ }
+
+ /** Check if service is up **/
+ if (!this.getServiceState().equals(STATE.STARTED)) {
+ isHealthy = false;
+ details.append("Cube metastore service is down");
+ log.error("Cube metastore service is down");
+ }
+
+ return isHealthy
+ ? new HealthStatus(isHealthy, "Cube metastore service is healthy.")
+ : new HealthStatus(isHealthy, details.toString());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index 2455232..e430324 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -29,11 +29,14 @@ import java.util.Locale;
import java.util.concurrent.TimeUnit;
import org.apache.lens.api.query.QueryStatus.Status;
-import org.apache.lens.server.LensService;
+import org.apache.lens.server.BaseLensService;
+import org.apache.lens.server.EventServiceImpl;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.events.LensEventService;
+import org.apache.lens.server.api.health.HealthStatus;
+import org.apache.lens.server.api.metastore.CubeMetastoreService;
import org.apache.lens.server.api.metrics.*;
import org.apache.lens.server.api.query.QueryExecutionService;
import org.apache.lens.server.api.query.StatusChange;
@@ -42,7 +45,12 @@ import org.apache.lens.server.api.session.SessionEvent;
import org.apache.lens.server.api.session.SessionExpired;
import org.apache.lens.server.api.session.SessionOpened;
import org.apache.lens.server.api.session.SessionService;
+import org.apache.lens.server.healthcheck.LensServiceHealthCheck;
+import org.apache.lens.server.query.QueryExecutionServiceImpl;
+import org.apache.lens.server.quota.QuotaServiceImpl;
+import org.apache.lens.server.scheduler.QuerySchedulerServiceImpl;
import org.apache.lens.server.session.DatabaseResourceService;
+import org.apache.lens.server.session.HiveSessionService;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.service.AbstractService;
@@ -252,6 +260,13 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
methodMetricsFactory = new MethodMetricsFactory(metricRegistry);
setEnableResourceMethodMetering(hiveConf.getBoolean(LensConfConstants.ENABLE_RESOURCE_METHOD_METERING, false));
healthCheck = new HealthCheckRegistry();
+ healthCheck.register(CubeMetastoreService.NAME, new LensServiceHealthCheck(CubeMetastoreService.NAME));
+ healthCheck.register(HiveSessionService.NAME, new LensServiceHealthCheck(HiveSessionService.NAME));
+ healthCheck.register(QueryExecutionServiceImpl.NAME, new LensServiceHealthCheck(QueryExecutionServiceImpl.NAME));
+ healthCheck.register(QuerySchedulerServiceImpl.NAME, new LensServiceHealthCheck(QuerySchedulerServiceImpl.NAME));
+ healthCheck.register(QuotaServiceImpl.NAME, new LensServiceHealthCheck(QuotaServiceImpl.NAME));
+ healthCheck.register(MetricsServiceImpl.NAME, new LensServiceHealthCheck(MetricsServiceImpl.NAME));
+ healthCheck.register(EventServiceImpl.NAME, new LensServiceHealthCheck(EventServiceImpl.NAME));
initCounters();
timeBetweenPolls = hiveConf.getInt(LensConfConstants.REPORTING_PERIOD, 10);
@@ -313,7 +328,7 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
new Gauge<Integer>() {
@Override
public Integer getValue() {
- return LensService.getNumberOfSessions();
+ return BaseLensService.getNumberOfSessions();
}
});
@@ -554,6 +569,31 @@ public class MetricsServiceImpl extends AbstractService implements MetricsServic
}
@Override
+ public HealthStatus getHealthStatus() {
+ boolean isHealthy = true;
+ StringBuilder details = new StringBuilder();
+
+ if (!this.getServiceState().equals(STATE.STARTED)) {
+ details.append("Metric service is down.");
+ isHealthy = false;
+ }
+
+ // Also unhealthy if 30% of queries have failed.
+ if (getTotalFailedQueries()/(float)getTotalAcceptedQueries() > 0.3) {
+ details.append("30% of queries have failed.");
+ isHealthy = false;
+ }
+
+ if (!isHealthy) {
+ log.error(details.toString());
+ }
+
+ return isHealthy
+ ? new HealthStatus(true, "Metric service is healthy.")
+ : new HealthStatus(false, details.toString());
+ }
+
+ @Override
public long getTotalOpenedSessions() {
return totalOpenedSessions.getCount();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 5e0f24a..1316257 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -44,13 +44,14 @@ import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
import org.apache.lens.api.result.LensErrorTO;
import org.apache.lens.driver.hive.HiveDriver;
+import org.apache.lens.server.BaseLensService;
import org.apache.lens.server.LensServerConf;
-import org.apache.lens.server.LensService;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.error.LensMultiCauseException;
import org.apache.lens.server.api.events.LensEventListener;
+import org.apache.lens.server.api.health.HealthStatus;
import org.apache.lens.server.api.metrics.MethodMetricsContext;
import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.metrics.MetricsService;
@@ -97,7 +98,7 @@ import lombok.extern.slf4j.Slf4j;
* The Class QueryExecutionServiceImpl.
*/
@Slf4j
-public class QueryExecutionServiceImpl extends LensService implements QueryExecutionService {
+public class QueryExecutionServiceImpl extends BaseLensService implements QueryExecutionService {
/**
* The Constant PREPARED_QUERIES_COUNTER.
@@ -2436,6 +2437,58 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
}
+ /**
+ * @inheritDoc
+ */
+ @Override
+ public HealthStatus getHealthStatus() {
+ boolean isHealthy = true;
+ StringBuilder details = new StringBuilder();
+
+ if (!this.getServiceState().equals(STATE.STARTED)) {
+ isHealthy = false;
+ details.append("Query execution service is down.");
+ }
+
+ if (!this.statusPoller.isAlive()) {
+ isHealthy = false;
+ details.append("Status poller thread is dead.");
+ }
+
+ if (!this.prepareQueryPurger.isAlive()) {
+ isHealthy = false;
+ details.append("PrepareQuery purger thread is dead.");
+ }
+
+ if (!this.queryPurger.isAlive()) {
+ isHealthy = false;
+ details.append("Query purger thread is dead.");
+ }
+
+ if (!this.querySubmitter.isAlive()) {
+ isHealthy = false;
+ details.append("Query submitter thread is dead.");
+ }
+
+ if (this.estimatePool.isShutdown() || this.estimatePool.isTerminated()) {
+ isHealthy = false;
+ details.append("Estimate Pool is dead.");
+ }
+
+ if (querySubmitterRunnable.pausedForTest) {
+ isHealthy = false;
+ details.append("QuerySubmitter paused for test.");
+ }
+
+ if (!isHealthy) {
+ log.error(details.toString());
+ }
+
+ return isHealthy
+ ? new HealthStatus(isHealthy, "QueryExecution service is healthy.")
+ : new HealthStatus(isHealthy, details.toString());
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/quota/QuotaServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/quota/QuotaServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/quota/QuotaServiceImpl.java
index 89cfe71..cc627cf 100644
--- a/lens-server/src/main/java/org/apache/lens/server/quota/QuotaServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/quota/QuotaServiceImpl.java
@@ -18,7 +18,8 @@
*/
package org.apache.lens.server.quota;
-import org.apache.lens.server.LensService;
+import org.apache.lens.server.BaseLensService;
+import org.apache.lens.server.api.health.HealthStatus;
import org.apache.lens.server.api.quota.QuotaService;
import org.apache.hive.service.cli.CLIService;
@@ -26,7 +27,12 @@ import org.apache.hive.service.cli.CLIService;
/**
* The Class QuotaServiceImpl.
*/
-public class QuotaServiceImpl extends LensService implements QuotaService {
+public class QuotaServiceImpl extends BaseLensService implements QuotaService {
+
+ /**
+ * The constant name for quota service.
+ */
+ public static final String NAME = "quota";
/**
* Instantiates a new quota service impl.
@@ -34,7 +40,16 @@ public class QuotaServiceImpl extends LensService implements QuotaService {
* @param cliService the cli service
*/
public QuotaServiceImpl(CLIService cliService) {
- super("quota", cliService);
+ super(NAME, cliService);
}
+ /**
+ * @inheritDoc
+ */
+ @Override
+ public HealthStatus getHealthStatus() {
+ return this.getServiceState().equals(STATE.STARTED)
+ ? new HealthStatus(true, "Quota service is healthy.")
+ : new HealthStatus(false, "Quota service is down.");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/scheduler/QuerySchedulerServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/scheduler/QuerySchedulerServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/scheduler/QuerySchedulerServiceImpl.java
index 32feeb9..3ea7524 100644
--- a/lens-server/src/main/java/org/apache/lens/server/scheduler/QuerySchedulerServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/scheduler/QuerySchedulerServiceImpl.java
@@ -18,7 +18,8 @@
*/
package org.apache.lens.server.scheduler;
-import org.apache.lens.server.LensService;
+import org.apache.lens.server.BaseLensService;
+import org.apache.lens.server.api.health.HealthStatus;
import org.apache.lens.server.api.scheduler.QuerySchedulerService;
import org.apache.hive.service.cli.CLIService;
@@ -26,7 +27,12 @@ import org.apache.hive.service.cli.CLIService;
/**
* The Class QuerySchedulerServiceImpl.
*/
-public class QuerySchedulerServiceImpl extends LensService implements QuerySchedulerService {
+public class QuerySchedulerServiceImpl extends BaseLensService implements QuerySchedulerService {
+
+ /**
+ * The constant name for scheduler service.
+ */
+ public static final String NAME = "scheduler";
/**
* Instantiates a new query scheduler service impl.
@@ -34,6 +40,16 @@ public class QuerySchedulerServiceImpl extends LensService implements QuerySched
* @param cliService the cli service
*/
public QuerySchedulerServiceImpl(CLIService cliService) {
- super("scheduler", cliService);
+ super(NAME, cliService);
+ }
+
+ /**
+ * @inheritDoc
+ */
+ @Override
+ public HealthStatus getHealthStatus() {
+ return this.getServiceState().equals(STATE.STARTED)
+ ? new HealthStatus(true, "Query scheduler service is healthy.")
+ : new HealthStatus(false, "Query scheduler service is down.");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index 3a8fc95..ab5be32 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -32,10 +32,11 @@ import javax.ws.rs.NotFoundException;
import javax.ws.rs.WebApplicationException;
import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.server.LensService;
+import org.apache.lens.server.BaseLensService;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.health.HealthStatus;
import org.apache.lens.server.api.session.SessionClosed;
import org.apache.lens.server.api.session.SessionExpired;
import org.apache.lens.server.api.session.SessionOpened;
@@ -66,7 +67,8 @@ import lombok.extern.slf4j.Slf4j;
* The Class HiveSessionService.
*/
@Slf4j
-public class HiveSessionService extends LensService implements SessionService {
+public class HiveSessionService extends BaseLensService implements SessionService {
+
/** The restorable sessions. */
private List<LensSessionImpl.LensSessionPersistInfo> restorableSessions;
@@ -102,7 +104,7 @@ public class HiveSessionService extends LensService implements SessionService {
public int addResourceToAllServices(LensSessionHandle sessionid, String type, String path) {
int numAdded = 0;
boolean error = false;
- for (LensService service : LensServices.get().getLensServices()) {
+ for (BaseLensService service : LensServices.get().getLensServices()) {
try {
service.addResource(sessionid, type, path);
numAdded++;
@@ -448,6 +450,16 @@ public class HiveSessionService extends LensService implements SessionService {
log.info("Session service pesristed " + SESSION_MAP.size() + " sessions");
}
+ /**
+ * @inheritDoc
+ */
+ @Override
+ public HealthStatus getHealthStatus() {
+ return this.getServiceState().equals(STATE.STARTED)
+ ? new HealthStatus(true, "Hive session service is healthy.")
+ : new HealthStatus(false, "Hive session service is down.");
+ }
+
/*
* (non-Javadoc)
*
@@ -484,7 +496,7 @@ public class HiveSessionService extends LensService implements SessionService {
private void closeInternal(LensSessionHandle sessionHandle) throws LensException {
super.closeSession(sessionHandle);
// Inform query service
- LensService svc = LensServices.get().getService(QueryExecutionServiceImpl.NAME);
+ BaseLensService svc = LensServices.get().getService(QueryExecutionServiceImpl.NAME);
if (svc instanceof QueryExecutionServiceImpl) {
((QueryExecutionServiceImpl) svc).closeDriverSessions(sessionHandle);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
index 0635e95..63d7b9c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/SessionResource.java
@@ -30,7 +30,7 @@ import org.apache.lens.api.APIResult.Status;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.StringList;
-import org.apache.lens.server.LensService;
+import org.apache.lens.server.BaseLensService;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.session.SessionService;
@@ -201,7 +201,7 @@ public class SessionResource {
int numDeleted = 0;
for(String matchedPath : scannedPaths) {
- for (LensService service : LensServices.get().getLensServices()) {
+ for (BaseLensService service : LensServices.get().getLensServices()) {
try {
if (matchedPath.startsWith("file:") && !matchedPath.startsWith("file://")) {
matchedPath = "file://" + matchedPath.substring("file:".length());
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index 697e867..b5235ad 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -121,6 +121,8 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
log.info("Server restart test");
QueryExecutionServiceImpl queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
+ Assert.assertTrue(queryService.getHealthStatus().isHealthy());
+
LensSessionHandle lensSessionId = queryService.openSession("foo", "bar", new HashMap<String, String>());
// Create data file
createRestartTestDataFile();
@@ -143,6 +145,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
try {
queryService.pauseQuerySubmitter();
log.info("Stopped query submitter");
+ Assert.assertFalse(queryService.getHealthStatus().isHealthy());
} catch (Exception exc) {
log.error("Could not kill query submitter", exc);
}
@@ -173,6 +176,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
restartLensServer();
log.info("Restarted lens server!");
queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
+ Assert.assertTrue(queryService.getHealthStatus().isHealthy());
// All queries should complete after server restart
for (QueryHandle handle : launchedQueries) {
@@ -213,6 +217,8 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
@Test
public void testHiveServerRestart() throws Exception {
QueryExecutionServiceImpl queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
+ Assert.assertTrue(queryService.getHealthStatus().isHealthy());
+
LensSessionHandle lensSessionId = queryService.openSession("foo", "bar", new HashMap<String, String>());
// set params
@@ -223,6 +229,8 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
// Add a resource to check if its added after server restart.
HiveSessionService sessionService = (HiveSessionService) LensServices.get().getService(SessionService.NAME);
+ Assert.assertTrue(sessionService.getHealthStatus().isHealthy());
+
sessionService.addResource(lensSessionId, "FILE", dataFile.toURI().toString());
queryService.getSession(lensSessionId).addResource("FILE", dataFile.toURI().toString());
log.info("@@ Added resource {}", dataFile.toURI());
@@ -379,6 +387,8 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
verifyParamOnRestart(restartTestSession);
HiveSessionService sessionService = LensServices.get().getService("session");
+ Assert.assertTrue(sessionService.getHealthStatus().isHealthy());
+
LensSessionImpl session = sessionService.getSession(restartTestSession);
Assert.assertEquals(session.getLensSessionPersistInfo().getResources().size(), 1);
LensSessionImpl.ResourceEntry resourceEntry = session.getLensSessionPersistInfo().getResources().get(0);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/lens-server/src/test/java/org/apache/lens/server/healthcheck/TestHealthChecks.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/healthcheck/TestHealthChecks.java b/lens-server/src/test/java/org/apache/lens/server/healthcheck/TestHealthChecks.java
new file mode 100644
index 0000000..3573a1e
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/healthcheck/TestHealthChecks.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.healthcheck;
+
+import static org.testng.Assert.*;
+
+import javax.ws.rs.core.Application;
+
+import org.apache.lens.server.EventServiceImpl;
+import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.LensServices;
+import org.apache.lens.server.api.LensService;
+import org.apache.lens.server.api.health.HealthStatus;
+import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.metastore.CubeMetastoreServiceImpl;
+import org.apache.lens.server.metastore.MetastoreApp;
+import org.apache.lens.server.metrics.MetricsServiceImpl;
+import org.apache.lens.server.quota.QuotaServiceImpl;
+import org.apache.lens.server.scheduler.QuerySchedulerServiceImpl;
+import org.apache.lens.server.session.HiveSessionService;
+
+import org.apache.log4j.BasicConfigurator;
+
+import org.glassfish.jersey.client.ClientConfig;
+import org.glassfish.jersey.media.multipart.MultiPartFeature;
+
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import com.codahale.metrics.health.HealthCheck;
+
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@Test(groups = "unit-test")
+public class TestHealthChecks extends LensJerseyTest {
+ @BeforeTest
+ public void setUp() throws Exception {
+ super.setUp();
+ BasicConfigurator.configure();
+ }
+
+ @AfterTest
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ @Override
+ protected Application configure() {
+ return new MetastoreApp();
+ }
+
+ @Override
+ protected void configureClient(ClientConfig config) {
+ config.register(MultiPartFeature.class);
+ }
+
+ @Test
+ public void testCubeMetastoreServiceHealth() throws Exception {
+ checkHealth(CubeMetastoreServiceImpl.NAME);
+ }
+
+ @Test
+ public void testEventServiceHealth() throws Exception {
+ checkHealth(EventServiceImpl.NAME);
+ }
+
+ @Test
+ public void testHiveSessionServiceHealth() throws Exception {
+ checkHealth(HiveSessionService.NAME);
+ }
+
+ @Test
+ public void testMetricsServiceHealth() throws Exception {
+ checkHealth(MetricsServiceImpl.NAME);
+ }
+
+ @Test
+ public void testQueryExecutionServiceHealth() throws Exception {
+ checkHealth(QueryExecutionService.NAME);
+ }
+
+ @Test
+ public void testQuerySchedulerServiceHealth() throws Exception {
+ checkHealth(QuerySchedulerServiceImpl.NAME);
+ }
+
+ @Test
+ public void testQuotaServiceHealth() throws Exception {
+ checkHealth(QuotaServiceImpl.NAME);
+ }
+
+ /**
+ * Utility method to check health for provided service name.
+ *
+ * @param serviceName
+ * @throws Exception
+ */
+ private void checkHealth(String serviceName) throws Exception {
+ /** Test via LensServiceHealthCheck **/
+ new LensServiceHealthCheck(serviceName).check().equals(HealthCheck.Result.healthy());
+
+ /** Also check directly via service **/
+ HealthStatus status;
+ LensService service = LensServices.get().getService(serviceName);
+ status = service.getHealthStatus();
+ assertTrue(status.isHealthy(), serviceName + " Service should had been healthy.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/651a9b6a/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index 63f2b3f..51257e0 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -237,9 +237,9 @@ User CLI Commands
*--+--+
|fact timelines [--fact_name] \<fact_name\> [--storage_name \<storage_name\>] [--update_period \<update_period\>] [--time_dimension \<time_dimension\>]|get timelines for fact. Can optionally specify storage, update period and time dimension to filter by. Instead of time dimension, partition column can be directly passed as <<<time_dimension>>>|
*--+--+
-|fact update partitions [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [--path] \<partition-list-spec-path\>|update multiple partition to fact <<<fact_name>>>'s storage <<<storage_name>>>, reading partition list spec from <<<partition-list-spec-path>>>|
+|fact update partitions [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [--path] \<partition-list-spec-path\>|update multiple partition of fact <<<fact_name>>>'s storage <<<storage_name>>>, reading partition list spec from <<<partition-list-spec-path>>> The partitions have to exist to be eligible for updation.|
*--+--+
-|fact update single-partition [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [--path] \<partition-spec-path\>|update single partition to fact <<<fact_name>>>'s storage <<<storage_name>>>, reading spec from <<<partition-spec-path>>>|
+|fact update single-partition [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [--path] \<partition-spec-path\>|update single partition to fact <<<fact_name>>>'s storage <<<storage_name>>>, reading spec from <<<partition-spec-path>>> The partition has to exist to be eligible for updation.|
*--+--+
|show facts [[--cube_name] \<cube_name\>]|display list of fact tables in current database. If optional <<<cube_name>>> is supplied, only facts belonging to cube <<<cube_name>>> will be displayed|
*--+--+
@@ -278,9 +278,9 @@ User CLI Commands
*--+--+
|dimtable list storages [--dimtable_name] \<dimtable_name\>|display list of storage associated to dimtable <<<dimtable_name>>>|
*--+--+
-|dimtable update partitions [--dimtable_name] \<dimtable_name\> [--storage_name] \<storage_name\> [--path] \<partition-list-spec-path\>|update multiple partition to dimtable <<<dimtable_name>>>'s storage <<<storage_name>>>, reading partition list spec from <<<partition-list-spec-path>>>|
+|dimtable update partitions [--dimtable_name] \<dimtable_name\> [--storage_name] \<storage_name\> [--path] \<partition-list-spec-path\>|update multiple partition to dimtable <<<dimtable_name>>>'s storage <<<storage_name>>>, reading partition list spec from <<<partition-list-spec-path>>> The partitions have to exist to be eligible for updation.|
*--+--+
-|dimtable update single-partition [--dimtable_name] \<dimtable_name\> [--storage_name] \<storage_name\> [--path] \<partition-spec-path\>|update single partition to dimtable <<<dimtable_name>>>'s storage <<<storage_name>>>, reading spec from <<<partition-spec-path>>>|
+|dimtable update single-partition [--dimtable_name] \<dimtable_name\> [--storage_name] \<storage_name\> [--path] \<partition-spec-path\>|update single partition to dimtable <<<dimtable_name>>>'s storage <<<storage_name>>>, reading spec from <<<partition-spec-path>>> The partition has to exist to be eligible for updation.|
*--+--+
|drop dimtable [--dimtable_name] \<dimtable_name\> [--cascade \<cascade\>]|drop dimtable <<<dimtable_name>>>. If <<<cascade>>> is true, all the storage tables associated with the dimtable <<<dimtable_name>>> are also dropped. By default <<<cascade>>> is false|
*--+--+
[18/50] [abbrv] incubator-lens git commit: LENS-669 : Add queryId in
LogSegregationContext
Posted by sh...@apache.org.
LENS-669 : Add queryId in LogSegregationContext
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/dfeea698
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/dfeea698
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/dfeea698
Branch: refs/heads/LENS-581
Commit: dfeea69890cf75b1713d645117a4e1ec01761f72
Parents: 46dc5ad
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Mon Jul 20 15:42:06 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Jul 20 15:42:06 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 2 +-
.../server/api/query/AbstractQueryContext.java | 4 +-
.../server/model/LogSegregationContext.java | 27 ++++++++-
.../MappedDiagnosticLogSegregationContext.java | 23 +++++++-
.../lens/server/api/query/MockQueryContext.java | 5 ++
...stMappedDiagnosticLogSegregationContext.java | 60 ++++++++++++++++++++
.../server/LensRequestContextInitFilter.java | 2 +-
.../java/org/apache/lens/server/LensServer.java | 2 +-
.../org/apache/lens/server/LensServices.java | 2 +-
.../lens/server/query/QueryEndNotifier.java | 2 +-
.../server/query/QueryExecutionServiceImpl.java | 11 ++--
.../lens/server/query/QueryServiceResource.java | 2 +-
.../lens/server/query/ResultFormatter.java | 2 +-
.../store/log/StatisticsLogFileScannerTask.java | 2 +-
14 files changed, 125 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index a09cd1b..9445436 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -294,7 +294,7 @@ public class JDBCDriver implements LensDriver {
@Override
public QueryResult call() {
- logSegregationContext.set(this.queryContext.getQueryHandleString());
+ logSegregationContext.setLogSegragationAndQueryId(this.queryContext.getQueryHandleString());
Statement stmt = null;
Connection conn = null;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 1334d9a..71a37f0 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -455,9 +455,7 @@ public abstract class AbstractQueryContext implements Serializable {
* Get handle of the query for logging purposes
* @return
*/
- public String getLogHandle() {
- return this.getUserQuery();
- }
+ public abstract String getLogHandle();
/**
* Returns database set while launching query
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
index 8691a05..f76e292 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/model/LogSegregationContext.java
@@ -25,11 +25,34 @@ public interface LogSegregationContext {
*
* @param id the id to be added to every log line of current thread
*/
- void set(final String id);
+ void setLogSegregationId(final String id);
/**
*
* @return the id being used by the current thread for log segregation
*/
- String get();
+ String getLogSegragationId();
+
+ /**
+ * Sets query id to be used by current thread for log segregation for identifying current query.
+ * The same id is set as log segregation in every log line as well.
+ *
+ * @param id the query id
+ */
+ void setLogSegragationAndQueryId(final String id);
+
+ /**
+ * Sets query id to be used by current thread for log segregation for identifying current query.
+ *
+ * @param id the query id
+ */
+ void setQueryId(final String id);
+
+ /**
+ * Get current query id in log segregation
+ *
+ * @return the query id being used by the current thread for log segregation
+ */
+ String getQueryId();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
index 989cb9d..fd5f012 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/model/MappedDiagnosticLogSegregationContext.java
@@ -22,15 +22,32 @@ import org.slf4j.MDC;
public class MappedDiagnosticLogSegregationContext implements LogSegregationContext {
- private static final String LOG_SEGREGATION_ID = "logSegregationId";
+ public static final String LOG_SEGREGATION_ID = "logSegregationId";
+ public static final String QUERY_LOG_ID = "queryLogId";
@Override
- public void set(String id) {
+ public void setLogSegregationId(String id) {
MDC.put(LOG_SEGREGATION_ID, id);
}
@Override
- public String get() {
+ public String getLogSegragationId() {
return MDC.get(LOG_SEGREGATION_ID);
}
+
+ @Override
+ public void setLogSegragationAndQueryId(String id) {
+ setLogSegregationId(id);
+ setQueryId(id);
+ }
+
+ @Override
+ public String getQueryId() {
+ return MDC.get(QUERY_LOG_ID);
+ }
+
+ @Override
+ public void setQueryId(String id) {
+ MDC.put(QUERY_LOG_ID, id);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
index 5a1ab66..3d38eab 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/MockQueryContext.java
@@ -33,4 +33,9 @@ public class MockQueryContext extends AbstractQueryContext {
final Configuration conf, final Collection<LensDriver> drivers) {
super(query, "testuser", qconf, conf, drivers, false);
}
+
+ @Override
+ public String getLogHandle() {
+ return super.getUserQuery();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java b/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java
new file mode 100644
index 0000000..227fa15
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/model/TestMappedDiagnosticLogSegregationContext.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.model;
+
+import static org.testng.Assert.*;
+
+import org.slf4j.MDC;
+import org.testng.annotations.Test;
+
+/**
+ * Tests for MDC log segregation
+ */
+public class TestMappedDiagnosticLogSegregationContext {
+
+ @Test
+ public void testIds() {
+ MappedDiagnosticLogSegregationContext testLogCtx = new MappedDiagnosticLogSegregationContext();
+ String logId = "logid";
+ testLogCtx.setLogSegregationId(logId);
+ assertEquals(testLogCtx.getLogSegragationId(), logId);
+ assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), logId);
+
+ String queryId = "queryId";
+ testLogCtx.setLogSegragationAndQueryId(queryId);
+ assertEquals(testLogCtx.getQueryId(), queryId);
+ assertEquals(testLogCtx.getLogSegragationId(), queryId);
+ assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), queryId);
+ assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.QUERY_LOG_ID), queryId);
+
+ String logId2 = "logid2";
+ testLogCtx.setLogSegregationId(logId2);
+ assertEquals(testLogCtx.getLogSegragationId(), logId2);
+ assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), logId2);
+ assertEquals(testLogCtx.getQueryId(), queryId);
+ assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.QUERY_LOG_ID), queryId);
+
+ String queryId2 = "queryId2";
+ testLogCtx.setQueryId(queryId2);
+ assertEquals(testLogCtx.getQueryId(), queryId2);
+ assertEquals(testLogCtx.getLogSegragationId(), logId2);
+ assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.LOG_SEGREGATION_ID), logId2);
+ assertEquals(MDC.get(MappedDiagnosticLogSegregationContext.QUERY_LOG_ID), queryId2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
index c9658fc..d2f2f91 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensRequestContextInitFilter.java
@@ -49,7 +49,7 @@ public class LensRequestContextInitFilter implements ContainerRequestFilter {
String uniqueRequesId = UUID.randomUUID().toString();
/* Add request id for appearing in every log line */
- new MappedDiagnosticLogSegregationContext().set(uniqueRequesId);
+ new MappedDiagnosticLogSegregationContext().setLogSegregationId(uniqueRequesId);
/* Add request id to headers */
requestContext.getHeaders().add(REQUEST_ID, uniqueRequesId);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/LensServer.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServer.java b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
index d3dfad4..d4c89c0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServer.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServer.java
@@ -185,7 +185,7 @@ public class LensServer {
public static void main(String[] args) throws Exception {
final String runId = UUID.randomUUID().toString();
- new MappedDiagnosticLogSegregationContext().set(runId);
+ new MappedDiagnosticLogSegregationContext().setLogSegregationId(runId);
printStartupMessage();
try {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/LensServices.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServices.java b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
index 5fe3935..fe58e96 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServices.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServices.java
@@ -280,7 +280,7 @@ public class LensServices extends CompositeService implements ServiceProvider {
public void run() {
try {
final String runId = UUID.randomUUID().toString();
- logSegregationContext.set(runId);
+ logSegregationContext.setLogSegregationId(runId);
persistLensServiceState();
LOG.info("SnapShot of Lens Services created");
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
index df36a21..757205a 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
@@ -117,7 +117,7 @@ public class QueryEndNotifier extends AsyncEventListener<QueryEnded> {
+ ". No email generated");
return;
}
- this.logSegregationContext.set(queryContext.getQueryHandleString());
+ this.logSegregationContext.setLogSegragationAndQueryId(queryContext.getQueryHandleString());
boolean whetherMailNotify = Boolean.parseBoolean(queryContext.getConf().get(LensConfConstants.QUERY_MAIL_NOTIFY,
LensConfConstants.WHETHER_MAIL_NOTIFY_DEFAULT));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 2d7a92a..47d27d0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -503,7 +503,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
QueryContext ctx = queuedQueries.take();
/* Setting log segregation id */
- logSegregationContext.set(ctx.getQueryHandleString());
+ logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString());
synchronized (ctx) {
if (ctx.getStatus().getStatus().equals(QUEUED)) {
@@ -587,7 +587,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
return;
}
- logSegregationContext.set(ctx.getQueryHandleString());
+ logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString());
log.info("Polling status for " + ctx.getQueryHandle());
try {
// session is not required to update status of the query
@@ -808,7 +808,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
FinishedQuery finished = null;
try {
finished = finishedQueries.take();
- logSegregationContext.set(finished.getQueryHandleString());
+ logSegregationContext.setLogSegragationAndQueryId(finished.getQueryHandleString());
} catch (InterruptedException e) {
log.info("QueryPurger has been interrupted, exiting");
return;
@@ -883,7 +883,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
while (!stopped && !prepareQueryPurger.isInterrupted()) {
try {
PreparedQueryContext prepared = preparedQueryQueue.take();
- logSegregationContext.set(prepared.getQueryHandleString());
+ logSegregationContext.setLogSegragationAndQueryId(prepared.getQueryHandleString());
destroyPreparedQuery(prepared);
log.info("Purged prepared query: " + prepared.getPrepareHandle());
} catch (LensException e) {
@@ -1220,7 +1220,8 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
@Override
public void run() {
try {
- logSegregationContext.set(ctx.getLogHandle());
+ // With following set - explain estimate calls are setting queryLogId as requestid in logSegregationContext
+ logSegregationContext.setLogSegragationAndQueryId(ctx.getLogHandle());
acquire(ctx.getLensSessionIdentifier());
MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true,
REWRITE_GAUGE);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
index 19ed94a..fce9cf5 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryServiceResource.java
@@ -210,7 +210,7 @@ public class QueryServiceResource {
@FormDataParam("conf") LensConf conf, @DefaultValue("30000") @FormDataParam("timeoutmillis") Long timeoutmillis,
@DefaultValue("") @FormDataParam("queryName") String queryName) throws LensException {
- final String requestId = this.logSegregationContext.get();
+ final String requestId = this.logSegregationContext.getLogSegragationId();
try {
validateSessionId(sessionid);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index c526c02..877e296 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -79,7 +79,7 @@ public class ResultFormatter extends AsyncEventListener<QueryExecuted> {
private void formatOutput(QueryExecuted event) {
QueryHandle queryHandle = event.getQueryHandle();
QueryContext ctx = queryService.getQueryContext(queryHandle);
- this.logSegregationContext.set(ctx.getQueryHandleString());
+ this.logSegregationContext.setLogSegragationAndQueryId(ctx.getQueryHandleString());
try {
if (!ctx.isPersistent()) {
LOG.info("No result formatting required for query " + queryHandle);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/dfeea698/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
index 7e9c74d..305c031 100644
--- a/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
+++ b/lens-server/src/main/java/org/apache/lens/server/stats/store/log/StatisticsLogFileScannerTask.java
@@ -76,7 +76,7 @@ public class StatisticsLogFileScannerTask extends TimerTask {
try {
final String runId = UUID.randomUUID().toString();
- this.logSegregationContext.set(runId);
+ this.logSegregationContext.setLogSegregationId(runId);
for (Map.Entry<String, String> entry : scanSet.entrySet()) {
File f = new File(entry.getValue()).getAbsoluteFile();
[21/50] [abbrv] incubator-lens git commit: LENS-652 : Add Update
partition api and cli command
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
index c5e1c14..646de8c 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java
@@ -327,7 +327,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
if (dimTable.getStorages() != null && !dimTable.getStorages().isEmpty()) {
for (String storageName : dimTable.getStorages()) {
XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
- msClient.getHiveTable(MetastoreUtil.getDimStorageTableName(dimTblName, storageName)));
+ msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName)));
tblElement.setStorageName(storageName);
UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName);
if (p != null) {
@@ -491,7 +491,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
for (String storageName : cft.getStorages()) {
Set<UpdatePeriod> updatePeriods = cft.getUpdatePeriods().get(storageName);
XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
- msClient.getHiveTable(MetastoreUtil.getFactStorageTableName(fact, storageName)));
+ msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName)));
tblElement.setStorageName(storageName);
for (UpdatePeriod p : updatePeriods) {
tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
@@ -605,7 +605,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
CubeFactTable factTable = msClient.getFactTable(fact);
Set<UpdatePeriod> updatePeriods = factTable.getUpdatePeriods().get(storageName);
XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
- msClient.getHiveTable(MetastoreUtil.getFactStorageTableName(fact, storageName)));
+ msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName)));
tblElement.setStorageName(storageName);
for (UpdatePeriod p : updatePeriods) {
tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
@@ -625,7 +625,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
CubeMetastoreClient msClient = getClient(sessionid);
CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName);
XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
- msClient.getHiveTable(MetastoreUtil.getDimStorageTableName(dimTblName, storageName)));
+ msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName)));
tblElement.setStorageName(storageName);
UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName);
if (p != null) {
@@ -696,17 +696,20 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
checkFactStorage(sessionid, fact, storageName);
- String storageTableName = MetastoreUtil.getFactStorageTableName(fact, storageName);
- List<Partition> parts = getClient(sessionid).getPartitionsByFilter(storageTableName, filter);
+ CubeMetastoreClient client = getClient(sessionid);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(fact,
+ storageName);
+ List<Partition> parts = client.getPartitionsByFilter(storageTableName, filter);
+ List<String> timePartCols = client.getTimePartColNamesOfTable(storageTableName);
if (parts != null) {
List<XPartition> result = new ArrayList<XPartition>(parts.size());
for (Partition p : parts) {
- XPartition xp = JAXBUtils.xpartitionFromPartition(p);
+ XPartition xp = JAXBUtils.xpartitionFromPartition(p, timePartCols);
result.add(xp);
}
return result;
} else {
- return new ArrayList<XPartition>();
+ return new ArrayList<>();
}
} catch (HiveException exc) {
throw new LensException(exc);
@@ -767,13 +770,15 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
try {
acquire(sessionid);
checkDimensionStorage(sessionid, dimension, storageName);
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimension,
+ CubeMetastoreClient client = getClient(sessionid);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimension,
storageName);
- List<Partition> partitions = getClient(sessionid).getPartitionsByFilter(storageTableName, filter);
+ List<Partition> partitions = client.getPartitionsByFilter(storageTableName, filter);
+ List<String> timePartCols = client.getTimePartColNamesOfTable(storageTableName);
if (partitions != null) {
List<XPartition> result = new ArrayList<XPartition>(partitions.size());
for (Partition p : partitions) {
- XPartition xp = JAXBUtils.xpartitionFromPartition(p);
+ XPartition xp = JAXBUtils.xpartitionFromPartition(p, timePartCols);
result.add(xp);
}
return result;
@@ -805,6 +810,46 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
}
@Override
+ public void updatePartition(LensSessionHandle sessionid, String tblName, String storageName,
+ XPartition xPartition) throws LensException {
+ try {
+ acquire(sessionid);
+ CubeMetastoreClient client = getClient(sessionid);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(tblName, storageName);
+ Partition existingPartition = client.getPartitionByFilter(storageTableName,
+ StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition)));
+ JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition);
+ client.updatePartition(tblName, storageName, existingPartition);
+ } catch (HiveException | ClassNotFoundException |InvalidOperationException | UnsupportedOperationException exc) {
+ throw new LensException(exc);
+ } finally {
+ release(sessionid);
+ }
+ }
+
+ @Override
+ public void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName,
+ XPartitionList xPartitions) throws LensException {
+ try {
+ acquire(sessionid);
+ CubeMetastoreClient client = getClient(sessionid);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(tblName, storageName);
+ List<Partition> partitionsToUpdate = new ArrayList<>(xPartitions.getPartition().size());
+ for (XPartition xPartition : xPartitions.getPartition()) {
+ Partition existingPartition = client.getPartitionByFilter(storageTableName,
+ StorageConstants.getPartFilter(JAXBUtils.getFullPartSpecAsMap(xPartition)));
+ JAXBUtils.updatePartitionFromXPartition(existingPartition, xPartition);
+ partitionsToUpdate.add(existingPartition);
+ }
+ client.updatePartitions(tblName, storageName, partitionsToUpdate);
+ } catch (HiveException | ClassNotFoundException | InvalidOperationException exc) {
+ throw new LensException(exc);
+ } finally {
+ release(sessionid);
+ }
+ }
+
+ @Override
public void addPartitionsToDimStorage(LensSessionHandle sessionid,
String dimTblName, String storageName, XPartitionList partitions) throws LensException {
try {
@@ -1411,7 +1456,7 @@ public class CubeMetastoreServiceImpl extends LensService implements CubeMetasto
throw new BadRequestException("Can't get join chains. '"
+ tableName + "' is neither a cube nor a dimension");
}
- XJoinChains xJoinChains= new XJoinChains();
+ XJoinChains xJoinChains = new XJoinChains();
List<XJoinChain> joinChains = xJoinChains.getJoinChain();
if (chains != null) {
for (JoinChain chain : chains) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
index 811fd32..2d0eba2 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/JAXBUtils.java
@@ -33,14 +33,16 @@ import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ParseException;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.mapred.InputFormat;
import com.google.common.base.Optional;
-
+import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
/**
@@ -783,9 +785,8 @@ public final class JAXBUtils {
return nonTimePartSpec;
}
- public static XPartition xpartitionFromPartition(Partition p) throws HiveException {
+ public static XPartition xpartitionFromPartition(Partition p, List<String> timePartCols) throws HiveException {
XPartition xp = new XPartition();
- xp.setFullPartitionSpec(new XPartSpec());
xp.setPartitionParameters(new XProperties());
xp.setSerdeParameters(new XProperties());
xp.setName(p.getCompleteName());
@@ -795,18 +796,58 @@ public final class JAXBUtils {
xp.getPartitionParameters().getProperty().addAll(xPropertiesFromMap(p.getParameters()));
String upParam = p.getParameters().get(MetastoreConstants.PARTITION_UPDATE_PERIOD);
xp.setUpdatePeriod(XUpdatePeriod.valueOf(upParam));
- for (Map.Entry<String, String> entry : p.getSpec().entrySet()) {
+ LinkedHashMap<String, String> partSpec = p.getSpec();
+ xp.setFullPartitionSpec(new XPartSpec());
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
XPartSpecElement e = new XPartSpecElement();
e.setKey(entry.getKey());
e.setValue(entry.getValue());
xp.getFullPartitionSpec().getPartSpecElement().add(e);
}
+ try {
+ xp.setTimePartitionSpec(new XTimePartSpec());
+ xp.setNonTimePartitionSpec(new XPartSpec());
+ for (Map.Entry<String, String> entry : partSpec.entrySet()) {
+ if (timePartCols.contains(entry.getKey())) {
+ XTimePartSpecElement timePartSpecElement = new XTimePartSpecElement();
+ timePartSpecElement.setKey(entry.getKey());
+ timePartSpecElement
+ .setValue(getXMLGregorianCalendar(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format().parse(
+ entry.getValue())));
+ xp.getTimePartitionSpec().getPartSpecElement().add(timePartSpecElement);
+ } else {
+ XPartSpecElement partSpecElement = new XPartSpecElement();
+ partSpecElement.setKey(entry.getKey());
+ partSpecElement.setValue(entry.getValue());
+ xp.getNonTimePartitionSpec().getPartSpecElement().add(partSpecElement);
+ }
+ }
+ } catch (java.text.ParseException exc) {
+ log.debug("can't form time part spec from " + partSpec, exc);
+ xp.setTimePartitionSpec(null);
+ xp.setNonTimePartitionSpec(null);
+ }
xp.setSerdeClassname(p.getTPartition().getSd().getSerdeInfo().getSerializationLib());
xp.getSerdeParameters().getProperty().addAll(xPropertiesFromMap(
p.getTPartition().getSd().getSerdeInfo().getParameters()));
return xp;
}
+ public static void updatePartitionFromXPartition(Partition partition, XPartition xp)
+ throws ClassNotFoundException, HiveException {
+ partition.getParameters().putAll(mapFromXProperties(xp.getPartitionParameters()));
+ partition.getTPartition().getSd().getSerdeInfo().setParameters(mapFromXProperties(xp.getSerdeParameters()));
+ partition.setLocation(xp.getLocation());
+ if (xp.getInputFormat() != null) {
+ partition.setInputFormatClass((Class<? extends InputFormat>) Class.forName(xp.getInputFormat()));
+ }
+ if (xp.getOutputFormat() != null) {
+ partition.setOutputFormatClass((Class<? extends HiveOutputFormat>) Class.forName(xp.getOutputFormat()));
+ }
+ partition.getParameters().put(MetastoreConstants.PARTITION_UPDATE_PERIOD, xp.getUpdatePeriod().name());
+ partition.getTPartition().getSd().getSerdeInfo().setSerializationLib(xp.getSerdeClassname());
+ }
+
public static StoragePartitionDesc storagePartSpecFromXPartition(
XPartition xpart) {
StoragePartitionDesc partDesc = new StoragePartitionDesc(
@@ -893,4 +934,20 @@ public final class JAXBUtils {
xtable.setTableType(table.getTableType().name());
return xtable;
}
+
+ public static Map<String, String> getFullPartSpecAsMap(XPartition partition) {
+ Map<String, String> spec = Maps.newHashMap();
+ if (partition.getTimePartitionSpec() != null) {
+ for (XTimePartSpecElement timePartSpecElement : partition.getTimePartitionSpec().getPartSpecElement()) {
+ spec.put(timePartSpecElement.getKey(), UpdatePeriod.valueOf(partition.getUpdatePeriod().name()).format()
+ .format(getDateFromXML(timePartSpecElement.getValue())));
+ }
+ }
+ if (partition.getNonTimePartitionSpec() != null) {
+ for (XPartSpecElement partSpecElement : partition.getNonTimePartitionSpec().getPartSpecElement()) {
+ spec.put(partSpecElement.getKey(), partSpecElement.getValue());
+ }
+ }
+ return spec;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
index bfe4813..cb9516b 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java
@@ -363,7 +363,7 @@ public class MetastoreResource {
* @param cubeName The cube name
* @param cube The {@link XCube} representation of the updated cube definition
* @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with
- * state {@link Status#FAILED}, if udpate has failed
+ * state {@link Status#FAILED}, if update has failed
*/
@PUT
@Path("/cubes/{cubeName}")
@@ -870,7 +870,7 @@ public class MetastoreResource {
* @param factName name of the fact table
* @param fact The {@link XFactTable} representation of the updated fact table definition
* @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with
- * state {@link Status#FAILED}, if udpate has failed
+ * state {@link Status#FAILED}, if update has failed
*/
@PUT
@Path("/facts/{factName}")
@@ -1108,6 +1108,34 @@ public class MetastoreResource {
}
return SUCCESS;
}
+ /**
+ * updates an existing partition for a storage of fact
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param factName fact table name
+ * @param storage storage name
+ * @param partition {@link XPartition} representation of partition.
+ * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful.
+ * {@link APIResult} with state
+ * {@link Status#FAILED}, if update has failed
+ */
+ @PUT
+ @Path("/facts/{factName}/storages/{storage}/partition")
+ public APIResult updatePartitionOfFactStorage(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("factName") String factName,
+ @PathParam("storage") String storage,
+ XPartition partition) {
+ checkSessionId(sessionid);
+ checkNonNullArgs("Partition is null", partition);
+ try {
+ getSvc().updatePartition(sessionid, factName, storage, partition);
+ } catch (LensException exc) {
+ checkTableNotFound(exc, factName);
+ LOG.error("Error adding partition to storage of fact" + factName + ":" + storage, exc);
+ return new APIResult(Status.FAILED, exc.getMessage());
+ }
+ return SUCCESS;
+ }
/**
* Batch Add partitions for a storage of fact
@@ -1136,6 +1164,34 @@ public class MetastoreResource {
}
return SUCCESS;
}
+ /**
+ * Batch Update partitions for a storage of fact
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param factName fact table name
+ * @param storage storage name
+ * @param partitions {@link XPartitionList} representation of partitions
+ * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful.
+ * {@link APIResult} with state
+ * {@link Status#FAILED}, if update has failed
+ */
+ @PUT
+ @Path("/facts/{factName}/storages/{storage}/partitions")
+ public APIResult updatePartitionsOfFactStorage(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("factName") String factName,
+ @PathParam("storage") String storage,
+ XPartitionList partitions) {
+ checkSessionId(sessionid);
+ checkNonNullArgs("Partition List is null", partitions);
+ try {
+ getSvc().updatePartitions(sessionid, factName, storage, partitions);
+ } catch (LensException exc) {
+ checkTableNotFound(exc, factName);
+ LOG.error("Error adding partition to storage of fact" + factName + ":" + storage, exc);
+ return new APIResult(Status.FAILED, exc.getMessage());
+ }
+ return SUCCESS;
+ }
/**
* Drop the partitions in the storage of a fact table, specified by exact values
@@ -1206,7 +1262,7 @@ public class MetastoreResource {
* @param sessionid The sessionid in which user is working
* @param dimensionTable The {@link XDimensionTable} representation of the updated dim table definition
* @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful. {@link APIResult} with
- * state {@link Status#FAILED}, if udpate has failed
+ * state {@link Status#FAILED}, if update has failed
*/
@PUT
@Path("/dimtables/{dimTableName}")
@@ -1481,6 +1537,33 @@ public class MetastoreResource {
}
return SUCCESS;
}
+ /**
+ * Updates an existing partition for a storage of dimension
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param dimTableName dimension table name
+ * @param storage storage name
+ * @param partition {@link XPartition} representation of partition
+ * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful.
+ * {@link APIResult} with state
+ * {@link Status#FAILED}, if update has failed
+ */
+ @PUT
+ @Path("/dimtables/{dimTableName}/storages/{storage}/partition")
+ public APIResult updatePartitionOfDimStorage(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("dimTableName") String dimTableName,
+ @PathParam("storage") String storage,
+ XPartition partition) {
+ checkSessionId(sessionid);
+ checkNonNullArgs("Partition is null", partition);
+ try {
+ getSvc().updatePartition(sessionid, dimTableName, storage, partition);
+ } catch (LensException exc) {
+ LOG.error("Error adding partition to storage of dimension table " + dimTableName + ":" + storage, exc);
+ return new APIResult(Status.FAILED, exc.getMessage());
+ }
+ return SUCCESS;
+ }
/**
* Add new partitions for a storage of dimension
@@ -1508,6 +1591,33 @@ public class MetastoreResource {
}
return SUCCESS;
}
+ /**
+ * Add new partitions for a storage of dimension
+ *
+ * @param sessionid The sessionid in which user is working
+ * @param dimTableName dimension table name
+ * @param storage storage name
+ * @param partitions {@link XPartitionList} representation of list of partitions
+ * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if update was successful.
+ * {@link APIResult} with state
+ * {@link Status#FAILED}, if update has failed
+ */
+ @PUT
+ @Path("/dimtables/{dimTableName}/storages/{storage}/partitions")
+ public APIResult updatePartitionsOfDimStorage(@QueryParam("sessionid") LensSessionHandle sessionid,
+ @PathParam("dimTableName") String dimTableName,
+ @PathParam("storage") String storage,
+ XPartitionList partitions) {
+ checkSessionId(sessionid);
+ checkNonNullArgs("Partition list is null", partitions);
+ try {
+ getSvc().updatePartitions(sessionid, dimTableName, storage, partitions);
+ } catch (LensException exc) {
+ LOG.error("Error adding partition to storage of dimension table " + dimTableName + ":" + storage, exc);
+ return new APIResult(Status.FAILED, exc.getMessage());
+ }
+ return SUCCESS;
+ }
/**
* Get flattened list of columns reachable from a cube or a dimension
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
index dc1dc1d..4eb52b9 100644
--- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestMetastoreService.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.server.metastore;
+import static org.apache.lens.cube.metadata.UpdatePeriod.*;
+
import static org.testng.Assert.*;
import java.util.*;
@@ -27,19 +29,13 @@ import javax.ws.rs.NotFoundException;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Application;
-import javax.ws.rs.core.GenericType;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
+import javax.ws.rs.core.*;
import javax.xml.bind.JAXBElement;
import javax.xml.datatype.DatatypeFactory;
import javax.xml.datatype.XMLGregorianCalendar;
-import org.apache.lens.api.APIResult;
+import org.apache.lens.api.*;
import org.apache.lens.api.APIResult.Status;
-import org.apache.lens.api.DateTime;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.StringList;
import org.apache.lens.api.metastore.*;
import org.apache.lens.cube.metadata.*;
import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
@@ -56,10 +52,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.log4j.BasicConfigurator;
import org.glassfish.jersey.client.ClientConfig;
-import org.glassfish.jersey.media.multipart.FormDataBodyPart;
-import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
-import org.glassfish.jersey.media.multipart.FormDataMultiPart;
-import org.glassfish.jersey.media.multipart.MultiPartFeature;
+import org.glassfish.jersey.media.multipart.*;
import org.testng.Assert;
import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
@@ -652,7 +645,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(hcube.getDimAttributeByName("dim1").getDescription(), "first dimension");
assertEquals(hcube.getDimAttributeByName("dim1").getDisplayString(), "Dimension1");
assertEquals((((BaseDimAttribute) hcube.getDimAttributeByName("dim1")).getNumOfDistinctValues().get()),
- Long.valueOf(2000));
+ Long.valueOf(2000));
assertNotNull(hcube.getDimAttributeByName("testdim2col2"));
assertEquals(hcube.getDimAttributeByName("testdim2col2").getDisplayString(), "Chained Dimension");
@@ -663,12 +656,12 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getChainName(), "chain1");
assertEquals(((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2")).getRefColumn(), "col2");
assertEquals((((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2"))
- .getNumOfDistinctValues().get()), Long.valueOf(1000));
+ .getNumOfDistinctValues().get()), Long.valueOf(1000));
assertEquals((((ReferencedDimAtrribute) hcube.getDimAttributeByName("testdim2col2"))
- .getNumOfDistinctValues().get()), Long.valueOf(1000));
+ .getNumOfDistinctValues().get()), Long.valueOf(1000));
assertEquals(((BaseDimAttribute) hcube.getDimAttributeByName("dim2")).getNumOfDistinctValues().isPresent(),
- false);
+ false);
assertNotNull(hcube.getMeasureByName("msr1"));
assertEquals(hcube.getMeasureByName("msr1").getDescription(), "first measure");
@@ -1019,7 +1012,7 @@ public class TestMetastoreService extends LensJerseyTest {
XDimension dimension = createDimension("testdim");
APIResult result = target().path("metastore").path("dimensions")
.queryParam("sessionid", lensSessionId).request(
- mediaType).post(Entity.xml(cubeObjectFactory.createXDimension(dimension)), APIResult.class);
+ mediaType).post(Entity.xml(cubeObjectFactory.createXDimension(dimension)), APIResult.class);
assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
XDimensionTable dt = createDimTable("testdim", dimTableName);
dt.getStorageTables().getStorageTable().add(createStorageTblElement("test", dimTableName, "HOURLY"));
@@ -1343,7 +1336,7 @@ public class TestMetastoreService extends LensJerseyTest {
// add one more storage table
createStorage("testAlterDimStorage");
XStorageTableElement newStorage = createStorageTblElement("testAlterDimStorage", dt3.getTableName(),
- (String[])null);
+ (String[]) null);
newStorage.getTableDesc().setFieldDelimiter(":");
dt3.getStorageTables().getStorageTable().add(newStorage);
// Update the table
@@ -1446,8 +1439,8 @@ public class TestMetastoreService extends LensJerseyTest {
CubeDimensionTable cdim = JAXBUtils.cubeDimTableFromDimTable(dimTable);
assertTrue(cdim.getStorages().contains("test"));
assertTrue(cdim.getStorages().contains("test2"));
- assertEquals(cdim.getSnapshotDumpPeriods().get("test2"), UpdatePeriod.DAILY);
- assertEquals(cdim.getSnapshotDumpPeriods().get("test"), UpdatePeriod.HOURLY);
+ assertEquals(cdim.getSnapshotDumpPeriods().get("test2"), DAILY);
+ assertEquals(cdim.getSnapshotDumpPeriods().get("test"), HOURLY);
result = target().path("metastore/dimtables/").path(table).path("storages").path("test")
.queryParam("sessionid", lensSessionId).request(mediaType)
@@ -1470,7 +1463,7 @@ public class TestMetastoreService extends LensJerseyTest {
cdim = JAXBUtils.cubeDimTableFromDimTable(dimTable);
assertFalse(cdim.getStorages().contains("test"));
assertTrue(cdim.getStorages().contains("test2"));
- assertEquals(cdim.getSnapshotDumpPeriods().get("test2"), UpdatePeriod.DAILY);
+ assertEquals(cdim.getSnapshotDumpPeriods().get("test2"), DAILY);
// add another storage without dump period
sTbl = createStorageTblElement("test3", table, (String[]) null);
@@ -1625,8 +1618,8 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(cf.getProperties().get("foo"), "bar");
assertTrue(cf.getStorages().contains("S1"));
assertTrue(cf.getStorages().contains("S2"));
- assertTrue(cf.getUpdatePeriods().get("S1").contains(UpdatePeriod.HOURLY));
- assertTrue(cf.getUpdatePeriods().get("S2").contains(UpdatePeriod.DAILY));
+ assertTrue(cf.getUpdatePeriods().get("S1").contains(HOURLY));
+ assertTrue(cf.getUpdatePeriods().get("S2").contains(DAILY));
// Finally, drop the fact table
result = target().path("metastore").path("facts").path(table)
@@ -1714,8 +1707,8 @@ public class TestMetastoreService extends LensJerseyTest {
CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact);
assertEquals(ucf.weight(), 20.0);
- assertTrue(ucf.getUpdatePeriods().get("S2").contains(UpdatePeriod.MONTHLY));
- assertTrue(ucf.getUpdatePeriods().get("S3").contains(UpdatePeriod.DAILY));
+ assertTrue(ucf.getUpdatePeriods().get("S2").contains(MONTHLY));
+ assertTrue(ucf.getUpdatePeriods().get("S3").contains(DAILY));
boolean foundC2 = false;
for (FieldSchema fs : cf.getColumns()) {
@@ -1820,9 +1813,9 @@ public class TestMetastoreService extends LensJerseyTest {
XFactTable gotFact = gotFactElement.getValue();
CubeFactTable ucf = JAXBUtils.cubeFactFromFactTable(gotFact);
- assertTrue(ucf.getUpdatePeriods().get("S3").contains(UpdatePeriod.MONTHLY));
- assertTrue(ucf.getUpdatePeriods().get("S3").contains(UpdatePeriod.DAILY));
- assertTrue(ucf.getUpdatePeriods().get("S3").contains(UpdatePeriod.HOURLY));
+ assertTrue(ucf.getUpdatePeriods().get("S3").contains(MONTHLY));
+ assertTrue(ucf.getUpdatePeriods().get("S3").contains(DAILY));
+ assertTrue(ucf.getUpdatePeriods().get("S3").contains(HOURLY));
// Drop new storage
result = target().path("metastore/facts").path(table).path("storages").path("S3")
@@ -1849,7 +1842,7 @@ public class TestMetastoreService extends LensJerseyTest {
XTimePartSpecElement timePart = cubeObjectFactory.createXTimePartSpecElement();
timePart.setKey(timeDimension);
- timePart.setValue(JAXBUtils.getXMLGregorianCalendar(partDate));
+ timePart.setValue(JAXBUtils.getXMLGregorianCalendar(HOURLY.truncate(partDate)));
return createPartition(cubeTableName, Arrays.asList(timePart));
}
@@ -2008,6 +2001,12 @@ public class TestMetastoreService extends LensJerseyTest {
.post(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
assertEquals(partAddResult.getStatus(), Status.SUCCEEDED);
+ xp.setLocation(xp.getLocation() + "/a/b/c");
+ APIResult partUpdateResult = target().path("metastore/facts/").path(table).path("storages/S2/partition")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .put(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
+ assertEquals(partUpdateResult.getStatus(), Status.SUCCEEDED);
+
JAXBElement<XPartitionList> partitionsElement = target().path("metastore/facts").path(table)
.path("storages/S2/partitions")
.queryParam("sessionid", lensSessionId).request(mediaType)
@@ -2016,7 +2015,16 @@ public class TestMetastoreService extends LensJerseyTest {
XPartitionList partitions = partitionsElement.getValue();
assertNotNull(partitions);
assertEquals(partitions.getPartition().size(), 1);
-
+ XPartition readPartition = partitions.getPartition().get(0);
+ assertEquals(readPartition.getLocation(), xp.getLocation());
+ assertEquals(readPartition.getTimePartitionSpec(), xp.getTimePartitionSpec());
+ assertEquals(readPartition.getNonTimePartitionSpec(), xp.getNonTimePartitionSpec());
+ assertNotNull(readPartition.getFullPartitionSpec());
+ XTimePartSpecElement timePartSpec = readPartition.getTimePartitionSpec().getPartSpecElement().iterator().next();
+ XPartSpecElement fullPartSpec = readPartition.getFullPartitionSpec().getPartSpecElement().iterator().next();
+ assertEquals(timePartSpec.getKey(), fullPartSpec.getKey());
+ assertEquals(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format().format(JAXBUtils.getDateFromXML(
+ timePartSpec.getValue())), fullPartSpec.getValue());
DateTime date =
target().path("metastore/cubes").path("testCube").path("latestdate").queryParam("timeDimension", "dt")
.queryParam("sessionid", lensSessionId).request(mediaType).get(DateTime.class);
@@ -2063,7 +2071,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(partitions.getPartition().size(), 1);
// Drop again by values
- String[] val = new String[]{UpdatePeriod.HOURLY.format().format(partDate)};
+ String[] val = new String[]{HOURLY.format().format(partDate)};
dropResult = target().path("metastore/facts").path(table).path("storages/S2/partition")
.queryParam("values", StringUtils.join(val, ","))
.queryParam("sessionid", lensSessionId).request(mediaType)
@@ -2112,6 +2120,12 @@ public class TestMetastoreService extends LensJerseyTest {
.post(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
assertEquals(partAddResult.getStatus(), Status.SUCCEEDED);
+ xp.setLocation(xp.getLocation() + "/a/b/c");
+ APIResult partUpdateResult = target().path("metastore/dimtables/").path(table).path("storages/test/partition")
+ .queryParam("sessionid", lensSessionId).request(mediaType)
+ .put(Entity.xml(cubeObjectFactory.createXPartition(xp)), APIResult.class);
+ assertEquals(partUpdateResult.getStatus(), Status.SUCCEEDED);
+
JAXBElement<XPartitionList> partitionsElement = target().path("metastore/dimtables").path(table)
.path("storages/test/partitions")
.queryParam("sessionid", lensSessionId).request(mediaType)
@@ -2121,6 +2135,37 @@ public class TestMetastoreService extends LensJerseyTest {
assertNotNull(partitions);
assertEquals(partitions.getPartition().size(), 2);
+ assertEquals(partitions.getPartition().get(0).getLocation(), xp.getLocation());
+ assertEquals(partitions.getPartition().get(1).getLocation(), xp.getLocation());
+
+ // one is latest partition.
+ assertTrue(partitions.getPartition().get(0).getTimePartitionSpec() == null
+ || partitions.getPartition().get(1).getTimePartitionSpec() == null);
+ XPartition postedPartition, latestPartition;
+ if (partitions.getPartition().get(0).getTimePartitionSpec() == null) {
+ postedPartition = partitions.getPartition().get(1);
+ latestPartition = partitions.getPartition().get(0);
+ } else {
+ postedPartition = partitions.getPartition().get(0);
+ latestPartition = partitions.getPartition().get(1);
+ }
+
+ assertEquals(postedPartition.getTimePartitionSpec(), xp.getTimePartitionSpec());
+ assertEquals(postedPartition.getNonTimePartitionSpec(), xp.getNonTimePartitionSpec());
+ assertNotNull(postedPartition.getFullPartitionSpec());
+
+ XTimePartSpecElement timePartSpec = postedPartition.getTimePartitionSpec().getPartSpecElement().iterator().next();
+ XPartSpecElement fullPartSpec = postedPartition.getFullPartitionSpec().getPartSpecElement().iterator().next();
+ assertEquals(timePartSpec.getKey(), fullPartSpec.getKey());
+ assertEquals(UpdatePeriod.valueOf(xp.getUpdatePeriod().name()).format().format(JAXBUtils.getDateFromXML(
+ timePartSpec.getValue())), fullPartSpec.getValue());
+
+ assertNull(latestPartition.getTimePartitionSpec());
+ assertNull(latestPartition.getNonTimePartitionSpec());
+ assertEquals(latestPartition.getFullPartitionSpec().getPartSpecElement().get(0).getValue(),
+ "latest");
+
+
// Drop the partitions
APIResult dropResult = target().path("metastore/dimtables").path(table).path("storages/test/partitions")
.queryParam("sessionid", lensSessionId).request(mediaType)
@@ -2159,7 +2204,7 @@ public class TestMetastoreService extends LensJerseyTest {
assertEquals(partitions.getPartition().size(), 2);
// Drop again by values
- String[] val = new String[]{UpdatePeriod.HOURLY.format().format(partDate)};
+ String[] val = new String[]{HOURLY.format().format(partDate)};
dropResult = target().path("metastore/dimtables").path(table).path("storages/test/partition")
.queryParam("values", StringUtils.join(val, ","))
.queryParam("sessionid", lensSessionId).request(mediaType)
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-storage-db/src/main/java/org/apache/lens/storage/db/DBStorage.java
----------------------------------------------------------------------
diff --git a/lens-storage-db/src/main/java/org/apache/lens/storage/db/DBStorage.java b/lens-storage-db/src/main/java/org/apache/lens/storage/db/DBStorage.java
index e6d4132..03b8d43 100644
--- a/lens-storage-db/src/main/java/org/apache/lens/storage/db/DBStorage.java
+++ b/lens-storage-db/src/main/java/org/apache/lens/storage/db/DBStorage.java
@@ -25,6 +25,7 @@ import org.apache.lens.cube.metadata.Storage;
import org.apache.lens.cube.metadata.StoragePartitionDesc;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
/**
@@ -158,6 +159,21 @@ public class DBStorage extends Storage {
public void rollbackAddPartitions(List<StoragePartitionDesc> arg0) throws HiveException {
}
+ @Override
+ public void preUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
+ public void commitUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
+ @Override
+ public void rollbackUpdatePartition(List<Partition> partitions) throws HiveException {
+
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java
----------------------------------------------------------------------
diff --git a/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java b/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java
index 1e24828..92a0027 100644
--- a/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java
+++ b/lens-storage-db/src/test/java/org/apache/lens/storage/db/TestDBStorage.java
@@ -148,7 +148,7 @@ public class TestDBStorage {
// Assert for storage tables
for (String storage : storageTables.keySet()) {
- String storageTableName = MetastoreUtil.getDimStorageTableName(dimTblName, storage);
+ String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storage);
Assert.assertTrue(client.tableExists(storageTableName));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/src/site/apt/admin/hivedriver-config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/hivedriver-config.apt b/src/site/apt/admin/hivedriver-config.apt
index b668e16..80a3028 100644
--- a/src/site/apt/admin/hivedriver-config.apt
+++ b/src/site/apt/admin/hivedriver-config.apt
@@ -46,15 +46,11 @@ Hive driver configuration
*--+--+---+--+
|11|lens.driver.hive.connection.class|org.apache.lens.driver.hive.EmbeddedThriftConnection|The connection class from HiveDriver to HiveServer. The default is an embedded connection which does not require a remote hive server. For connecting to a hiveserver end point, remote connection should be used. The possible values are org.apache.lens.driver.hive.EmbeddedThriftConnection and org.apache.lens.driver.hive.RemoteThriftConnection.|
*--+--+---+--+
-|12|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2|
+|12|lens.driver.hive.cost.calculator.class|org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator|Cost calculator class. By default calculating cost through fact partitions.|
*--+--+---+--+
-|13|lens.driver.hive.priority.partition.weight.daily|0.75|Weight of monthly partition in cost calculation|
+|13|lens.driver.hive.hs2.connection.expiry.delay|600000|The idle time (in milliseconds) for expiring connection from hivedriver to HiveServer2|
*--+--+---+--+
-|14|lens.driver.hive.priority.partition.weight.hourly|1.0|Weight of monthly partition in cost calculation|
-*--+--+---+--+
-|15|lens.driver.hive.priority.partition.weight.monthly|0.5|Weight of monthly partition in cost calculation|
-*--+--+---+--+
-|16|lens.driver.hive.priority.ranges|VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW|Priority Ranges. The numbers are the costs of the query. \ |
+|14|lens.driver.hive.priority.ranges|VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW|Priority Ranges. The numbers are the costs of the query. \ |
| | | |The cost is calculated based on partition weights and fact weights. The interpretation of the default config is: \ |
| | | | \ |
| | | |cost \<= 7\ \ \ \ \ \ \ \ \ \ \ :\ \ \ \ \ Priority = VERY_HIGH \ |
@@ -70,4 +66,6 @@ Hive driver configuration
| | | |One use case in range tuning can be that you never want queries to run with VERY_HIGH, assuming no other changes, you'll modify the value of this param in hivedriver-site.xml to be HIGH,30.0,NORMAL,90,LOW\ |
| | | |via the configs, you can tune both the ranges and partition weights. this would give the end user more control. |
*--+--+---+--+
+|15|lens.driver.hive.query.hook.class|org.apache.lens.server.api.driver.NoOpDriverQueryHook|The query hook class for hive driver. By default hook is No op. To add a hook, you should look at the default implementation and from there it'll be easy to derive what value can be added through a new hook|
+*--+--+---+--+
The configuration parameters and their default values
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/7de74eb3/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index 353c171..63f2b3f 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -92,7 +92,7 @@ User CLI Commands
*--+--+
|close/bye|Releases all resources of the server session and exits the shell|
*--+--+
-|debug [[--enable] To print all logs on cli for debugging purpose]|prints all class lelvel logs on cli for debugging purpose|
+|debug [[--enable] To print all logs on cli for debugging purpose]|prints all class level logs and verbose logs on cli for debugging purpose. 'debug false' to turn off all class level logging and verbose level logging |
*--+--+
|get [--key] \<key\>|Fetches and prints session parameter specified with name <<<key>>> from lens server|
*--+--+
@@ -106,7 +106,7 @@ User CLI Commands
*--+--+
|show params|Fetches and prints all session parameter from lens server|
*--+--+
-|verbose [[--enable] Print the clilogger logs on cli]|Show cliLogger logs for command|
+|verbose [[--enable] Print the clilogger logs on cli]|Show cliLogger logs on cli'verbose false' turns off the cliLogger logs on console|
*--+--+
<<Lens Connection Commands>>
@@ -121,7 +121,7 @@ User CLI Commands
*--+--+
|create database [--db] \<database-name\> [--ignoreIfExists \<ignore-if-exists\>]|create a database with specified name. if <<<ignore-if-exists>>> is true, create will not be tried if already exists. Default is false|
*--+--+
-|drop database [--db] \<database-name\>|drop a database with specified name|
+|drop database [--db] \<database-name\> [--cascade ]|drop a database with specified name|
*--+--+
|show databases|displays list of all databases|
*--+--+
@@ -237,6 +237,10 @@ User CLI Commands
*--+--+
|fact timelines [--fact_name] \<fact_name\> [--storage_name \<storage_name\>] [--update_period \<update_period\>] [--time_dimension \<time_dimension\>]|get timelines for fact. Can optionally specify storage, update period and time dimension to filter by. Instead of time dimension, partition column can be directly passed as <<<time_dimension>>>|
*--+--+
+|fact update partitions [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [--path] \<partition-list-spec-path\>|update multiple partition to fact <<<fact_name>>>'s storage <<<storage_name>>>, reading partition list spec from <<<partition-list-spec-path>>>|
+*--+--+
+|fact update single-partition [--fact_name] \<fact_name\> [--storage_name] \<storage_name\> [--path] \<partition-spec-path\>|update single partition to fact <<<fact_name>>>'s storage <<<storage_name>>>, reading spec from <<<partition-spec-path>>>|
+*--+--+
|show facts [[--cube_name] \<cube_name\>]|display list of fact tables in current database. If optional <<<cube_name>>> is supplied, only facts belonging to cube <<<cube_name>>> will be displayed|
*--+--+
|update fact [--fact_name] \<fact_name\> [--path] \<path-to-fact-spec\>|update fact <<<fact_name>>> taking spec from <<<path-to-fact-spec>>>|
@@ -274,6 +278,10 @@ User CLI Commands
*--+--+
|dimtable list storages [--dimtable_name] \<dimtable_name\>|display list of storage associated to dimtable <<<dimtable_name>>>|
*--+--+
+|dimtable update partitions [--dimtable_name] \<dimtable_name\> [--storage_name] \<storage_name\> [--path] \<partition-list-spec-path\>|update multiple partition to dimtable <<<dimtable_name>>>'s storage <<<storage_name>>>, reading partition list spec from <<<partition-list-spec-path>>>|
+*--+--+
+|dimtable update single-partition [--dimtable_name] \<dimtable_name\> [--storage_name] \<storage_name\> [--path] \<partition-spec-path\>|update single partition to dimtable <<<dimtable_name>>>'s storage <<<storage_name>>>, reading spec from <<<partition-spec-path>>>|
+*--+--+
|drop dimtable [--dimtable_name] \<dimtable_name\> [--cascade \<cascade\>]|drop dimtable <<<dimtable_name>>>. If <<<cascade>>> is true, all the storage tables associated with the dimtable <<<dimtable_name>>> are also dropped. By default <<<cascade>>> is false|
*--+--+
|show dimtables [[--dimension_name] \<dimension_name\>]|display list of dimtables in current database. If optional <<<dimension_name>>> is supplied, only facts belonging to dimension <<<dimension_name>>> will be displayed|
[26/50] [abbrv] incubator-lens git commit: LENS-679 : Use slf4j in
api, regression, ml and client modules
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java
index ca59e3e..e7bc700 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/helpers/SessionHelper.java
@@ -35,15 +35,11 @@ import org.apache.lens.regression.util.AssertUtil;
import org.apache.lens.regression.util.Util;
import org.apache.lens.server.api.error.LensException;
-import org.apache.log4j.Logger;
-
-
-
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class SessionHelper extends ServiceManagerHelper {
- private static Logger logger = Logger.getLogger(SessionHelper.class);
-
private WebTarget servLens = ServiceManagerHelper.getServerLens();
private String sessionHandleString = ServiceManagerHelper.getSessionHandle();
@@ -75,7 +71,7 @@ public class SessionHelper extends ServiceManagerHelper {
formData.getForm());
AssertUtil.assertSucceededResponse(response);
String newSessionHandleString = response.readEntity(String.class);
- logger.info("Session Handle String" + newSessionHandleString);
+ log.info("Session Handle String:{}", newSessionHandleString);
return newSessionHandleString;
}
@@ -102,7 +98,7 @@ public class SessionHelper extends ServiceManagerHelper {
if (result.getMessage() == null) {
throw new LensException("Status message is null");
}
- logger.info("Closed Session : " + sessionHandleString);
+ log.info("Closed Session : {}", sessionHandleString);
}
/**
@@ -127,12 +123,12 @@ public class SessionHelper extends ServiceManagerHelper {
response = this.exec("get", "/session/params", servLens, null, query);
AssertUtil.assertSucceededResponse(response);
String responseString = response.readEntity(String.class);
- logger.info(responseString);
+ log.info(responseString);
HashMap<String, String> map = Util.stringListToMap(responseString);
if (!map.get(param).equals(value)) {
throw new LensException("Could not set property");
}
- logger.info("Added property " + param + " = " + value);
+ log.info("Added property {}={}", param, value);
}
public void setAndValidateParam(String param, String value) throws Exception {
@@ -156,7 +152,7 @@ public class SessionHelper extends ServiceManagerHelper {
* @param sessionHandleString
*/
public void addResourcesJar(String path, String sessionHandleString) throws JAXBException, LensException {
- logger.info("Adding Resources " + path);
+ log.info("Adding Resources {}", path);
FormBuilder formData = new FormBuilder();
formData.add("sessionid", sessionHandleString);
formData.add("type", "jar");
@@ -164,7 +160,7 @@ public class SessionHelper extends ServiceManagerHelper {
Response response = this
.exec("put", "/session/resources/add", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, null,
formData.getForm());
- logger.info("Response : " + response);
+ log.info("Response : {}", response);
AssertUtil.assertSucceeded(response);
}
@@ -179,7 +175,7 @@ public class SessionHelper extends ServiceManagerHelper {
* @param sessionHandleString
*/
public void removeResourcesJar(String path, String sessionHandleString) throws JAXBException, LensException {
- logger.info("Removing Resources " + path);
+ log.info("Removing Resources {}", path);
FormBuilder formData = new FormBuilder();
formData.add("sessionid", sessionHandleString);
formData.add("type", "jar");
@@ -187,7 +183,7 @@ public class SessionHelper extends ServiceManagerHelper {
Response response = this
.exec("put", "/session/resources/delete", servLens, null, null, MediaType.MULTIPART_FORM_DATA_TYPE, null,
formData.getForm());
- logger.info("Response : " + response);
+ log.info("Response : {}", response);
AssertUtil.assertSucceeded(response);
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
index abcea2b..62bd160 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/util/AssertUtil.java
@@ -24,15 +24,8 @@ import javax.ws.rs.core.Response;
import org.apache.lens.api.APIResult;
import org.apache.lens.server.api.error.LensException;
-import org.apache.log4j.Logger;
-
-
-
-
public class AssertUtil {
- private static final Logger LOGGER = Logger.getLogger(AssertUtil.class);
-
private AssertUtil() {
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
index 5db4fa6..6a2bc6b 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/util/Util.java
@@ -33,17 +33,17 @@ import org.apache.lens.api.APIResult;
import org.apache.lens.api.StringList;
import org.apache.lens.api.metastore.ObjectFactory;
-import org.apache.log4j.Logger;
-
import com.jcraft.jsch.ChannelExec;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class Util {
- private static final Logger LOGGER = Logger.getLogger(Util.class);
private static final String PROPERTY_FILE = "lens.properties";
private static Properties properties;
@@ -55,7 +55,7 @@ public class Util {
try {
if (properties == null) {
properties = new Properties();
- LOGGER.info("filename: " + filename);
+ log.info("filename: {}", filename);
InputStream confStream = Util.class.getResourceAsStream("/" + filename);
properties.load(confStream);
confStream.close();
@@ -63,7 +63,7 @@ public class Util {
return properties;
} catch (IOException e) {
- LOGGER.info(e.getStackTrace());
+ log.info("Error in getProperies:", e);
}
return null;
}
@@ -81,7 +81,7 @@ public class Util {
String host = Util.getProperty("lens.remote.host");
String password = Util.getProperty("lens.remote.password");
- LOGGER.info("Running command : " + command + " on host : " + host + " with user as " + userName);
+ log.info("Running command : {} on host : {} with user as {}", command, host, userName);
JSch jsch = new JSch();
Session session = jsch.getSession(userName, host, 22);
@@ -106,7 +106,7 @@ public class Util {
outputBuffer.append(toAppend);
readByte = commandOutput.read();
if (toAppend == '\n') {
- LOGGER.info(print.toString());
+ log.info(print.toString());
print = new StringBuilder();
} else {
print.append(toAppend);
@@ -123,7 +123,7 @@ public class Util {
out.println(str);
out.close();
} catch (IOException e) {
- LOGGER.info("File Exception : " + e);
+ log.info("File Exception : ", e);
}
}
@@ -177,7 +177,7 @@ public class Util {
try {
getMarshaller(clazz).marshal(root, stringWriter);
} catch (JAXBException e) {
- LOGGER.error("Not able to marshall", e);
+ log.error("Not able to marshall", e);
}
return stringWriter.toString();
}
@@ -191,7 +191,7 @@ public class Util {
return jaxbMarshaller;
} catch (JAXBException e) {
- LOGGER.error("Error : ", e);
+ log.error("Error : ", e);
}
return null;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java b/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
index 595ef67..cf7b17b 100644
--- a/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
+++ b/lens-regression/src/test/java/org/apache/lens/regression/sanity/ITSmokeTest.java
@@ -24,21 +24,20 @@ import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Method;
-
import org.apache.lens.regression.core.testHelper.BaseTestClass;
import org.apache.lens.regression.util.Util;
-import org.apache.log4j.Logger;
-
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.jcraft.jsch.JSchException;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
public class ITSmokeTest extends BaseTestClass {
- private static final Logger LOGGER = Logger.getLogger(ITSmokeTest.class);
private final String resourceDir = "src/test/resources";
private final String resultFile = resourceDir + "/result.data";
private final String smokeOutput = resourceDir + "/output.txt";
@@ -47,7 +46,7 @@ public class ITSmokeTest extends BaseTestClass {
@BeforeMethod(alwaysRun = true)
public void setUp(Method method) throws Exception {
- LOGGER.info("Test Name: " + method.getName());
+ log.info("Test Name: {}", method.getName());
}
@Test(enabled = true, groups = { "integration" })
@@ -58,23 +57,23 @@ public class ITSmokeTest extends BaseTestClass {
String populateMetastoreCommand = "bash " + exampleScript + " populate-metastore -db " + dbName;
String runQueriesCommand = "bash " + exampleScript + " runqueries -db " + dbName;
- LOGGER.info("Creating schema : ");
+ log.info("Creating schema : ");
String output = Util.runRemoteCommand(sampleMetastoreCommand);
System.out.println("Output : " + output);
- LOGGER.info("Output : " + output);
+ log.info("Output : {}", output);
- LOGGER.info("Populating Metastore : ");
+ log.info("Populating Metastore : ");
output = Util.runRemoteCommand(populateMetastoreCommand);
- LOGGER.info("Output : " + output);
+ log.info("Output : {}", output);
- LOGGER.info("Running Queries in Background : ");
+ log.info("Running Queries in Background : ");
output = Util.runRemoteCommand("nohup " + runQueriesCommand + " > smoke.log 2>&1 &");
- LOGGER.info("Output : " + output);
+ log.info("Output : {}", output);
waitToComplete();
output = Util.runRemoteCommand("cat smoke.log");
- LOGGER.info("Output : " + output);
+ log.info("Output : {}", output);
Util.writeFile(smokeOutput, output);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
index c3fe8eb..a3fd6ac 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/MinQueryCostSelector.java
@@ -25,10 +25,8 @@ import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.Logger;
public class MinQueryCostSelector implements DriverSelector {
- public static final Logger LOG = Logger.getLogger(MinQueryCostSelector.class);
/**
* Returns the driver that has the minimum query cost.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
index 5b6ec3b..17578b6 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metrics/MethodMetricsFactory.java
@@ -30,7 +30,6 @@ import org.apache.lens.server.model.LensResourceMethod;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.Logger;
import org.glassfish.jersey.server.ContainerRequest;
import org.glassfish.jersey.server.model.ResourceMethod;
@@ -38,14 +37,16 @@ import org.glassfish.jersey.server.model.ResourceMethod;
import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
+
import lombok.Getter;
import lombok.NonNull;
+import lombok.extern.slf4j.Slf4j;
/**
* Factory for creating MethodMetrics objects on demand.
*/
+@Slf4j
public class MethodMetricsFactory {
- public static final Logger LOG = Logger.getLogger(MethodMetricsFactory.class);
private final MetricRegistry metricRegistry;
@@ -83,7 +84,7 @@ public class MethodMetricsFactory {
if (result == null) {
synchronized (this) {
result = methodMetricsMap.get(name);
- LOG.info("Creating MethodMetrics of name: " + name);
+ log.info("Creating MethodMetrics of name: {}", name);
result = new MethodMetrics(
metricRegistry.meter(name(name, "meter")),
metricRegistry.timer(name(name, "timer")),
@@ -146,7 +147,7 @@ public class MethodMetricsFactory {
*/
public void clear() {
synchronized (this) {
- LOG.info("clearing factory");
+ log.info("clearing factory");
for (Map.Entry<String, MethodMetrics> entry : methodMetricsMap.entrySet()) {
metricRegistry.remove(name(entry.getKey(), "meter"));
metricRegistry.remove(name(entry.getKey(), "timer"));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 71a37f0..8046927 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -34,24 +34,19 @@ import org.apache.lens.server.api.query.DriverSelectorQueryContext.DriverQueryCo
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.util.LensUtil;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.session.SessionState;
import lombok.Getter;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public abstract class AbstractQueryContext implements Serializable {
private static final long serialVersionUID = 1L;
/**
- * The Constant LOG
- */
- public static final Log LOG = LogFactory.getLog(AbstractQueryContext.class);
-
- /**
* The user query.
*/
@Getter
@@ -129,7 +124,7 @@ public abstract class AbstractQueryContext implements Serializable {
if (conf.getBoolean(LensConfConstants.ENABLE_QUERY_METRICS, LensConfConstants.DEFAULT_ENABLE_QUERY_METRICS)) {
UUID metricId = UUID.randomUUID();
conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, metricId.toString());
- LOG.info("Generated metric id: " + metricId + " for query: " + query);
+ log.info("Generated metric id: {} for query: {}", metricId, query);
}
driverContext = new DriverSelectorQueryContext(query, conf, drivers, mergeDriverConf);
userQuery = query;
@@ -178,7 +173,7 @@ public abstract class AbstractQueryContext implements Serializable {
public void estimateCostForDrivers() throws LensException {
Map<LensDriver, DriverEstimateRunnable> estimateRunnables = getDriverEstimateRunnables();
for (LensDriver driver : estimateRunnables.keySet()) {
- LOG.info("Running estimate for driver " + driver);
+ log.info("Running estimate for driver {}", driver);
estimateRunnables.get(driver).run();
}
}
@@ -258,7 +253,7 @@ public abstract class AbstractQueryContext implements Serializable {
.append(" Cause :")
.append(expMsg)
.toString();
- LOG.error("Setting driver cost failed for driver " + driver + " Cause: " + failureCause, e);
+ log.error("Setting driver cost failed for driver {} Cause: {}", driver, failureCause, e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/90415a10/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 3e7de17..7a2da99 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -28,22 +28,16 @@ import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import lombok.Getter;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+@Slf4j
public class DriverSelectorQueryContext {
/**
- * The constant LOG
- */
- public static final Log LOG = LogFactory.getLog(DriverSelectorQueryContext.class);
-
-
- /**
* The selected driver.
*/
@Getter
@@ -200,7 +194,7 @@ public class DriverSelectorQueryContext {
driverQueryContext.setDriverQueryPlan(driver.explain(qctx));
succeededOnAtleastOneDriver = true;
} catch (Exception e) {
- LOG.error("Setting driver plan failed for driver " + driver, e);
+ log.error("Setting driver plan failed for driver {}", driver, e);
String expMsg = LensUtil.getCauseMessage(e);
driverQueryContext.setDriverQueryPlanGenerationError(e);
detailedFailureCause.append("\n Driver :").append(driver.getClass().getName());
[03/50] [abbrv] incubator-lens git commit: [maven-release-plugin]
prepare release apache-lens-2.2.0-beta-incubating
Posted by sh...@apache.org.
[maven-release-plugin] prepare release apache-lens-2.2.0-beta-incubating
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/9c45f1cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/9c45f1cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/9c45f1cb
Branch: refs/heads/LENS-581
Commit: 9c45f1cb4c69ec5de6fe3320abdd5bd85c250e9f
Parents: f85e2c2
Author: jdhok <jd...@apache.org>
Authored: Tue Jul 7 16:50:23 2015 +0530
Committer: jdhok <jd...@apache.org>
Committed: Tue Jul 7 16:50:23 2015 +0530
----------------------------------------------------------------------
checkstyle/pom.xml | 9 ++++++---
lens-api/pom.xml | 5 ++---
lens-cli/pom.xml | 5 ++---
lens-client/pom.xml | 5 ++---
lens-cube/pom.xml | 5 ++---
lens-dist/pom.xml | 5 ++---
lens-driver-hive/pom.xml | 5 ++---
lens-driver-jdbc/pom.xml | 20 +++++++++-----------
lens-examples/pom.xml | 5 ++---
lens-ml-dist/pom.xml | 2 +-
lens-ml-lib/pom.xml | 5 ++---
lens-query-lib/pom.xml | 5 ++---
lens-regression/pom.xml | 5 ++---
lens-server-api/pom.xml | 5 ++---
lens-server/pom.xml | 22 ++++++++++------------
lens-storage-db/pom.xml | 5 ++---
pom.xml | 11 +++++------
17 files changed, 55 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/checkstyle/pom.xml
----------------------------------------------------------------------
diff --git a/checkstyle/pom.xml b/checkstyle/pom.xml
index 9bbbe6c..cd45d00 100644
--- a/checkstyle/pom.xml
+++ b/checkstyle/pom.xml
@@ -17,8 +17,7 @@
limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -30,7 +29,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
<name>Lens Checkstyle Rules</name>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
<build>
<plugins>
@@ -50,4 +49,8 @@
</plugin>
</plugins>
</build>
+
+ <scm>
+ <tag>apache-lens-2.2.0-beta-incubating</tag>
+ </scm>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-api/pom.xml b/lens-api/pom.xml
index 184f8a8..250102d 100644
--- a/lens-api/pom.xml
+++ b/lens-api/pom.xml
@@ -19,14 +19,13 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens API</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-cli/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cli/pom.xml b/lens-cli/pom.xml
index 7b98523..26d34d7 100644
--- a/lens-cli/pom.xml
+++ b/lens-cli/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens CLI</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-client/pom.xml
----------------------------------------------------------------------
diff --git a/lens-client/pom.xml b/lens-client/pom.xml
index 540c203..708c399 100644
--- a/lens-client/pom.xml
+++ b/lens-client/pom.xml
@@ -19,13 +19,12 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-client</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-cube/pom.xml
----------------------------------------------------------------------
diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml
index 47601c1..eb750e1 100644
--- a/lens-cube/pom.xml
+++ b/lens-cube/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Cube</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-cube</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-dist/pom.xml b/lens-dist/pom.xml
index 8581362..b5c6b6c 100644
--- a/lens-dist/pom.xml
+++ b/lens-dist/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Distribution</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-driver-hive/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/pom.xml b/lens-driver-hive/pom.xml
index d156d0c..cebed79 100644
--- a/lens-driver-hive/pom.xml
+++ b/lens-driver-hive/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Hive Driver</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-driver-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/pom.xml b/lens-driver-jdbc/pom.xml
index 380b16a..e73895d 100644
--- a/lens-driver-jdbc/pom.xml
+++ b/lens-driver-jdbc/pom.xml
@@ -19,14 +19,13 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Driver for JDBC</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-driver-jdbc</artifactId>
@@ -103,17 +102,16 @@
</goals>
<configuration>
<target>
- <property name="build.compiler" value="extJavac"/>
- <property name="compile_classpath" refid="maven.compile.classpath"/>
- <mkdir dir="${project.build.directory}/additional-classes"/>
- <javac includeantruntime="false" srcdir="src/additional/java"
- destdir="${project.build.directory}/additional-classes" classpath="${compile_classpath}"/>
- <mkdir dir="${project.build.directory}/testjars"/>
+ <property name="build.compiler" value="extJavac" />
+ <property name="compile_classpath" refid="maven.compile.classpath" />
+ <mkdir dir="${project.build.directory}/additional-classes" />
+ <javac includeantruntime="false" srcdir="src/additional/java" destdir="${project.build.directory}/additional-classes" classpath="${compile_classpath}" />
+ <mkdir dir="${project.build.directory}/testjars" />
<jar destfile="${project.build.directory}/testjars/test.jar">
- <fileset dir="${project.build.directory}/additional-classes" includes="ClassLoaderTestClass.class"/>
+ <fileset dir="${project.build.directory}/additional-classes" includes="ClassLoaderTestClass.class" />
</jar>
<jar destfile="${project.build.directory}/testjars/serde.jar">
- <fileset dir="${project.build.directory}/additional-classes" includes="DatabaseJarSerde.class"/>
+ <fileset dir="${project.build.directory}/additional-classes" includes="DatabaseJarSerde.class" />
</jar>
</target>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-examples/pom.xml
----------------------------------------------------------------------
diff --git a/lens-examples/pom.xml b/lens-examples/pom.xml
index aae70a4..e9ae2a8 100644
--- a/lens-examples/pom.xml
+++ b/lens-examples/pom.xml
@@ -19,14 +19,13 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Examples</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-ml-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
index 26be98d..d7b69f5 100644
--- a/lens-ml-dist/pom.xml
+++ b/lens-ml-dist/pom.xml
@@ -26,7 +26,7 @@
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-ml-dist</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-ml-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index 63d72da..32a3be8 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens ML Lib</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<properties>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-query-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-query-lib/pom.xml b/lens-query-lib/pom.xml
index 05c42b0..6a7837f 100644
--- a/lens-query-lib/pom.xml
+++ b/lens-query-lib/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Query Library</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-query-lib</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-regression/pom.xml
----------------------------------------------------------------------
diff --git a/lens-regression/pom.xml b/lens-regression/pom.xml
index 59d1284..5679a72 100644
--- a/lens-regression/pom.xml
+++ b/lens-regression/pom.xml
@@ -19,14 +19,13 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Regression</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-regression</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-server-api/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server-api/pom.xml b/lens-server-api/pom.xml
index 45ea8e4..09871f4 100644
--- a/lens-server-api/pom.xml
+++ b/lens-server-api/pom.xml
@@ -19,13 +19,12 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-server-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 1a90f69..e436e32 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens Server</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<properties>
@@ -289,20 +288,19 @@
</goals>
<configuration>
<target>
- <property name="build.compiler" value="extJavac"/>
- <property name="compile_classpath" refid="maven.compile.classpath"/>
- <mkdir dir="${project.build.directory}/additional-classes"/>
- <javac includeantruntime="false" srcdir="src/additional/java"
- destdir="${project.build.directory}/additional-classes" classpath="${compile_classpath}"/>
- <mkdir dir="${project.build.directory}/testjars"/>
+ <property name="build.compiler" value="extJavac" />
+ <property name="compile_classpath" refid="maven.compile.classpath" />
+ <mkdir dir="${project.build.directory}/additional-classes" />
+ <javac includeantruntime="false" srcdir="src/additional/java" destdir="${project.build.directory}/additional-classes" classpath="${compile_classpath}" />
+ <mkdir dir="${project.build.directory}/testjars" />
<jar destfile="${project.build.directory}/testjars/test.jar">
- <fileset dir="${project.build.directory}/additional-classes" includes="ClassLoaderTestClass.class"/>
+ <fileset dir="${project.build.directory}/additional-classes" includes="ClassLoaderTestClass.class" />
</jar>
<jar destfile="${project.build.directory}/testjars/test2.jar">
- <fileset dir="${project.build.directory}/additional-classes" includes="ClassLoaderTestClass2.class"/>
+ <fileset dir="${project.build.directory}/additional-classes" includes="ClassLoaderTestClass2.class" />
</jar>
<jar destfile="${project.build.directory}/testjars/serde.jar">
- <fileset dir="${project.build.directory}/additional-classes" includes="DatabaseJarSerde.class"/>
+ <fileset dir="${project.build.directory}/additional-classes" includes="DatabaseJarSerde.class" />
</jar>
</target>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/lens-storage-db/pom.xml
----------------------------------------------------------------------
diff --git a/lens-storage-db/pom.xml b/lens-storage-db/pom.xml
index baca606..ce27408 100644
--- a/lens-storage-db/pom.xml
+++ b/lens-storage-db/pom.xml
@@ -19,15 +19,14 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<name>Lens DB storage</name>
<parent>
<artifactId>apache-lens</artifactId>
<groupId>org.apache.lens</groupId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</parent>
<artifactId>lens-storage-db</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/9c45f1cb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a937dda..e665568 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,8 +19,7 @@
under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -115,7 +114,7 @@
<groupId>org.apache.lens</groupId>
<artifactId>apache-lens</artifactId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
<name>Lens</name>
<packaging>pom</packaging>
<description>Unified Analytics Platform</description>
@@ -300,7 +299,7 @@
<connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-lens.git</connection>
<developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-lens.git</developerConnection>
<url>https://git-wip-us.apache.org/repos/asf/incubator-lens.git</url>
- <tag>HEAD</tag>
+ <tag>apache-lens-2.2.0-beta-incubating</tag>
</scm>
<reporting>
@@ -575,7 +574,7 @@
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</dependency>
</dependencies>
<executions>
@@ -1323,7 +1322,7 @@
<dependency>
<groupId>org.apache.lens</groupId>
<artifactId>checkstyle</artifactId>
- <version>2.2.0-beta-incubating-SNAPSHOT</version>
+ <version>2.2.0-beta-incubating</version>
</dependency>
</dependencies>
<executions>
[25/50] [abbrv] incubator-lens git commit: LENS-670: Add REST API for
serving logs for better debuggability
Posted by sh...@apache.org.
LENS-670: Add REST API for serving logs for better debuggability
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/5e70beec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/5e70beec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/5e70beec
Branch: refs/heads/LENS-581
Commit: 5e70beec40cbf25d81709df74898a55ce21de158
Parents: c65fa3c
Author: Amareshwari Sriramadasu <am...@gmail.com>
Authored: Wed Jul 22 12:31:34 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Wed Jul 22 12:31:34 2015 +0530
----------------------------------------------------------------------
.../org/apache/lens/server/LogResource.java | 87 ++++++++++++
.../server/query/QueryExecutionServiceImpl.java | 18 +--
.../apache/lens/server/util/UtilityMethods.java | 19 +++
.../src/main/resources/lensserver-default.xml | 8 +-
.../org/apache/lens/server/LensJerseyTest.java | 1 +
.../apache/lens/server/TestLensApplication.java | 13 ++
.../org/apache/lens/server/TestLogResource.java | 131 +++++++++++++++++++
lens-server/src/test/resources/lens-site.xml | 2 +-
lens-server/src/test/resources/log4j.properties | 9 ++
src/site/apt/admin/config.apt | 130 +++++++++---------
src/site/apt/admin/deployment.apt | 24 ++++
11 files changed, 359 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/main/java/org/apache/lens/server/LogResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/LogResource.java b/lens-server/src/main/java/org/apache/lens/server/LogResource.java
new file mode 100644
index 0000000..d6d3348
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/LogResource.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import javax.ws.rs.core.StreamingOutput;
+
+import org.apache.lens.server.util.UtilityMethods;
+
+/**
+ * The logs resource
+ */
+@Path("/logs")
+public class LogResource {
+
+ private static final String LOG_FILE_EXTN = ".log";
+
+ /**
+ * Tells whether log resource if up or not
+ *
+ * @return message
+ */
+ @GET
+ @Produces(MediaType.TEXT_PLAIN)
+ public String getMessage() {
+ return "Logs resource is up!";
+ }
+
+ /**
+ * Get logs corresponding to logSegregrationStr
+ *
+ * @param logSegregrationStr log segregation string - can be request id for all requests; query handle for queries
+ *
+ * @return streaming log
+ */
+ @GET
+ @Path("/{logSegregrationStr}")
+ @Produces({MediaType.APPLICATION_OCTET_STREAM})
+ public Response getLogs(@PathParam("logSegregrationStr") String logSegregrationStr) {
+ String logFileName = System.getProperty("lens.log.dir") + "/" + logSegregrationStr + LOG_FILE_EXTN;
+ final File logFile = new File(logFileName);
+ if (!logFile.exists()) {
+ return Response.status(Status.NOT_FOUND).entity("Log file for '" + logSegregrationStr + "' not found").build();
+ }
+
+ StreamingOutput stream = new StreamingOutput() {
+ @Override
+ public void write(OutputStream os) throws IOException {
+ FileInputStream fin = new FileInputStream(logFile);
+ try {
+ UtilityMethods.pipe(fin, os);
+ } finally {
+ fin.close();
+ }
+ }
+ };
+ return Response.ok(stream).type(MediaType.APPLICATION_OCTET_STREAM).build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 47d27d0..63569d8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -2292,22 +2292,6 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
log.info("Persisted " + allQueries.size() + " queries");
}
- /**
- * Pipe.
- *
- * @param is the is
- * @param os the os
- * @throws IOException Signals that an I/O exception has occurred.
- */
- private void pipe(InputStream is, OutputStream os) throws IOException {
- int n;
- byte[] buffer = new byte[4096];
- while ((n = is.read(buffer)) > -1) {
- os.write(buffer, 0, n);
- os.flush();
- }
- }
-
/*
* (non-Javadoc)
*
@@ -2348,7 +2332,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
try {
FileSystem fs = resultPath.getFileSystem(ctx.getConf());
fin = fs.open(resultPath);
- pipe(fin, os);
+ UtilityMethods.pipe(fin, os);
} finally {
if (fin != null) {
fin.close();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java b/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
index 715347f..0ee1c04 100644
--- a/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
+++ b/lens-server/src/main/java/org/apache/lens/server/util/UtilityMethods.java
@@ -18,6 +18,9 @@
*/
package org.apache.lens.server.util;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Map;
@@ -149,4 +152,20 @@ public final class UtilityMethods {
}
return sb.toString();
}
+
+ /**
+ * Pipe input stream to output stream
+ *
+ * @param is the is
+ * @param os the os
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public static void pipe(InputStream is, OutputStream os) throws IOException {
+ int n;
+ byte[] buffer = new byte[4096];
+ while ((n = is.read(buffer)) > -1) {
+ os.write(buffer, 0, n);
+ os.flush();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index 2b540b8..d26beb7 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -360,7 +360,7 @@
<property>
<name>lens.server.ws.resourcenames</name>
- <value>session,metastore,query,quota,scheduler,index</value>
+ <value>session,metastore,query,quota,scheduler,index,log</value>
<description>These JAX-RS resources would be started in the specified order when lens-server starts up</description>
</property>
@@ -389,6 +389,12 @@
</property>
<property>
+ <name>lens.server.log.ws.resource.impl</name>
+ <value>org.apache.lens.server.LogResource</value>
+ <description>Implementation class for Log Resource</description>
+ </property>
+
+ <property>
<name>lens.server.metastore.ws.resource.impl</name>
<value>org.apache.lens.server.metastore.MetastoreResource</value>
<description>Implementation class for Metastore Resource</description>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
index fb3e67b..44aa0aa 100644
--- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java
@@ -113,6 +113,7 @@ public abstract class LensJerseyTest extends JerseyTest {
@BeforeSuite
public void startAll() throws Exception {
LOG.info("Before suite");
+ System.setProperty("lens.log.dir", "target/");
TestRemoteHiveDriver.createHS2Service();
System.out.println("Remote hive server started!");
HiveConf hiveConf = new HiveConf();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/test/java/org/apache/lens/server/TestLensApplication.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestLensApplication.java b/lens-server/src/test/java/org/apache/lens/server/TestLensApplication.java
index a267fe0..4c8a583 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestLensApplication.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestLensApplication.java
@@ -74,6 +74,19 @@ public class TestLensApplication extends LensJerseyTest {
Assert.assertEquals(response.readEntity(String.class), "OK");
}
+ /**
+ * Test log resources loaded.
+ *
+ * @throws InterruptedException the interrupted exception
+ */
+ @Test
+ public void testLogResourceLoaded() throws InterruptedException {
+ final WebTarget target = target().path("logs");
+ final Response response = target.request().get();
+ Assert.assertEquals(response.getStatus(), 200);
+ Assert.assertEquals(response.readEntity(String.class), "Logs resource is up!");
+ }
+
@Test
public void testMetricService() {
MetricsService metrics = ((MetricsService) LensServices.get().getService(MetricsService.NAME));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/test/java/org/apache/lens/server/TestLogResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestLogResource.java b/lens-server/src/test/java/org/apache/lens/server/TestLogResource.java
new file mode 100644
index 0000000..f2cf37c
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/TestLogResource.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server;
+
+import static org.testng.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.Response;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.test.TestProperties;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Tests log resource
+ */
+@Test(groups = "unit-test")
+@Slf4j
+public class TestLogResource extends LensJerseyTest {
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.glassfish.jersey.test.JerseyTest#setUp()
+ */
+ @BeforeTest
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.glassfish.jersey.test.JerseyTest#tearDown()
+ */
+ @AfterTest
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.glassfish.jersey.test.JerseyTest#configure()
+ */
+ @Override
+ protected Application configure() {
+ enable(TestProperties.LOG_TRAFFIC);
+ enable(TestProperties.DUMP_ENTITY);
+
+ return new ResourceConfig(LogResource.class);
+ }
+
+ /**
+ * Test response up.
+ */
+ @Test
+ public void testResponseUp() {
+ log.info("Testing log resource");
+ WebTarget target = target().path("logs");
+ String s = target.request().get(String.class);
+ assertEquals("Logs resource is up!", s);
+ }
+
+ /**
+ * Test not found
+ */
+ @Test
+ public void testRandomResponse() {
+ log.info("Testing random log");
+ WebTarget target = target().path("logs");
+ Response response = target.path("random").request().get();
+ assertEquals(response.getStatus(), 404);
+ final String expectedErrMsg = "Log file for 'random' not found";
+ assertEquals(response.readEntity(String.class), expectedErrMsg);
+ }
+
+ /**
+ * Test log file
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testLogFile() throws IOException {
+ log.info("Testing logfile");
+ WebTarget target = target().path("logs");
+ Response response = target.path("logresource").request().get();
+ assertEquals(response.getStatus(), 200);
+
+ InputStream in = (InputStream) response.getEntity();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try {
+ IOUtils.copyBytes(in, bos, new Configuration());
+ } finally {
+ bos.close();
+ in.close();
+ }
+
+ String logs = new String(bos.toByteArray());
+ assertTrue(logs.contains("Testing logfile"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index 3bcbf0e..41c7eb6 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -96,7 +96,7 @@
<property>
<name>lens.server.ws.resourcenames</name>
- <value>session,metastore,query,quota,scheduler,index,test</value>
+ <value>session,metastore,query,quota,scheduler,index,log,test</value>
<description>These JAX-RS resources would be started in the specified order when lens-server starts up</description>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/lens-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/log4j.properties b/lens-server/src/test/resources/log4j.properties
index e0f9daa..69ff32c 100644
--- a/lens-server/src/test/resources/log4j.properties
+++ b/lens-server/src/test/resources/log4j.properties
@@ -28,6 +28,9 @@ log4j.additivity.org.apache.lens=false
log4j.logger.org.apache.lens.server.stats.TestLogStatisticsStore$MyLoggableLens=INFO, STDOUT, TEST_LOG_FILE
log4j.additivity.org.apache.lens.server.stats.TestLogStatisticsStore$MyLoggableLens=false
+log4j.logger.org.apache.lens.server.TestLogResource=INFO, STDOUT, TEST_LOG_RESOURCE
+log4j.additivity.org.apache.lens.server.TestLogResource=true
+
# Configuration for appenders
log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender
log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout
@@ -39,3 +42,9 @@ log4j.appender.TEST_LOG_FILE.File=target/test.log
log4j.appender.TEST_LOG_FILE.layout=org.apache.log4j.PatternLayout
log4j.appender.TEST_LOG_FILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %X{logSegregationId} [%t] %-5p %c %x - %m%n
log4j.appender.TEST_LOG_FILE.Threshold=INFO
+
+log4j.appender.TEST_LOG_RESOURCE=org.apache.log4j.RollingFileAppender
+log4j.appender.TEST_LOG_RESOURCE.File=target/logresource.log
+log4j.appender.TEST_LOG_RESOURCE.layout=org.apache.log4j.PatternLayout
+log4j.appender.TEST_LOG_RESOURCE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %X{logSegregationId} [%t] %-5p %c %x - %m%n
+log4j.appender.TEST_LOG_RESOURCE.Threshold=INFO
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index c5bf70c..f850190 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -89,134 +89,136 @@ Lens server configuration
*--+--+---+--+
|31|lens.server.index.ws.resource.impl|org.apache.lens.server.IndexResource|Implementation class for Index Resource|
*--+--+---+--+
-|32|lens.server.mail.from.address|blah@company.com|The from field in the notifier mail to the submitter.|
+|32|lens.server.log.ws.resource.impl|org.apache.lens.server.LogResource|Implementation class for Log Resource|
*--+--+---+--+
-|33|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail|
+|33|lens.server.mail.from.address|blah@company.com|The from field in the notifier mail to the submitter.|
*--+--+---+--+
-|34|lens.server.mail.port|25|SMTP Port|
+|34|lens.server.mail.host|mail-host.company.com|SMTP Host for sending mail|
*--+--+---+--+
-|35|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.|
+|35|lens.server.mail.port|25|SMTP Port|
*--+--+---+--+
-|36|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.|
+|36|lens.server.mail.smtp.connectiontimeout|15000|Socket connection timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 15 seconds.|
*--+--+---+--+
-|37|lens.server.max.finished.queries|100|Maximum number of finished queries which lens server will keep in memory before purging.|
+|37|lens.server.mail.smtp.timeout|30000|Socket read timeout value in milliseconds. This timeout is implemented by java.net.Socket. Default is 30 seconds.|
*--+--+---+--+
-|38|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
+|38|lens.server.max.finished.queries|100|Maximum number of finished queries which lens server will keep in memory before purging.|
*--+--+---+--+
-|39|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource|
+|39|lens.server.metastore.service.impl|org.apache.lens.server.metastore.CubeMetastoreServiceImpl|Implementation class for metastore service|
*--+--+---+--+
-|40|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.|
+|40|lens.server.metastore.ws.resource.impl|org.apache.lens.server.metastore.MetastoreResource|Implementation class for Metastore Resource|
*--+--+---+--+
-|41|lens.server.metrics.ganglia.host| |The ganglia host name|
+|41|lens.server.metrics.csv.directory.path|metrics/|Path of the directory in which to report metrics as separate csv files.|
*--+--+---+--+
-|42|lens.server.metrics.ganglia.port| |The ganglia port|
+|42|lens.server.metrics.ganglia.host| |The ganglia host name|
*--+--+---+--+
-|43|lens.server.metrics.graphite.host| |The graphite host name|
+|43|lens.server.metrics.ganglia.port| |The ganglia port|
*--+--+---+--+
-|44|lens.server.metrics.graphite.port| |The graphite port|
+|44|lens.server.metrics.graphite.host| |The graphite host name|
*--+--+---+--+
-|45|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
+|45|lens.server.metrics.graphite.port| |The graphite port|
*--+--+---+--+
-|46|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.|
+|46|lens.server.metrics.reporting.period|10|The reporting period for metrics. The value is in seconds|
*--+--+---+--+
-|47|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource|
+|47|lens.server.mode|OPEN|The mode in which server should run. Allowed values are OPEN, READ_ONLY, METASTORE_READONLY, METASTORE_NODROP. OPEN mode will allow all requests. READ_ONLY mode will allow all requests on session resouce and only GET requests on all other resources. METASTORE_READONLY will allow GET on metastore and all other requests in other services. METASTORE_NODROP will not allow DELETE on metastore, will allow all other requests.|
*--+--+---+--+
-|48|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location|
+|48|lens.server.multipart.ws.feature.impl|org.glassfish.jersey.media.multipart.MultiPartFeature|Implementation class for query scheduler resource|
*--+--+---+--+
-|49|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.|
+|49|lens.server.persist.location|file:///tmp/lensserver|The directory in which lens server will persist its state when it is going down. The location be on any Hadoop compatible file system. Server will read from the location when it is restarted and recovery is enabled. So, Server should have both read and write permissions to the location|
*--+--+---+--+
-|50|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ |
+|50|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.|
+*--+--+---+--+
+|51|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ |
| | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ |
| | | |Or define shortcuts for certain frequently used queries :) |
*--+--+---+--+
-|51|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
+|52|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
*--+--+---+--+
-|52|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
+|53|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
*--+--+---+--+
-|53|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
+|54|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
*--+--+---+--+
-|54|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
+|55|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
*--+--+---+--+
-|55|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
+|56|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
*--+--+---+--+
-|56|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
+|57|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
*--+--+---+--+
-|57|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
+|58|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
*--+--+---+--+
-|58|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
+|59|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
*--+--+---+--+
-|59|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
+|60|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
*--+--+---+--+
-|60|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
+|61|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
*--+--+---+--+
-|61|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
+|62|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
*--+--+---+--+
-|62|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
+|63|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|63|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
+|64|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
*--+--+---+--+
-|64|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|65|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|65|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|66|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|66|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|67|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|67|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
+|68|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
*--+--+---+--+
-|68|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|69|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|69|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|70|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|70|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|71|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|71|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|72|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|72|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|73|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|73|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
+|74|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
*--+--+---+--+
-|74|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
+|75|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
*--+--+---+--+
-|75|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|76|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|76|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
+|77|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|77|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|78|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|78|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|79|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|79|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|80|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|80|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|81|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|81|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|82|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|82|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|83|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|83|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|84|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|84|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|85|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|85|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|86|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|86|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|87|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|87|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|88|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|88|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|89|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|89|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|90|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|90|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|91|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|91|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|92|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
-|92|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|93|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|93|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|94|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|94|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|95|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
*--+--+---+--+
-|95|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|96|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/5e70beec/src/site/apt/admin/deployment.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/deployment.apt b/src/site/apt/admin/deployment.apt
index 088a695..b4f4d0a 100644
--- a/src/site/apt/admin/deployment.apt
+++ b/src/site/apt/admin/deployment.apt
@@ -71,3 +71,27 @@ Lens server deployment
* METASTORE_NODROP: DELETE requests on metastore are not accepted, all other requests are accepted
* OPEN: This is the default mode. All requests are accepted
+
+* Serving logs over REST
+
+ Lens server comes with LogResource which can serve logs under LENS_LOG_DIR over REST. To enable serving logs,
+ administrator can enable log resource by adding log to configuration property "lens.server.ws.resourcenames".
+ Config changes look the following :
+
+===
+
+ <property>
+ <name>lens.server.ws.resourcenames</name>
+ <value>session,metastore,query,index,log</value>
+ <description>These JAX-RS resources would be started in the specified order when lens-server starts up</description>
+ </property>
+
+ <property>
+ <name>lens.server.log.ws.resource.impl</name>
+ <value>org.apache.lens.server.LogResource</value>
+ <description>Implementation class for Log Resource</description>
+ </property>
+
+===
+
+ If resourcenames does not contain "log", it is not enabled.
[17/50] [abbrv] incubator-lens git commit: LENS-674 : Highlight the
website running part in website/documentation
Posted by sh...@apache.org.
LENS-674 : Highlight the website running part in website/documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/46dc5adf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/46dc5adf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/46dc5adf
Branch: refs/heads/LENS-581
Commit: 46dc5adf482d4985d048d72c14891439e0c7abbe
Parents: bbb9351
Author: Yash Sharma <ya...@gmail.com>
Authored: Mon Jul 20 15:40:47 2015 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Jul 20 15:40:47 2015 +0530
----------------------------------------------------------------------
src/site/apt/developer/contribute.apt | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/46dc5adf/src/site/apt/developer/contribute.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/developer/contribute.apt b/src/site/apt/developer/contribute.apt
index 73206a8..642993c 100644
--- a/src/site/apt/developer/contribute.apt
+++ b/src/site/apt/developer/contribute.apt
@@ -509,8 +509,15 @@ Developer Documentation : How to contribute to Apache Lens?
The new doc files can added in src/site of parent module. Or doc change can be done in existing files under src/site.
Update the config files and run the TestGenerateConfigDoc test calss for updating the config docs.
- Once the changes are done, contributor can run mvn site:run, which will start localhost doc server on port 8080,
- which can opened through browser and doc can be validated.
+ Once the changes are done, contributor can run the below command.
+
++---+
+
+ mvn site:run
+
++---+
+
+ This will start localhost doc server on {{{http://localhost:8080}http://localhost:8080}}, which can opened through browser and doc can be validated.
** How to update the config docs?
[28/50] [abbrv] incubator-lens git commit: LENS-619: Applying Query
Launching Constraints before allowing a query to be launched (Also contains
LENS-687 and LENS-688)
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java
new file mode 100644
index 0000000..4dde18a
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultEstimatedQueryCollectionTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import static org.apache.lens.server.query.collect.QueryCollectUtil.createQueriesSetWithUserStubbing;
+
+import static org.mockito.Mockito.*;
+
+import static org.testng.Assert.assertEquals;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import org.testng.annotations.Test;
+
+import com.beust.jcommander.internal.Sets;
+import com.google.common.collect.Iterables;
+
+public class DefaultEstimatedQueryCollectionTest {
+
+ private static final String MOCK_USER = "MockUserEmail";
+
+ @Test
+ public void testGetTotalQueryCostForUserWithZeroLaunchedQueries() throws LensException {
+
+ QueryCollection mockQueries = mock(QueryCollection.class);
+ when(mockQueries.getQueries(MOCK_USER)).thenReturn(Sets.<QueryContext>newLinkedHashSet());
+
+ EstimatedQueryCollection queries = new DefaultEstimatedQueryCollection(mockQueries);
+ QueryCost actualQueryCost = queries.getTotalQueryCost(MOCK_USER);
+ assertEquals(actualQueryCost, new FactPartitionBasedQueryCost(0));
+ }
+
+ @Test
+ public void testGetTotalQueryCostForUserWithMoreThanOneLaunchedQueries() throws LensException {
+
+ QueryCollection mockQueries = mock(QueryCollection.class);
+ Set<QueryContext> mockQueriesSet = createQueriesSetWithUserStubbing(2, MOCK_USER);
+ when(mockQueries.getQueries(MOCK_USER)).thenReturn(mockQueriesSet);
+
+ final QueryContext query0 = Iterables.get(mockQueriesSet, 0);
+ final QueryContext query1 = Iterables.get(mockQueriesSet, 1);
+
+ final QueryCost mockCost0 = mock(QueryCost.class);
+ final QueryCost mockCost1 = mock(QueryCost.class);
+ final QueryCost mockCost0Plus0 = mock(QueryCost.class);
+ final QueryCost mockCost0Plus0Plus1 = mock(QueryCost.class);
+
+ when(query0.getSelectedDriverQueryCost()).thenReturn(mockCost0);
+ when(query1.getSelectedDriverQueryCost()).thenReturn(mockCost1);
+
+ when(mockCost0.add(mockCost0)).thenReturn(mockCost0Plus0);
+ when(mockCost0Plus0.add(mockCost1)).thenReturn(mockCost0Plus0Plus1);
+
+ QueryCost actualQueryCost = new DefaultEstimatedQueryCollection(mockQueries).getTotalQueryCost(MOCK_USER);
+ assertEquals(actualQueryCost, mockCost0Plus0Plus1);
+ }
+
+ @Test
+ public void testAddAndRemoveAndGetQueriesMethod() throws LensException {
+
+ QueryContext mockQuery = mock(QueryContext.class);
+ LensDriver mockSelectedDriver = mock(LensDriver.class);
+ QueryCost mockQueryCost = mock(QueryCost.class);
+ when(mockQuery.getSelectedDriver()).thenReturn(mockSelectedDriver);
+ when(mockQuery.getSelectedDriverQueryCost()).thenReturn(mockQueryCost);
+
+ QueryCollection mockQueries = mock(QueryCollection.class);
+ EstimatedQueryCollection queries = new DefaultEstimatedQueryCollection(mockQueries);
+
+ queries.add(mockQuery);
+ assertEquals(queries.getQueriesCount(mockSelectedDriver), 1);
+ verify(mockQueries, times(1)).add(mockQuery);
+
+ queries.remove(mockQuery);
+ assertEquals(queries.getQueriesCount(mockSelectedDriver), 0);
+ verify(mockQueries, times(1)).remove(mockQuery);
+ }
+
+ @Test(expectedExceptions = IllegalStateException.class)
+ public void testCheckStateMustRecognizeIllegalStateWhenSelectedDriverIsNotSet() throws LensException {
+
+ QueryContext mockQuery = mock(QueryContext.class);
+ /* Setting selected driver cost, however since selected driver is not set. This should result in
+ IllegalStateException */
+ when(mockQuery.getSelectedDriverQueryCost()).thenReturn(mock(QueryCost.class));
+
+ QueryCollection mockQueries = mock(QueryCollection.class);
+ DefaultEstimatedQueryCollection queries = new DefaultEstimatedQueryCollection(mockQueries);
+ queries.checkState(mockQuery);
+ }
+
+ @Test(expectedExceptions = IllegalStateException.class)
+ public void testCheckStateMustRecognizeIllegalStateWhenQueryCostIsNotSet() {
+
+ QueryContext mockQuery = mock(QueryContext.class);
+ /* Selected Driver is set, however since selected driver query cost is not set. This should result in
+ IllegalStateException. */
+ when(mockQuery.getSelectedDriver()).thenReturn(mock(LensDriver.class));
+
+ QueryCollection mockQueries = mock(QueryCollection.class);
+ DefaultEstimatedQueryCollection queries = new DefaultEstimatedQueryCollection(mockQueries);
+ queries.checkState(mockQuery);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultQueryCollectionTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultQueryCollectionTest.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultQueryCollectionTest.java
new file mode 100644
index 0000000..7a81e83
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/DefaultQueryCollectionTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import static org.apache.lens.server.query.collect.QueryCollectUtil.*;
+
+import static org.mockito.Mockito.mock;
+
+import static org.testng.Assert.assertEquals;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+import org.testng.annotations.Test;
+
+public class DefaultQueryCollectionTest {
+
+ private static final String MOCK_USER = "MockUserEmail";
+
+ /* Note: Since verification of addition/removal required calling get methods,
+ hence methods getQueriesCount and getQueries(user) are indirectly getting tested in these tests */
+
+ @Test
+ public void testAddMethodAddsQueriesToAllViews(){
+
+ /* Initialization */
+ final int noOfQueriesUsedInTest = 2;
+ QueryCollection queries = createQueriesInstanceWithUserStubbing(noOfQueriesUsedInTest, MOCK_USER);
+
+ /* Verification 1: Verifies that queries were added to queries list by calling getQueriesCount which gets results
+ from queries list */
+ assertEquals(queries.getQueriesCount(), noOfQueriesUsedInTest);
+
+ /* Verification 2: Verifies that queries were added to queries per user map by calling getQueries(user) method,
+ which gets information from queries per user map */
+
+ assertEquals(queries.getQueries(MOCK_USER).size(), noOfQueriesUsedInTest);
+ }
+
+ @Test
+ public void testRemoveMethodRemovesFromAllViews() {
+
+ /* Initialization */
+ QueryContext mockQuery = mock(QueryContext.class);
+ QueryCollection queries = stubMockQueryAndCreateQueriesInstance(mockQuery, MOCK_USER);
+
+ /* Execution */
+ queries.remove(mockQuery);
+
+ /* Verification 1: Verifies that queries were removed from queries list by calling getQueriesCount which gets
+ results from queries list */
+ assertEquals(queries.getQueriesCount(), 0);
+
+ /* Verification 2: Verifies that queries were removed from queries per user map by calling getQueries(user) method,
+ which gets information from queries per user map */
+
+ assertEquals(queries.getQueries(MOCK_USER).size(), 0);
+ }
+
+ @Test
+ public void testGetQueriesMustReturnCopyOfUnderlyingCollection() {
+
+ /* Initialization */
+ final int noOfQueriesUsedInTest = 2;
+ QueryCollection queries = createQueriesInstanceWithMockedQueries(noOfQueriesUsedInTest);
+
+ /* Execution: Get queries and empty returned collection */
+ Set<QueryContext> copiedSet = queries.getQueries();
+ copiedSet.clear();
+
+ /* System under set (queries) should still have the added queries, which can be verified by checking count */
+ assertEquals(queries.getQueriesCount(), noOfQueriesUsedInTest);
+ }
+
+ @Test
+ public void testGetQueriesPerUserMustReturnCopyOfUnderlyingCollection() {
+
+ /* Initialization */
+ final int noOfQueriesUsedInTest = 2;
+ QueryCollection queries = createQueriesInstanceWithUserStubbing(noOfQueriesUsedInTest, MOCK_USER);
+
+ /* Execution: Get queries for user and empty returned collection */
+ Set<QueryContext> copiedSet = queries.getQueries(MOCK_USER);
+ copiedSet.clear();
+
+ /* System under set (queries) should still have the added queries, which can be verified by checking count */
+ assertEquals(queries.getQueries(MOCK_USER).size(), noOfQueriesUsedInTest);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelectorTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelectorTest.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelectorTest.java
new file mode 100644
index 0000000..62e371d
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/IntersectingWaitingQueriesSelectorTest.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+
+import org.testng.annotations.Test;
+import org.testng.collections.Sets;
+
+import com.beust.jcommander.internal.Lists;
+import com.google.common.collect.ImmutableSet;
+
+public class IntersectingWaitingQueriesSelectorTest {
+
+ @Test
+ public void testFindCommonQueriesWhenInputIsEmptyListOfSets() {
+
+ List<Set<QueryContext>> candidateQueriesSets = Lists.newArrayList();
+ Set<QueryContext> expectedEligibleQueries = Sets.newHashSet();
+
+ testFindCommonQueries(candidateQueriesSets, expectedEligibleQueries);
+ }
+
+ @Test
+ public void testFindCommonQueriesWhenNoCommonQueryIsPresent() {
+
+ QueryContext eligibleQuery1 = mock(QueryContext.class);
+ QueryContext eligibleQuery2 = mock(QueryContext.class);
+
+ Set<QueryContext> eligibleQueriesSetA = Sets.newHashSet(Arrays.asList(eligibleQuery1));
+ Set<QueryContext> eligibleQueriesSetB = Sets.newHashSet(Arrays.asList(eligibleQuery2));
+
+ List<Set<QueryContext>> candidateQueriesSets = Arrays.asList(eligibleQueriesSetA, eligibleQueriesSetB);
+ Set<QueryContext> expectedEligibleQueries = Sets.newHashSet();
+
+ testFindCommonQueries(candidateQueriesSets, expectedEligibleQueries);
+ }
+
+ @Test
+ public void testFindCommonQueriesWhenACommonQueryIsPresent() {
+
+ QueryContext eligibleQuery1 = mock(QueryContext.class);
+ QueryContext eligibleQuery2 = mock(QueryContext.class);
+ QueryContext eligibleQuery3 = mock(QueryContext.class);
+
+ Set<QueryContext> eligibleQueriesSetA = Sets.newHashSet(Arrays.asList(eligibleQuery1, eligibleQuery2));
+ Set<QueryContext> eligibleQueriesSetB = Sets.newHashSet(Arrays.asList(eligibleQuery1, eligibleQuery3));
+
+ List<Set<QueryContext>> candidateQueriesSets = Arrays.asList(eligibleQueriesSetA, eligibleQueriesSetB);
+ Set<QueryContext> expectedEligibleQueries = Sets.newHashSet(Arrays.asList(eligibleQuery1));
+
+ testFindCommonQueries(candidateQueriesSets, expectedEligibleQueries);
+ }
+
+ private void testFindCommonQueries(final List<Set<QueryContext>> queriesSets,
+ final Set<QueryContext> expectedEligibleQueries) {
+
+ IntersectingWaitingQueriesSelector selector = new IntersectingWaitingQueriesSelector(mock(ImmutableSet.class));
+
+ Set<QueryContext> actualCommonQueries = selector.findCommonQueries(queriesSets);
+ assertEquals(actualCommonQueries, expectedEligibleQueries);
+ }
+
+ @Test
+ public void testPrepareAllSelectionPolicies() {
+
+ WaitingQueriesSelectionPolicy p1 = mock(WaitingQueriesSelectionPolicy.class);
+ WaitingQueriesSelectionPolicy p2 = mock(WaitingQueriesSelectionPolicy.class);
+ WaitingQueriesSelectionPolicy dp1 = mock(WaitingQueriesSelectionPolicy.class);
+ WaitingQueriesSelectionPolicy dp2 = mock(WaitingQueriesSelectionPolicy.class);
+
+ FinishedLensQuery mockFinishedQuery = mock(FinishedLensQuery.class);
+ when(mockFinishedQuery.getDriverSelectionPolicies()).thenReturn(ImmutableSet.of(dp1, dp2));
+
+ IntersectingWaitingQueriesSelector selector = new IntersectingWaitingQueriesSelector(ImmutableSet.of(p1, p2));
+ assertEquals(selector.prepareAllSelectionPolicies(mockFinishedQuery), ImmutableSet.of(p1, p2, dp1, dp2));
+ }
+
+ @Test
+ public void testPrepareAllSelectionPoliciesWithNoDriverSelectionPolicy() {
+
+ WaitingQueriesSelectionPolicy p1 = mock(WaitingQueriesSelectionPolicy.class);
+ WaitingQueriesSelectionPolicy p2 = mock(WaitingQueriesSelectionPolicy.class);
+
+ final ImmutableSet<WaitingQueriesSelectionPolicy> emptySet = ImmutableSet.copyOf(
+ Sets.<WaitingQueriesSelectionPolicy>newHashSet());
+
+ FinishedLensQuery mockFinishedQuery = mock(FinishedLensQuery.class);
+ when(mockFinishedQuery.getDriverSelectionPolicies()).thenReturn(emptySet);
+
+ IntersectingWaitingQueriesSelector selector = new IntersectingWaitingQueriesSelector(ImmutableSet.of(p1, p2));
+
+ assertEquals(selector.prepareAllSelectionPolicies(mockFinishedQuery), ImmutableSet.of(p1, p2));
+ }
+
+ @Test
+ public void testSelectQueriesCommonBetweenAllSelectionPolicies(){
+
+ QueryContext q1 = mock(QueryContext.class);
+ QueryContext q2 = mock(QueryContext.class);
+ QueryContext q3 = mock(QueryContext.class);
+
+ /* eligibleQueriesSet1, eligibleQueriesSet2, eligibleQueriesSet3 have q1 in common */
+ Set<QueryContext> eligibleQueriesSet1 = Sets.newHashSet(Arrays.asList(q1, q2));
+ Set<QueryContext> eligibleQueriesSet2 = Sets.newHashSet(Arrays.asList(q1, q3));
+ Set<QueryContext> eligibleQueriesSet3 = Sets.newHashSet(Arrays.asList(q1, q2));
+
+ FinishedLensQuery mockFinishedQuery = mock(FinishedLensQuery.class);
+ EstimatedImmutableQueryCollection mockWaitingQueries = mock(EstimatedImmutableQueryCollection.class);
+ WaitingQueriesSelectionPolicy policy1 = mock(WaitingQueriesSelectionPolicy.class);
+ WaitingQueriesSelectionPolicy policy2 = mock(WaitingQueriesSelectionPolicy.class);
+ WaitingQueriesSelectionPolicy driverSelectionPolicy = mock(WaitingQueriesSelectionPolicy.class);
+
+ when(mockFinishedQuery.getDriverSelectionPolicies()).thenReturn(ImmutableSet.of(driverSelectionPolicy));
+
+ /* selection policy1 will return eligibleQueriesSet1 */
+ when(policy1.selectQueries(mockFinishedQuery, mockWaitingQueries)).thenReturn(eligibleQueriesSet1);
+
+ /* selection policy2 will return eligibleQueriesSet2 */
+ when(policy2.selectQueries(mockFinishedQuery, mockWaitingQueries)).thenReturn(eligibleQueriesSet2);
+
+ /* driver selection policy will return eligibleQueriesSet3 */
+ when(driverSelectionPolicy.selectQueries(mockFinishedQuery, mockWaitingQueries)).thenReturn(eligibleQueriesSet3);
+
+ WaitingQueriesSelector selector = new IntersectingWaitingQueriesSelector(ImmutableSet.of(policy1, policy2));
+
+ /* selector should return only eligibleQuery1, as this is the only common eligible waiting query returned
+ * by both selection policies */
+ Set<QueryContext> actualEligibleQueries = selector.selectQueries(mockFinishedQuery, mockWaitingQueries);
+ Set<QueryContext> expectedEligibleQueries = Sets.newHashSet(Arrays.asList(q1));
+
+ assertEquals(actualEligibleQueries, expectedEligibleQueries);
+ }
+
+ @Test(expectedExceptions = NullPointerException.class)
+ public void testSelectorMustNotAcceptNullAsSelectionPolicies() {
+ new IntersectingWaitingQueriesSelector(null);
+ }
+
+ @Test
+ public void testSelectQueriesWithNoSelectionPolicies(){
+
+ FinishedLensQuery mockFinishedQuery = mock(FinishedLensQuery.class);
+ EstimatedImmutableQueryCollection mockWaitingQueries = mock(EstimatedImmutableQueryCollection.class);
+ Set<WaitingQueriesSelectionPolicy> emptySetOfPolicies = Sets.newHashSet();
+
+ when(mockFinishedQuery.getDriverSelectionPolicies()).thenReturn(ImmutableSet.copyOf(emptySetOfPolicies));
+
+ WaitingQueriesSelector selector = new IntersectingWaitingQueriesSelector(ImmutableSet.copyOf(emptySetOfPolicies));
+
+ /* selector should return an empty set as no selection policy is available */
+ Set<QueryContext> actualEligibleQueries = selector.selectQueries(mockFinishedQuery, mockWaitingQueries);
+
+ assertTrue(actualEligibleQueries.isEmpty());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java
new file mode 100644
index 0000000..51fcf00
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import static java.lang.reflect.Modifier.isPublic;
+import static java.lang.reflect.Modifier.isSynchronized;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertTrue;
+
+import java.lang.reflect.Method;
+import java.util.Set;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+import com.google.common.collect.Sets;
+
+public class QueryCollectUtil {
+
+ protected QueryCollectUtil() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static Set<QueryContext> createQueriesSetWithUserStubbing(final int reqNoOfMockQueries,
+ final String mockUser) {
+
+ final Set<QueryContext> mockQueries = getMockQueriesSet(reqNoOfMockQueries);
+ stubSubmittedUserInMockQueries(mockQueries, mockUser);
+ return mockQueries;
+ }
+
+ public static void stubSubmittedUserInMockQueries(final Set<QueryContext> mockQueries, final String mockUser) {
+
+ for (QueryContext mockQuery : mockQueries) {
+ when(mockQuery.getSubmittedUser()).thenReturn(mockUser);
+ }
+ }
+
+ public static Set<QueryContext> getMockQueriesSet(final int reqNoOfMockQueries) {
+
+ Set<QueryContext> mockQueries = Sets.newLinkedHashSet();
+
+ for (int i = 1; i <= reqNoOfMockQueries; ++i) {
+ mockQueries.add(mock(QueryContext.class));
+ }
+ return mockQueries;
+ }
+
+ public static QueryCollection createQueriesInstanceWithMockedQueries(final int reqNoOfMockQueries) {
+
+ final Set<QueryContext> mockQueries = getMockQueriesSet(reqNoOfMockQueries);
+ return new DefaultQueryCollection(mockQueries);
+ }
+
+ public static QueryCollection createQueriesInstanceWithUserStubbing(final int reqNoOfMockQueries,
+ final String mockUser) {
+
+ final Set<QueryContext> mockQueries = createQueriesSetWithUserStubbing(reqNoOfMockQueries, mockUser);
+ return new DefaultQueryCollection(mockQueries);
+ }
+
+ public static QueryCollection stubMockQueryAndCreateQueriesInstance(final QueryContext mockQuery,
+ final String mockUser) {
+
+ Set<QueryContext> mockQueries = Sets.newHashSet(mockQuery);
+ stubSubmittedUserInMockQueries(mockQueries, mockUser);
+ return new DefaultQueryCollection(mockQueries);
+ }
+
+ public static <T> void testAllMethodsHaveSynchronizedKeyword(final Class<T> clazz) {
+
+ Method[] allMethods = clazz.getDeclaredMethods();
+
+ for (Method method : allMethods) {
+
+ int modifierSet = method.getModifiers();
+ if (isPublic(modifierSet)) {
+ /* If the method is public, then it should also have synchronized modifier on it */
+ assertTrue(isSynchronized(modifierSet), "method [" + method.getName() + "] in class ["
+ + clazz.getName() + "] does not have synchronized modifier on it");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/collect/ThreadSafetyTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/ThreadSafetyTest.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/ThreadSafetyTest.java
new file mode 100644
index 0000000..74a0389
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/ThreadSafetyTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import org.testng.annotations.Test;
+
+public class ThreadSafetyTest {
+
+ @Test
+ public void testThreadSafetyForThreadSafeQueries() {
+
+ QueryCollectUtil.testAllMethodsHaveSynchronizedKeyword(ThreadSafeQueryCollection.class);
+ }
+
+ @Test
+ public void testThreadSafetyForThreadSafeEstimatedQueries() {
+
+ QueryCollectUtil.testAllMethodsHaveSynchronizedKeyword(ThreadSafeEstimatedQueryCollection.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyTest.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyTest.java
new file mode 100644
index 0000000..b3e9d38
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/UserSpecificWaitingQueriesSelectionPolicyTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.collect;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+
+import org.testng.annotations.Test;
+
+public class UserSpecificWaitingQueriesSelectionPolicyTest {
+
+ @Test
+ public void testSelectQueries() {
+
+ FinishedLensQuery mockFinishedQuery = mock(FinishedLensQuery.class);
+ EstimatedImmutableQueryCollection mockWaitingQueries = mock(EstimatedImmutableQueryCollection.class);
+ String mockUser = "MockUser";
+ Set expectedQueriesSet = mock(Set.class);
+
+ when(mockFinishedQuery.getSubmitter()).thenReturn(mockUser);
+ when(mockWaitingQueries.getQueries(mockUser)).thenReturn(expectedQueriesSet);
+
+ WaitingQueriesSelectionPolicy selectionPolicy = new UserSpecificWaitingQueriesSelectionPolicy();
+ Set actualEligibleQueries = selectionPolicy.selectQueries(mockFinishedQuery, mockWaitingQueries);
+
+ assertEquals(actualEligibleQueries, expectedQueriesSet);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java
new file mode 100644
index 0000000..ab030cc
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.constraint;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+public class DefaultQueryLaunchingConstraintsCheckerTest {
+
+ @Test
+ public void testCanLaunchShouldReturnFalseWhenAtleastOneConstraintFails() {
+
+ QueryContext mockCandidateQuery = mock(QueryContext.class);
+ EstimatedImmutableQueryCollection mockRunningQueries = mock(EstimatedImmutableQueryCollection.class);
+
+ final QueryLaunchingConstraint constraint1 = mock(QueryLaunchingConstraint.class);
+ final QueryLaunchingConstraint constraint2 = mock(QueryLaunchingConstraint.class);
+ QueryLaunchingConstraintsChecker constraintsChecker
+ = new DefaultQueryLaunchingConstraintsChecker(ImmutableSet.of(constraint1, constraint2));
+
+ final QueryLaunchingConstraint driverConstraint = mock(QueryLaunchingConstraint.class);
+ when(mockCandidateQuery.getSelectedDriverQueryConstraints()).thenReturn(ImmutableSet.of(driverConstraint));
+
+ /* Constraint1 stubbed to pass */
+ when(constraint1.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
+
+ /* Constraint2 stubbed to fail */
+ when(constraint2.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(false);
+
+ /* DriverConstraint stubbed to fail */
+ when(driverConstraint.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(false);
+
+ /* Execute test */
+ boolean canLaunchQuery = constraintsChecker.canLaunch(mockCandidateQuery, mockRunningQueries);
+
+ /* Verify */
+ Assert.assertFalse(canLaunchQuery);
+ }
+
+ @Test
+ public void testCanLaunchShouldReturnTrueWhenAllConstraintPass() {
+
+ QueryContext mockCandidateQuery = mock(QueryContext.class);
+ EstimatedImmutableQueryCollection mockRunningQueries = mock(EstimatedImmutableQueryCollection.class);
+
+ final QueryLaunchingConstraint constraint1 = mock(QueryLaunchingConstraint.class);
+ final QueryLaunchingConstraint constraint2 = mock(QueryLaunchingConstraint.class);
+
+ QueryLaunchingConstraintsChecker constraintsChecker
+ = new DefaultQueryLaunchingConstraintsChecker(ImmutableSet.of(constraint1, constraint2));
+
+ final QueryLaunchingConstraint driverConstraint = mock(QueryLaunchingConstraint.class);
+ when(mockCandidateQuery.getSelectedDriverQueryConstraints()).thenReturn(ImmutableSet.of(driverConstraint));
+
+ /* all constraints stubbed to pass */
+ when(constraint1.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
+ when(constraint2.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
+ when(driverConstraint.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
+
+ /* Execute test */
+ boolean canLaunchQuery = constraintsChecker.canLaunch(mockCandidateQuery, mockRunningQueries);
+
+ /* Verify */
+ Assert.assertTrue(canLaunchQuery);
+ }
+
+ @Test
+ public void testCanLaunchShouldReturnTrueWhenConstraintsSetIsEmpty() {
+
+ QueryContext mockCandidateQuery = mock(QueryContext.class);
+ EstimatedImmutableQueryCollection mockRunningQueries = mock(EstimatedImmutableQueryCollection.class);
+
+ ImmutableSet<QueryLaunchingConstraint> emptySetOfConstraints
+ = ImmutableSet.copyOf(Sets.<QueryLaunchingConstraint>newHashSet());
+
+ when(mockCandidateQuery.getSelectedDriverQueryConstraints()).thenReturn(emptySetOfConstraints);
+
+ QueryLaunchingConstraintsChecker constraintsChecker
+ = new DefaultQueryLaunchingConstraintsChecker(ImmutableSet.copyOf(emptySetOfConstraints));
+
+ boolean canLaunchQuery = constraintsChecker.canLaunch(mockCandidateQuery, mockRunningQueries);
+ Assert.assertTrue(canLaunchQuery);
+ }
+
+ @Test(expectedExceptions = NullPointerException.class)
+ public void testConstraintsCheckerMustNotAcceptNullConstraintsSet() {
+ new DefaultQueryLaunchingConstraintsChecker(null);
+ }
+
+ @Test
+ public void testPrepareAllConstraints() {
+
+ QueryLaunchingConstraint c1 = mock(QueryLaunchingConstraint.class);
+ QueryLaunchingConstraint c2 = mock(QueryLaunchingConstraint.class);
+
+ QueryLaunchingConstraint dc1 = mock(QueryLaunchingConstraint.class);
+ QueryLaunchingConstraint dc2 = mock(QueryLaunchingConstraint.class);
+
+ QueryContext mockCandidateQuery = mock(QueryContext.class);
+ when(mockCandidateQuery.getSelectedDriverQueryConstraints()).thenReturn(ImmutableSet.of(dc1, dc2));
+
+ DefaultQueryLaunchingConstraintsChecker constraintsChecker
+ = new DefaultQueryLaunchingConstraintsChecker(ImmutableSet.of(c1, c2));
+ assertEquals(constraintsChecker.prepareAllConstraints(mockCandidateQuery), ImmutableSet.of(c1, c2, dc1, dc2));
+ }
+
+ @Test
+ public void testPrepareAllConstraintsWithNoDriverConstraints() {
+
+ QueryLaunchingConstraint c1 = mock(QueryLaunchingConstraint.class);
+ QueryLaunchingConstraint c2 = mock(QueryLaunchingConstraint.class);
+
+ ImmutableSet<QueryLaunchingConstraint> emptySet = ImmutableSet.copyOf(Sets.<QueryLaunchingConstraint>newHashSet());
+ QueryContext mockCandidateQuery = mock(QueryContext.class);
+ when(mockCandidateQuery.getSelectedDriverQueryConstraints()).thenReturn(emptySet);
+
+ DefaultQueryLaunchingConstraintsChecker constraintsChecker
+ = new DefaultQueryLaunchingConstraintsChecker(ImmutableSet.of(c1, c2));
+ assertEquals(constraintsChecker.prepareAllConstraints(mockCandidateQuery), ImmutableSet.of(c1, c2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactoryTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactoryTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactoryTest.java
new file mode 100644
index 0000000..f42e79e
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintFactoryTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.constraint;
+
+import static org.testng.Assert.assertEquals;
+
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Optional;
+
+public class TotalQueryCostCeilingConstraintFactoryTest {
+
+ @DataProvider
+ public Object[][] dpTestCreate() {
+ return new Object[][] {
+ {"-1.0", new TotalQueryCostCeilingConstraint(Optional.<QueryCost>absent())},
+ {"0.0", new TotalQueryCostCeilingConstraint(Optional.<QueryCost>of(new FactPartitionBasedQueryCost(0.0)))},
+ {"90.0", new TotalQueryCostCeilingConstraint(Optional.<QueryCost>of(new FactPartitionBasedQueryCost(90.0)))},
+ };
+ }
+ @Test(dataProvider = "dpTestCreate")
+ public void testCreate(final String totalQueryCostCeilingPerUser, final QueryLaunchingConstraint expectedConstraint) {
+
+ Configuration conf = new Configuration();
+ conf.set(LensConfConstants.TOTAL_QUERY_COST_CEILING_PER_USER_KEY, totalQueryCostCeilingPerUser);
+
+ ConfigBasedObjectCreationFactory<QueryLaunchingConstraint> fac = new TotalQueryCostCeilingConstraintFactory();
+ assertEquals(fac.create(conf), expectedConstraint);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java
new file mode 100644
index 0000000..460190a
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.query.constraint;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Optional;
+
+public class TotalQueryCostCeilingConstraintTest {
+
+ @DataProvider
+ public Object[][] dpTestAllowsLaunchOfQuery() {
+ return new Object[][] { {7.0, true} , {90.0, true}, {91.0, false}};
+ }
+
+ @Test(dataProvider = "dpTestAllowsLaunchOfQuery")
+ public void testAllowsLaunchOfQuery(final double totalQueryCostForCurrentUser, final boolean expectedCanLaunchQuery) {
+
+ final QueryCost totalQueryCostCeilingPerUser = new FactPartitionBasedQueryCost(90.0);
+ final QueryLaunchingConstraint queryConstraint
+ = new TotalQueryCostCeilingConstraint(Optional.of(totalQueryCostCeilingPerUser));
+
+ final QueryContext query = mock(QueryContext.class);
+ final EstimatedImmutableQueryCollection launchedQueries = mock(EstimatedImmutableQueryCollection.class);
+ final String mockUser = "MockUser";
+
+ when(query.getSubmittedUser()).thenReturn(mockUser);
+ when(launchedQueries.getTotalQueryCost(mockUser))
+ .thenReturn(new FactPartitionBasedQueryCost(totalQueryCostForCurrentUser));
+
+ boolean actualCanLaunchQuery = queryConstraint.allowsLaunchOf(query, launchedQueries);
+ assertEquals(actualCanLaunchQuery, expectedCanLaunchQuery);
+ }
+
+ @Test
+ public void testAllowsLaunchOfQueryWhenCeilingIsAbsent() {
+
+ final QueryLaunchingConstraint queryConstraint
+ = new TotalQueryCostCeilingConstraint(Optional.<QueryCost>absent());
+
+ final QueryContext query = mock(QueryContext.class);
+ final EstimatedImmutableQueryCollection launchedQueries = mock(EstimatedImmutableQueryCollection.class);
+
+ boolean actualCanLaunchQuery = queryConstraint.allowsLaunchOf(query, launchedQueries);
+ assertEquals(actualCanLaunchQuery, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index 41c7eb6..bd93828 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -163,5 +163,4 @@
<name>lens.server.estimate.timeout.millis</name>
<value>120000</value>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/lens-server/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/log4j.properties b/lens-server/src/test/resources/log4j.properties
index 69ff32c..9cc1e36 100644
--- a/lens-server/src/test/resources/log4j.properties
+++ b/lens-server/src/test/resources/log4j.properties
@@ -35,7 +35,7 @@ log4j.additivity.org.apache.lens.server.TestLogResource=true
log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender
log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout
log4j.appender.STDOUT.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} [%X{logSegregationId}] [%t] %-5p %c %x - %m%n
-log4j.appender.STDOUT.Threshold=INFO
+log4j.appender.STDOUT.Threshold=ERROR
log4j.appender.TEST_LOG_FILE=org.apache.log4j.RollingFileAppender
log4j.appender.TEST_LOG_FILE.File=target/test.log
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/src/site/apt/admin/config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/config.apt b/src/site/apt/admin/config.apt
index f850190..2e1dbe7 100644
--- a/src/site/apt/admin/config.apt
+++ b/src/site/apt/admin/config.apt
@@ -127,98 +127,106 @@ Lens server configuration
*--+--+---+--+
|50|lens.server.query.acceptors| |Query Acceptors configured. Query acceptors are consulted first, before anything happens for the given query. They can either return null or return a messaging indicating why the given query shouldn't be accepted. These can be used to filter out queries at the earliest.|
*--+--+---+--+
-|51|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ |
+|51|lens.server.query.launching.constraint.factories|org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory|Factories used to instantiate constraints enforced on queries by lens. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint. A query will be launched only if all constraints pass.|
+*--+--+---+--+
+|52|lens.server.query.phase1.rewriters| |Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will be passed for validation and rewriting to hql query.\ |
| | | |Use cases will be to use extra intelligence to convert user query to optimized cube query. \ |
| | | |Or define shortcuts for certain frequently used queries :) |
*--+--+---+--+
-|52|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
+|53|lens.server.query.service.impl|org.apache.lens.server.query.QueryExecutionServiceImpl|Implementation class for query execution service|
+*--+--+---+--+
+|54|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
+*--+--+---+--+
+|55|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
+*--+--+---+--+
+|56|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
*--+--+---+--+
-|53|lens.server.query.state.logger.enabled|true|Disable or enable the query state logger with this config. The location for the logger can be specified in log4j properties for the class org.apache.lens.server.query.QueryExecutionServiceImpl.QueryStatusLogger|
+|57|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
*--+--+---+--+
-|54|lens.server.query.ws.resource.impl|org.apache.lens.server.query.QueryServiceResource|Implementation class for Query Resource|
+|58|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
*--+--+---+--+
-|55|lens.server.quota.service.impl|org.apache.lens.server.quota.QuotaServiceImpl|Implementation class for quota service|
+|59|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
*--+--+---+--+
-|56|lens.server.quota.ws.resource.impl|org.apache.lens.server.quota.QuotaResource|Implementation class for Quota Resource|
+|60|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
*--+--+---+--+
-|57|lens.server.recover.onrestart|true|If the flag is enabled, all the services will be started from last saved state, if disabled all the services will start afresh|
+|61|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
*--+--+---+--+
-|58|lens.server.restart.enabled|true|If flag is enabled, all the services will be persisted to persistent location passed.|
+|62|lens.server.scheduling.queue.poll.interval.millisec|2000|The interval at which submission thread will poll scheduling queue to fetch the next query for submission. If value is less than equal to 0, then it would mean that thread will continuosly poll without sleeping. The interval has to be given in milliseconds.|
*--+--+---+--+
-|59|lens.server.scheduler.service.impl|org.apache.lens.server.scheduler.QuerySchedulerServiceImpl|Implementation class for query scheduler service|
+|63|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
*--+--+---+--+
-|60|lens.server.scheduler.ws.resource.impl|org.apache.lens.server.scheduler.ScheduleResource|Implementation class for query scheduler resource|
+|64|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
*--+--+---+--+
-|61|lens.server.serverMode.ws.filter.impl|org.apache.lens.server.ServerModeFilter|Implementation class for ServerMode Filter|
+|65|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|62|lens.server.service.provider.factory|org.apache.lens.server.ServiceProviderFactoryImpl|Service provider factory implementation class. This parameter is used to lookup the factory implementation class name that would provide an instance of ServiceProvider. Users should instantiate the class to obtain its instance. Example -- Class spfClass = conf.getClass("lens.server.service.provider.factory", null, ServiceProviderFactory.class); ServiceProviderFactory spf = spfClass.newInstance(); ServiceProvider serviceProvider = spf.getServiceProvider(); -- This is not supposed to be overridden by users.|
+|66|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
*--+--+---+--+
-|63|lens.server.servicenames|session,query,metastore,scheduler,quota|These services would be started in the specified order when lens-server starts up|
+|67|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
*--+--+---+--+
-|64|lens.server.session.expiry.service.interval.secs|3600|Interval at which lens session expiry service runs|
+|68|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
*--+--+---+--+
-|65|lens.server.session.service.impl|org.apache.lens.server.session.HiveSessionService|Implementation class for session service|
+|69|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
*--+--+---+--+
-|66|lens.server.session.timeout.seconds|86400|Lens session timeout in seconds.If there is no activity on the session for this period then the session will be closed.Default timeout is one day.|
+|70|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
*--+--+---+--+
-|67|lens.server.session.ws.resource.impl|org.apache.lens.server.session.SessionResource|Implementation class for Session Resource|
+|71|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
*--+--+---+--+
-|68|lens.server.snapshot.interval|300000|Snapshot interval time in miliseconds for saving lens server state.|
+|72|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
*--+--+---+--+
-|69|lens.server.state.persist.out.stream.buffer.size|1048576|Output Stream Buffer Size used in writing lens server state to file system. Size is in bytes.|
+|73|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
*--+--+---+--+
-|70|lens.server.statistics.db|lensstats|Database to which statistics tables are created and partitions are added.|
+|74|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
*--+--+---+--+
-|71|lens.server.statistics.log.rollover.interval|3600000|Default rate which log statistics store scans for rollups in milliseconds.|
+|75|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
*--+--+---+--+
-|72|lens.server.statistics.store.class|org.apache.lens.server.stats.store.log.LogStatisticsStore|Default implementation of class used to persist Lens Statistics.|
+|76|lens.server.total.query.cost.ceiling.per.user|-1.0|A query submitted by user will be launched only if total query cost of all current launched queries of user is less than or equal to total query cost ceiling defined by this property. This configuration value is only useful when TotalQueryCostCeilingConstraint is enabled by using org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory as one of the factories in lens.server.query.constraint.factories property. Default is -1.0 which means that there is no limit on the total query cost of launched queries submitted by a user.|
*--+--+---+--+
-|73|lens.server.statistics.warehouse.dir|file:///tmp/lens/statistics/warehouse|Default top level location where stats are moved by the log statistics store.|
+|77|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
*--+--+---+--+
-|74|lens.server.ui.base.uri|http://0.0.0.0:19999/|The base url for the Lens UI Server|
+|78|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
*--+--+---+--+
-|75|lens.server.ui.enable|true|Bringing up the ui server is optional. By default it brings up UI server.|
+|79|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
*--+--+---+--+
-|76|lens.server.ui.enable.caching|true|Set this to false to disable static file caching in the UI server|
+|80|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
*--+--+---+--+
-|77|lens.server.ui.static.dir|webapp/lens-server/static|The base directory to server UI static files from|
+|81|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
*--+--+---+--+
-|78|lens.server.user.resolver.custom.class|full.package.name.Classname|Required for CUSTOM user resolver. In case the provided implementations are not sufficient for user config resolver, a custom classname can be provided. Class should extend org.apache.lens.server.user.UserConfigLoader|
+|82|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
*--+--+---+--+
-|79|lens.server.user.resolver.db.keys|lens.session.cluster.user,mapred.job.queue.name|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loaders, the conf keys that will be loaded from database.|
+|83|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
*--+--+---+--+
-|80|lens.server.user.resolver.db.query|select clusteruser,queue from user_config_table where username=?|Required for DATABASE and LDAP_BACKED_DATABASE user resolvers. For database based user config loader, this query will be run with single argument = logged in user and the result columns will be assigned to lens.server.user.resolver.db.keys in order. For ldap backed database resolver, the argument to this query will be the intermediate values obtained from ldap.|
+|84|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
*--+--+---+--+
-|81|lens.server.user.resolver.fixed.value| |Required for FIXED user resolver. when lens.server.user.resolver.type=FIXED, This will be the value cluster user will resolve to.|
+|85|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
*--+--+---+--+
-|82|lens.server.user.resolver.ldap.bind.dn| |Required for LDAP_BACKED_DATABASE user resolvers. ldap dn for admin binding example: CN=company-it-admin,ou=service-account,ou=company-service-account,dc=dc1,dc=com...|
+|86|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
*--+--+---+--+
-|83|lens.server.user.resolver.ldap.bind.password| |Required for LDAP_BACKED_DATABASE user resolvers. ldap password for admin binding above|
+|87|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
*--+--+---+--+
-|84|lens.server.user.resolver.ldap.fields|department|Required for LDAP_BACKED_DATABASE user resolvers. list of fields to be obtained from ldap. These will be cached by the intermediate db.|
+|88|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
*--+--+---+--+
-|85|lens.server.user.resolver.ldap.intermediate.db.delete.sql|delete from user_department where username=?|Required for LDAP_BACKED_DATABASE user resolvers. query to delete intermediate values from database backing ldap as cache. one argument: logged in user.|
+|89|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
*--+--+---+--+
-|86|lens.server.user.resolver.ldap.intermediate.db.insert.sql|insert into user_department (username, department, expiry) values (?, ?, ?)|Required for LDAP_BACKED_DATABASE user resolvers. query to insert intermediate values from database backing ldap as cache. arguments: first logged in user, then all intermediate values, then current time + expiration time|
+|90|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
*--+--+---+--+
-|87|lens.server.user.resolver.ldap.intermediate.db.query|select department from user_department where username=? and expiry>?|Required for LDAP_BACKED_DATABASE user resolvers. query to obtain intermediate values from database backing ldap as cache. two arguments: logged in user and current time.|
+|91|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
*--+--+---+--+
-|88|lens.server.user.resolver.ldap.search.base| |Required for LDAP_BACKED_DATABASE user resolvers. for searching intermediate values for a user, the search keys. example: cn=users,dc=dc1,dc=dc2...|
+|92|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
*--+--+---+--+
-|89|lens.server.user.resolver.ldap.search.filter|(&(objectClass=user)(sAMAccountName=%s))|Required for LDAP_BACKED_DATABASE user resolvers. filter pattern for ldap search|
+|93|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
*--+--+---+--+
-|90|lens.server.user.resolver.ldap.url| |Required for LDAP_BACKED_DATABASE user resolvers. ldap url to connect to.|
+|94|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
*--+--+---+--+
-|91|lens.server.user.resolver.propertybased.filename|/path/to/propertyfile|Required for PROPERTYBASED user resolver. when lens.server.user.resolver.type is PROPERTYBASED, then this file will be read and parsed to determine cluster user. Each line should contain username followed by DOT followed by property full name followed by equal-to sign and followed by value. example schema of the file is: user1.lens.server.cluster.user=clusteruser1 user1.mapred.job.queue.name=queue1 *.lens.server.cluster.user=defaultclusteruser *.mapred.job.queue.name=default|
+|95|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
*--+--+---+--+
-|92|lens.server.user.resolver.type|FIXED|Type of user config resolver. allowed values are FIXED, PROPERTYBASED, DATABASE, LDAP_BACKED_DATABASE, CUSTOM.|
+|96|lens.server.waiting.queries.selection.policy.factories|org.apache.lens.server.query.collect.UserSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
*--+--+---+--+
-|93|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
+|97|lens.server.ws.featurenames|multipart|These JAX-RS Feature(s) would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|94|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
+|98|lens.server.ws.filternames|authentication,consistentState,serverMode|These JAX-RS filters would be started in the specified order when lens-server starts up|
*--+--+---+--+
-|95|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
+|99|lens.server.ws.listenernames|appevent|These listeners would be called in the specified order when lens-server starts up|
*--+--+---+--+
-|96|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up|
+|100|lens.server.ws.resourcenames|session,metastore,query,quota,scheduler,index,log|These JAX-RS resources would be started in the specified order when lens-server starts up|
*--+--+---+--+
The configuration parameters and their default values
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/src/site/apt/admin/hivedriver-config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/hivedriver-config.apt b/src/site/apt/admin/hivedriver-config.apt
index 80a3028..b1a25c3 100644
--- a/src/site/apt/admin/hivedriver-config.apt
+++ b/src/site/apt/admin/hivedriver-config.apt
@@ -68,4 +68,8 @@ Hive driver configuration
*--+--+---+--+
|15|lens.driver.hive.query.hook.class|org.apache.lens.server.api.driver.NoOpDriverQueryHook|The query hook class for hive driver. By default hook is No op. To add a hook, you should look at the default implementation and from there it'll be easy to derive what value can be added through a new hook|
*--+--+---+--+
+|16|lens.driver.hive.query.launching.constraint.factories| |Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.|
+*--+--+---+--+
+|17|lens.driver.hive.waiting.queries.selection.policy.factories| |Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
+*--+--+---+--+
The configuration parameters and their default values
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/c879f991/src/site/apt/admin/jdbcdriver-config.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/admin/jdbcdriver-config.apt b/src/site/apt/admin/jdbcdriver-config.apt
index b3d25f6..7b42309 100644
--- a/src/site/apt/admin/jdbcdriver-config.apt
+++ b/src/site/apt/admin/jdbcdriver-config.apt
@@ -24,56 +24,62 @@ Jdbc driver configuration
*--+--+---+--+
|<<No.>>|<<Property Name>>|<<Default Value>>|<<Description>>|
*--+--+---+--+
-|1|lens.cube.query.driver.supported.storages| |List of comma separated storage names that supported by a driver. If no value is specified, all storages are valid|
+|1|driver.max.concurrent.launched.queries|15|Maximum queries which can be launched simultaneously on this driver. This should be equal to lens.driver.jdbc.pool.max.size. This configuration value is only useful when MaxConcurrentDriverQueriesConstraint is enabled by using org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory as one of the factories in lens.driver.jdbc.query.constraint.factories property.|
*--+--+---+--+
-|2|lens.cube.query.partition.where.clause.format|yyyy-MM-dd HH:mm:ss|The simple date format of how the queried partition should be put in where clause. If nothing is specified, it will use the format from org.apache.lens.cube.metadata.UpdatePeriod for each type of partition|
+|2|lens.cube.query.driver.supported.storages| |List of comma separated storage names that supported by a driver. If no value is specified, all storages are valid|
*--+--+---+--+
-|3|lens.cube.query.replace.timedim|false|Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.|
+|3|lens.cube.query.partition.where.clause.format|yyyy-MM-dd HH:mm:ss|The simple date format of how the queried partition should be put in where clause. If nothing is specified, it will use the format from org.apache.lens.cube.metadata.UpdatePeriod for each type of partition|
*--+--+---+--+
-|4|lens.cube.query.time.range.writer.class|org.apache.lens.cube.parse.BetweenTimeRangeWriter|The timerange writer class which specifies how the resolved partitions in timeranges should be written in final query. Available writers are org.apache.lens.cube.parse.ORTimeRangeWriter and org.apache.lens.cube.parse.BetweenTimeRangeWriter|
+|4|lens.cube.query.replace.timedim|false|Tells whether timedim attribute queried in the time range should be replaced with its corresponding partition column name.|
*--+--+---+--+
-|5|lens.driver.jdbc.connection.provider| |A contract for obtaining JDBC connections|
+|5|lens.cube.query.time.range.writer.class|org.apache.lens.cube.parse.BetweenTimeRangeWriter|The timerange writer class which specifies how the resolved partitions in timeranges should be written in final query. Available writers are org.apache.lens.cube.parse.ORTimeRangeWriter and org.apache.lens.cube.parse.BetweenTimeRangeWriter|
*--+--+---+--+
-|6|lens.driver.jdbc.db.password| |The database user's password|
+|6|lens.driver.jdbc.connection.provider| |A contract for obtaining JDBC connections|
*--+--+---+--+
-|7|lens.driver.jdbc.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname|
+|7|lens.driver.jdbc.db.password| |The database user's password|
*--+--+---+--+
-|8|lens.driver.jdbc.db.user| |The database user on whose behalf the connection is being made|
+|8|lens.driver.jdbc.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname|
*--+--+---+--+
-|9|lens.driver.jdbc.driver.class|com.mysql.jdbc.Driver|Type of JDBC driver used to connect backend database|
+|9|lens.driver.jdbc.db.user| |The database user on whose behalf the connection is being made|
*--+--+---+--+
-|10|lens.driver.jdbc.enable.resultset.streaming.retrieval|false|Flag to enable row by row retrieval of result set from the database server. This is used to enable streaming result sets for MySQL. This is set to false by default.|
+|10|lens.driver.jdbc.driver.class|com.mysql.jdbc.Driver|Type of JDBC driver used to connect backend database|
*--+--+---+--+
-|11|lens.driver.jdbc.estimate.db.password| |The database user's password, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.password would be used. Override this property to tune estimate connection pool|
+|11|lens.driver.jdbc.enable.resultset.streaming.retrieval|false|Flag to enable row by row retrieval of result set from the database server. This is used to enable streaming result sets for MySQL. This is set to false by default.|
*--+--+---+--+
-|12|lens.driver.jdbc.estimate.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.|
+|12|lens.driver.jdbc.estimate.db.password| |The database user's password, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.password would be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|13|lens.driver.jdbc.estimate.db.user| |The database user on whose behalf the connection is being made, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.user would be used. Override this property to tune estimate connection pool|
+|13|lens.driver.jdbc.estimate.db.uri| |JDBC connection URL in the format jdbc:dbms://host:port/dbname for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.uri will be used.|
*--+--+---+--+
-|14|lens.driver.jdbc.estimate.driver.class| |Type of JDBC driver used to connect backend database for estimate queries. If This property is not specified, value for lens.driver.jdbc.driver.class will be used. Override this property to tune estimate connection pool|
+|14|lens.driver.jdbc.estimate.db.user| |The database user on whose behalf the connection is being made, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.db.user would be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|15|lens.driver.jdbc.estimate.get.connection.timeout| |Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout would be used. Override this property to tune estimate connection pool.|
+|15|lens.driver.jdbc.estimate.driver.class| |Type of JDBC driver used to connect backend database for estimate queries. If This property is not specified, value for lens.driver.jdbc.driver.class will be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|16|lens.driver.jdbc.estimate.pool.idle.time| |Maximum idle time in sec before a connection is closed, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. Override this property to tune estimate connection pool.|
+|16|lens.driver.jdbc.estimate.get.connection.timeout| |Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket , for estimate queries. If this property is not specified, value for lens.driver.jdbc.get.connection.timeout would be used. Override this property to tune estimate connection pool.|
*--+--+---+--+
-|17|lens.driver.jdbc.estimate.pool.max.size| |Maximum number of concurrent connections allowed in pool, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. Override this property to tune estimate connection pool|
+|17|lens.driver.jdbc.estimate.pool.idle.time| |Maximum idle time in sec before a connection is closed, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.idle.time would be used. Override this property to tune estimate connection pool.|
*--+--+---+--+
-|18|lens.driver.jdbc.estimate.pool.max.statements| |Maximum number of prepared statements to cache per connection, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.|
+|18|lens.driver.jdbc.estimate.pool.max.size| |Maximum number of concurrent connections allowed in pool, for estimate queries. If this property is unspecified, value for lens.driver.jdbc.pool.max.size would be used. Override this property to tune estimate connection pool|
*--+--+---+--+
-|19|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database|
+|19|lens.driver.jdbc.estimate.pool.max.statements| |Maximum number of prepared statements to cache per connection, for estimate queries. If this property is not specified, value for lens.driver.jdbc.pool.max.statements would be used.|
*--+--+---+--+
-|20|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set|
+|20|lens.driver.jdbc.explain.keyword|Explain|Explain keyword used to get the query plan of underlying database|
*--+--+---+--+
-|21|lens.driver.jdbc.get.connection.timeout|10000|Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket|
+|21|lens.driver.jdbc.fetch.size|1000|Fetch size for JDBC result set|
*--+--+---+--+
-|22|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed|
+|22|lens.driver.jdbc.get.connection.timeout|10000|Response timeout in milliseconds of any JDBC call invoking data transmission over a connection socket|
*--+--+---+--+
-|23|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool|
+|23|lens.driver.jdbc.pool.idle.time|600|Maximum idle time in sec before a connection is closed|
*--+--+---+--+
-|24|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection|
+|24|lens.driver.jdbc.pool.max.size|15|Maximum number of concurrent connections allowed in pool|
*--+--+---+--+
-|25|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries|
+|25|lens.driver.jdbc.pool.max.statements|20|Maximum number of prepared statements to cache per connection|
*--+--+---+--+
-|26|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.|
+|26|lens.driver.jdbc.query.launching.constraint.factories|org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory|Factories used to instantiate constraints enforced on queries by driver. A query will be launched only if all constraints pass. Every Factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint.|
+*--+--+---+--+
+|27|lens.driver.jdbc.query.rewriter|org.apache.lens.driver.jdbc.ColumnarSQLRewriter|Rewriting the HQL to optimized sql queries|
+*--+--+---+--+
+|28|lens.driver.jdbc.validate.through.prepare|true|Flag to enable query syntactic and semantic validation using prepared statement.|
+*--+--+---+--+
+|29|lens.driver.jdbc.waiting.queries.selection.policy.factories|org.apache.lens.server.api.query.collect.DriverSpecificWaitingQueriesSelectionPolicyFactory|Factories used to instantiate driver specific waiting queries selection policies. Every factory should be an implementation of org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory and create an implementation of org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy.|
*--+--+---+--+
The configuration parameters and their default values
[02/50] [abbrv] incubator-lens git commit: LENS-569 : Fix setting
parameters upon session restoration
Posted by sh...@apache.org.
LENS-569 : Fix setting parameters upon session restoration
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/8ebe7f41
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/8ebe7f41
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/8ebe7f41
Branch: refs/heads/LENS-581
Commit: 8ebe7f4186a1cef2f4e60ae9a83aa2603fd01582
Parents: aa1c7d4
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Tue Jul 7 10:14:16 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Jul 7 10:14:16 2015 +0530
----------------------------------------------------------------------
.../lens/server/session/HiveSessionService.java | 63 +++++++++------
.../lens/server/session/LensSessionImpl.java | 7 +-
.../apache/lens/server/TestServerRestart.java | 84 ++++++++++----------
3 files changed, 82 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8ebe7f41/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
index 36adee0..3a8fc95 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/HiveSessionService.java
@@ -56,6 +56,7 @@ import org.apache.hive.service.cli.CLIService;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.OperationHandle;
+import com.google.common.collect.Maps;
import lombok.AccessLevel;
import lombok.Getter;
@@ -205,7 +206,7 @@ public class HiveSessionService extends LensService implements SessionService {
*/
@Override
public LensSessionHandle openSession(String username, String password, String database,
- Map<String, String> configuration)
+ Map<String, String> configuration)
throws LensException {
LensSessionHandle sessionid = super.openSession(username, password, configuration);
log.info("Opened session " + sessionid + " for user " + username);
@@ -294,42 +295,56 @@ public class HiveSessionService extends LensService implements SessionService {
public void setSessionParameter(LensSessionHandle sessionid, String key, String value) {
setSessionParameter(sessionid, key, value, true);
}
-
/**
* Sets the session parameter.
*
* @param sessionid the sessionid
- * @param key the key
- * @param value the value
+ * @param config map of string-string. each entry represents key and the value to be set for that key
* @param addToSession the add to session
*/
- protected void setSessionParameter(LensSessionHandle sessionid, String key, String value, boolean addToSession) {
- log.info("Request to Set param key:" + key + " value:" + value);
- String command = "set" + " " + key + "= " + value;
+
+ protected void setSessionParameters(LensSessionHandle sessionid, Map<String, String> config, boolean addToSession) {
+ log.info("Request to Set params:" + config);
try {
acquire(sessionid);
// set in session conf
- String var;
- if (key.indexOf(SetProcessor.HIVECONF_PREFIX) == 0) {
- var = key.substring(SetProcessor.HIVECONF_PREFIX.length());
- } else {
- var = key;
+ for(Map.Entry<String, String> entry: config.entrySet()) {
+ String var = entry.getKey();
+ if (var.indexOf(SetProcessor.HIVECONF_PREFIX) == 0) {
+ var = var.substring(SetProcessor.HIVECONF_PREFIX.length());
+ }
+ getSession(sessionid).getSessionConf().set(var, entry.getValue());
+ if (addToSession) {
+ String command = "set" + " " + entry.getKey() + "= " + entry.getValue();
+ closeCliServiceOp(getCliService().executeStatement(getHiveSessionHandle(sessionid), command, null));
+ } else {
+ getSession(sessionid).getHiveConf().set(entry.getKey(), entry.getValue());
+ }
}
- getSession(sessionid).getSessionConf().set(var, value);
- // set in underlying cli session
- closeCliServiceOp(getCliService().executeStatement(getHiveSessionHandle(sessionid), command, null));
-
// add to persist
if (addToSession) {
- getSession(sessionid).setConfig(key, value);
+ getSession(sessionid).setConfig(config);
}
- log.info("Set param key:" + key + " value:" + value);
+ log.info("Set params:" + config);
} catch (HiveSQLException e) {
throw new WebApplicationException(e);
} finally {
release(sessionid);
}
}
+ /**
+ * Sets the session parameter.
+ *
+ * @param sessionid the sessionid
+ * @param key the key
+ * @param value the value
+ * @param addToSession the add to session
+ */
+ protected void setSessionParameter(LensSessionHandle sessionid, String key, String value, boolean addToSession) {
+ HashMap<String, String> config = Maps.newHashMap();
+ config.put(key, value);
+ setSessionParameters(sessionid, config, addToSession);
+ }
/*
* (non-Javadoc)
@@ -384,13 +399,11 @@ public class HiveSessionService extends LensService implements SessionService {
}
// Add config for restored sessions
- for (Map.Entry<String, String> cfg : session.getConfig().entrySet()) {
- try {
- setSessionParameter(sessionHandle, cfg.getKey(), cfg.getValue(), false);
- } catch (Exception e) {
- log.error("Error setting parameter " + cfg.getKey() + "=" + cfg.getValue()
- + " for session: " + session, e);
- }
+ try{
+ setSessionParameters(sessionHandle, session.getConfig(), false);
+ } catch (Exception e) {
+ log.error("Error setting parameters " + session.getConfig()
+ + " for session: " + session, e);
}
log.info("Restored session " + persistInfo.getSessionHandle().getPublicId());
notifyEvent(new SessionRestored(System.currentTimeMillis(), sessionHandle));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8ebe7f41/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
index 30016a2..5344fee 100644
--- a/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/session/LensSessionImpl.java
@@ -240,11 +240,10 @@ public class LensSessionImpl extends HiveSessionImpl {
/**
* Sets the config.
*
- * @param key the key
- * @param value the value
+ * @param config the config to overlay
*/
- public void setConfig(String key, String value) {
- persistInfo.getConfig().put(key, value);
+ public void setConfig(Map<String, String> config) {
+ persistInfo.getConfig().putAll(config);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/8ebe7f41/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index 537e750..a84fcd3 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -18,29 +18,17 @@
*/
package org.apache.lens.server;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
+import java.io.*;
+import java.util.*;
import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
-import org.apache.lens.api.APIResult;
+import org.apache.lens.api.*;
import org.apache.lens.api.APIResult.Status;
-import org.apache.lens.api.LensConf;
-import org.apache.lens.api.LensSessionHandle;
-import org.apache.lens.api.StringList;
-import org.apache.lens.api.query.LensQuery;
-import org.apache.lens.api.query.PersistentQueryResult;
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.api.query.*;
import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.driver.hive.TestRemoteHiveDriver;
import org.apache.lens.server.api.error.LensException;
@@ -172,7 +160,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
new LensConf(), MediaType.APPLICATION_XML_TYPE));
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
@@ -228,6 +216,10 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
public void testHiveServerRestart() throws Exception {
QueryExecutionServiceImpl queryService = (QueryExecutionServiceImpl) LensServices.get().getService("query");
LensSessionHandle lensSessionId = queryService.openSession("foo", "bar", new HashMap<String, String>());
+
+ // set params
+ setParams(lensSessionId);
+
// Create data file
createRestartTestDataFile();
@@ -240,7 +232,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
// Create a test table
LensTestUtil.createTable("test_hive_server_restart", target(), lensSessionId);
LensTestUtil.loadData("test_hive_server_restart", TestResourceFile.TEST_DATA_FILE.getValue(), target(),
- lensSessionId);
+ lensSessionId);
LOG.info("Loaded data");
LOG.info("Hive Server restart test");
@@ -258,7 +250,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
MediaType.APPLICATION_XML_TYPE));
QueryHandle handle = target.request()
.post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
@@ -287,6 +279,10 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
Thread.sleep(10000);
LOG.info("Server restarted");
+ // Check params to be set
+ verifyParamOnRestart(lensSessionId);
+
+
// Poll for first query, we should not get any exception
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
@@ -312,7 +308,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
new LensConf(), MediaType.APPLICATION_XML_TYPE));
handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
+ new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
Assert.assertNotNull(handle);
// Poll for second query, this should finish successfully
@@ -364,17 +360,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
Assert.assertNotNull(restartTestSession);
// Set a param
- WebTarget paramTarget = sessionTarget.path("params");
- FormDataMultiPart setpart = new FormDataMultiPart();
- setpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), restartTestSession,
- MediaType.APPLICATION_XML_TYPE));
- setpart
- .bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("key").build(), "lens.session.testRestartKey"));
- setpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("value").build(), "myvalue"));
- APIResult result = paramTarget.request().put(Entity.entity(setpart, MediaType.MULTIPART_FORM_DATA_TYPE),
- APIResult.class);
- Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
-
+ setParams(restartTestSession);
// Add resource
// add a resource
final WebTarget resourcetarget = target().path("session/resources");
@@ -384,25 +370,16 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
mp1.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), "file"));
mp1.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("path").build(),
"target/test-classes/lens-site.xml"));
- result = resourcetarget.path("add").request()
+ APIResult result = resourcetarget.path("add").request()
.put(Entity.entity(mp1, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
Assert.assertEquals(result.getStatus(), Status.SUCCEEDED);
// restart server
restartLensServer();
- // Check session exists in the server
-
- // Try and get params back
- StringList sessionParams = paramTarget.queryParam("sessionid", restartTestSession).queryParam("verbose", true)
- .request().get(StringList.class);
- sessionParams = paramTarget.queryParam("sessionid", restartTestSession)
- .queryParam("key", "lens.session.testRestartKey").request().get(StringList.class);
- System.out.println("Session params:" + sessionParams.getElements());
- Assert.assertEquals(sessionParams.getElements().size(), 1);
- Assert.assertTrue(sessionParams.getElements().contains("lens.session.testRestartKey=myvalue"));
-
// Check resources added again
+ verifyParamOnRestart(restartTestSession);
+
HiveSessionService sessionService = LensServices.get().getService("session");
LensSessionImpl session = sessionService.getSession(restartTestSession);
Assert.assertEquals(session.getLensSessionPersistInfo().getResources().size(), 1);
@@ -415,4 +392,25 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
}
+ private void setParams(LensSessionHandle lensSessionHandle) {
+ FormDataMultiPart setpart = new FormDataMultiPart();
+ setpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionHandle,
+ MediaType.APPLICATION_XML_TYPE));
+ setpart
+ .bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("key").build(), "lens.session.testRestartKey"));
+ setpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("value").build(), "myvalue"));
+ APIResult result = target().path("session").path("params").request()
+ .put(Entity.entity(setpart, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
+ Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+ }
+
+ private void verifyParamOnRestart(LensSessionHandle lensSessionHandle) {
+
+ StringList sessionParams = target().path("session").path("params").queryParam("sessionid", lensSessionHandle)
+ .queryParam("verbose", true).queryParam("key", "lens.session.testRestartKey").request().get(StringList.class);
+ System.out.println("Session params:" + sessionParams.getElements());
+ Assert.assertEquals(sessionParams.getElements().size(), 1);
+ Assert.assertTrue(sessionParams.getElements().contains("lens.session.testRestartKey=myvalue"));
+
+ }
}
[13/50] [abbrv] incubator-lens git commit: LENS-24 : Add support for
User Query Rewriter chain before Cube Rewrites
Posted by sh...@apache.org.
LENS-24 : Add support for User Query Rewriter chain before Cube Rewrites
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/689c9a99
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/689c9a99
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/689c9a99
Branch: refs/heads/LENS-581
Commit: 689c9a99b112b21e9ecba9a8461cf9672b88ba2f
Parents: 9cfb4fe
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Thu Jul 16 15:26:20 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Thu Jul 16 15:26:20 2015 +0530
----------------------------------------------------------------------
.../apache/lens/driver/cube/RewriteUtil.java | 393 ----------------
.../apache/lens/driver/cube/TestRewriting.java | 464 -------------------
.../lens/driver/jdbc/ColumnarSQLRewriter.java | 2 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 1 +
.../lens/server/api/LensConfConstants.java | 1 +
.../lens/server/api/error/LensException.java | 15 +-
.../server/api/query/AbstractQueryContext.java | 7 +
.../lens/server/api/query/QueryRewriter.java | 50 --
.../api/query/rewrite/Phase1Rewriter.java | 29 ++
.../server/api/query/rewrite/QueryRewriter.java | 50 ++
lens-server/pom.xml | 10 +
.../server/query/QueryExecutionServiceImpl.java | 13 +-
.../apache/lens/server/rewrite/RewriteUtil.java | 395 ++++++++++++++++
.../rewrite/UserQueryToCubeQueryRewriter.java | 69 +++
.../src/main/resources/lensserver-default.xml | 10 +
.../lens/server/query/TestQueryService.java | 317 ++++++-------
.../lens/server/rewrite/TestRewriting.java | 461 ++++++++++++++++++
.../TestUserQueryToCubeQueryRewriter.java | 82 ++++
src/site/apt/admin/config.apt | 94 ++--
19 files changed, 1347 insertions(+), 1116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java b/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
deleted file mode 100644
index 5ecee19..0000000
--- a/lens-cube/src/main/java/org/apache/lens/driver/cube/RewriteUtil.java
+++ /dev/null
@@ -1,393 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.cube;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.lens.cube.parse.CubeQueryContext;
-import org.apache.lens.cube.parse.CubeQueryRewriter;
-import org.apache.lens.cube.parse.HQLParser;
-import org.apache.lens.server.api.driver.DriverQueryPlan;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.metrics.MethodMetricsContext;
-import org.apache.lens.server.api.metrics.MethodMetricsFactory;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.HiveParser;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.log4j.Logger;
-
-import lombok.Getter;
-
-/**
- * The Class RewriteUtil.
- */
-public final class RewriteUtil {
- private RewriteUtil() {
-
- }
- public static final Logger LOG = Logger.getLogger(RewriteUtil.class);
-
- /** The cube pattern. */
- static Pattern cubePattern = Pattern.compile(".*CUBE(.*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE
- | Pattern.DOTALL);
-
- /** The matcher. */
- static Matcher matcher = null;
-
- /**
- * The Class CubeQueryInfo.
- */
- static class CubeQueryInfo {
-
- /** The start pos. */
- int startPos;
-
- /** The end pos. */
- int endPos;
-
- /** The query. */
- String query;
-
- /** The cube ast. */
- ASTNode cubeAST;
- }
-
- /**
- * Find cube positions.
- *
- * @param query the query
- * @return the list
- * @throws SemanticException the semantic exception
- * @throws ParseException the parse exception
- */
- static List<CubeQueryInfo> findCubePositions(String query, HiveConf conf)
- throws SemanticException, LensException {
-
- ASTNode ast = HQLParser.parseHQL(query, conf);
- LOG.debug("User query AST:" + ast.dump());
- List<CubeQueryInfo> cubeQueries = new ArrayList<CubeQueryInfo>();
- findCubePositions(ast, cubeQueries, query);
- for (CubeQueryInfo cqi : cubeQueries) {
- cqi.query = query.substring(cqi.startPos, cqi.endPos);
- }
- return cubeQueries;
- }
-
- /**
- * Find cube positions.
- *
- * @param ast the ast
- * @param cubeQueries the cube queries
- * @param originalQuery the original query
- * @throws SemanticException the semantic exception
- */
- private static void findCubePositions(ASTNode ast, List<CubeQueryInfo> cubeQueries, String originalQuery)
- throws SemanticException {
- int childCount = ast.getChildCount();
- if (ast.getToken() != null) {
- if (ast.getChild(0) != null) {
- LOG.debug("First child:" + ast.getChild(0) + " Type:"
- + ((ASTNode) ast.getChild(0)).getToken().getType());
- }
- if (ast.getToken().getType() == HiveParser.TOK_QUERY
- && ((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) {
- LOG.debug("Inside cube clause");
- CubeQueryInfo cqi = new CubeQueryInfo();
- cqi.cubeAST = ast;
- if (ast.getParent() != null) {
- ASTNode parent = (ASTNode) ast.getParent();
- cqi.startPos = ast.getCharPositionInLine();
- int ci = ast.getChildIndex();
- if (parent.getToken() == null || parent.getToken().getType() == HiveParser.TOK_EXPLAIN
- || parent.getToken().getType() == HiveParser.TOK_CREATETABLE) {
- // Not a sub query
- cqi.endPos = originalQuery.length();
- } else if (parent.getChildCount() > ci + 1) {
- if (parent.getToken().getType() == HiveParser.TOK_SUBQUERY) {
- // less for the next start and for close parenthesis
- cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine(), ")");
- } else if (parent.getToken().getType() == HiveParser.TOK_UNION) {
- // one less for the next start and less the size of string 'UNION ALL'
- cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine() - 1, "UNION ALL");
- } else {
- // Not expected to reach here
- LOG.warn("Unknown query pattern found with AST:" + ast.dump());
- throw new SemanticException("Unknown query pattern");
- }
- } else {
- // last child of union all query
- // one for next AST
- // and one for the close parenthesis if there are no more unionall
- // or one for the string 'UNION ALL' if there are more union all
- LOG.debug("Child of union all");
- cqi.endPos = getEndPos(originalQuery, parent.getParent().getChild(1).getCharPositionInLine(), ")",
- "UNION ALL");
- }
- }
- LOG.debug("Adding cqi " + cqi + " query:" + originalQuery.substring(cqi.startPos, cqi.endPos));
- cubeQueries.add(cqi);
- } else {
- for (int childPos = 0; childPos < childCount; ++childPos) {
- findCubePositions((ASTNode) ast.getChild(childPos), cubeQueries, originalQuery);
- }
- }
- } else {
- LOG.warn("Null AST!");
- }
- }
-
- /**
- * Gets the end pos.
- *
- * @param query the query
- * @param backTrackIndex the back track index
- * @param backTrackStr the back track str
- * @return the end pos
- */
- private static int getEndPos(String query, int backTrackIndex, String... backTrackStr) {
- if (backTrackStr != null) {
- String q = query.substring(0, backTrackIndex).toLowerCase();
- for (int i = 0; i < backTrackStr.length; i++) {
- if (q.trim().endsWith(backTrackStr[i].toLowerCase())) {
- backTrackIndex = q.lastIndexOf(backTrackStr[i].toLowerCase());
- break;
- }
- }
- }
- while (Character.isSpaceChar(query.charAt(backTrackIndex - 1))) {
- backTrackIndex--;
- }
- return backTrackIndex;
- }
-
- /**
- * Gets the rewriter.
- *
- * @param queryConf the query conf
- * @return the rewriter
- * @throws SemanticException the semantic exception
- */
- static CubeQueryRewriter getCubeRewriter(Configuration queryConf, HiveConf hconf) throws SemanticException {
- return new CubeQueryRewriter(queryConf, hconf);
- }
-
- /**
- * Replaces new lines with spaces; '&&' with AND; '||' with OR // these two can be removed once HIVE-5326 gets
- * resolved.
- *
- * @param query the query
- * @return the replaced query
- */
- static String getReplacedQuery(final String query) {
- return query.replaceAll("[\\n\\r]", " ").replaceAll("&&", " AND ").replaceAll("\\|\\|", " OR ").trim();
- }
-
- private static final String REWRITE_QUERY_GAUGE = RewriteUtil.class.getSimpleName() + "-rewriteQuery";
- private static final String TOHQL_GAUGE = RewriteUtil.class.getSimpleName() + "-rewriteQuery-toHQL";
-
- /**
- * Rewrite query.
- *
- * @param ctx the query context
- * @return the map
- * @throws LensException the lens exception
- */
- public static Map<LensDriver, DriverRewriterRunnable> rewriteQuery(AbstractQueryContext ctx) throws LensException {
- try {
- String replacedQuery = getReplacedQuery(ctx.getUserQuery());
- Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<LensDriver, DriverRewriterRunnable>();
- List<RewriteUtil.CubeQueryInfo> cubeQueries = findCubePositions(replacedQuery, ctx.getHiveConf());
-
- for (LensDriver driver : ctx.getDriverContext().getDrivers()) {
- runnables.put(driver, new DriverRewriterRunnable(driver, ctx, cubeQueries, replacedQuery));
- }
-
- return runnables;
- } catch (LensException e) {
-
- /* REST layer has to catch LensException and add additional information to it.
- To enable REST layer to call methods on LensException instance, we have to propagate
- LensException to the REST layer. Since there is a Exception e catch here,
- we need to explicitly catch and rethrow LensException to enable us for the same.*/
-
- throw e;
- } catch (Exception e) {
- throw new LensException("Rewriting failed, cause :" + e.getMessage(), e);
- }
- }
-
- public static DriverQueryPlan getRewriterPlan(DriverRewriterRunnable rewriter) {
- RewriterPlan plan = new RewriterPlan(rewriter.cubeQueryCtx);
- return plan;
- }
-
- public static class DriverRewriterRunnable implements Runnable {
- @Getter
- private final LensDriver driver;
- private final AbstractQueryContext ctx;
- private final List<CubeQueryInfo> cubeQueries;
- private final String replacedQuery;
- /** Cube query context - set after rewriting */
- private List<CubeQueryContext> cubeQueryCtx;
-
- @Getter
- /** Indicate if rewrite operation succeeded */
- private boolean succeeded;
-
- @Getter
- /** Get cause of rewrite failure if rewrite operation failed */
- private String failureCause = null;
-
- @Getter
- private LensException cause;
-
- @Getter
- /** Get eventual rewritten query */
- private String rewrittenQuery;
-
- public DriverRewriterRunnable(LensDriver driver,
- AbstractQueryContext ctx,
- List<CubeQueryInfo> cubeQueries,
- String replacedQuery) {
- this.driver = driver;
- this.ctx = ctx;
- this.cubeQueries = cubeQueries;
- this.replacedQuery = replacedQuery;
- if (cubeQueries != null) {
- cubeQueryCtx = new ArrayList<CubeQueryContext>(cubeQueries.size());
- }
- }
-
- @Override
- public void run() {
- String lowerCaseQuery = replacedQuery.toLowerCase();
- if (lowerCaseQuery.startsWith("add") || lowerCaseQuery.startsWith("set")) {
- rewrittenQuery = replacedQuery;
- return;
- }
-
- MethodMetricsContext rewriteGauge = MethodMetricsFactory
- .createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_QUERY_GAUGE);
- StringBuilder builder = new StringBuilder();
- int start = 0;
- CubeQueryRewriter rewriter = null;
- try {
- if (cubeQueries.size() > 0) {
- // avoid creating rewriter if there are no cube queries
- rewriter = getCubeRewriter(ctx.getDriverContext().getDriverConf(driver), ctx.getHiveConf());
- ctx.setOlapQuery(true);
- }
-
- // We have to rewrite each sub cube query which might be present in the original
- // user query. We are looping through all sub queries here.
- int qIndex = 1;
- for (RewriteUtil.CubeQueryInfo cqi : cubeQueries) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Rewriting cube query:" + cqi.query);
- }
-
- if (start != cqi.startPos) {
- builder.append(replacedQuery.substring(start, cqi.startPos));
- }
-
- // Parse and rewrite individual cube query
- CubeQueryContext cqc = rewriter.rewrite(cqi.query);
- MethodMetricsContext toHQLGauge = MethodMetricsFactory
- .createMethodGauge(ctx.getDriverConf(driver), true, qIndex + "-" + TOHQL_GAUGE);
- // toHQL actually generates the rewritten query
- String hqlQuery = cqc.toHQL();
- cubeQueryCtx.add(cqc);
- toHQLGauge.markSuccess();
- qIndex++;
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Rewritten query:" + hqlQuery);
- }
-
- builder.append(hqlQuery);
- start = cqi.endPos;
- }
-
- builder.append(replacedQuery.substring(start));
-
- rewrittenQuery = builder.toString();
- // set rewriter plan
- ctx.getDriverContext().setDriverRewriterPlan(driver, getRewriterPlan(this));
- succeeded = true;
- ctx.setDriverQuery(driver, rewrittenQuery);
- LOG.info("Final rewritten query for driver:" + driver + " is: " + rewrittenQuery);
-
- } catch (final LensException e) {
-
- this.cause = e;
- captureExceptionInformation(e);
- } catch (Exception e) {
-
- // we are catching all exceptions sothat other drivers can be picked in case of driver bugs
- captureExceptionInformation(e);
- } finally {
- if (rewriter != null) {
- rewriter.clear();
- }
- rewriteGauge.markSuccess();
- }
- }
-
- @Override
- public String toString() {
- return "Rewrite runnable for " + driver;
- }
-
- private void captureExceptionInformation(final Exception e) {
-
- LOG.warn("Driver : " + driver + " Skipped for the query rewriting due to ", e);
- ctx.setDriverRewriteError(driver, e);
- failureCause = new StringBuilder(" Driver :")
- .append(driver.getClass().getName())
- .append(" Cause :" + e.getLocalizedMessage())
- .toString();
- }
- }
-
- /**
- * Checks if is cube query.
- *
- * @param query the query
- * @return true, if is cube query
- */
- public static boolean isCubeQuery(String query) {
- if (matcher == null) {
- matcher = cubePattern.matcher(query);
- } else {
- matcher.reset(query);
- }
- return matcher.matches();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
----------------------------------------------------------------------
diff --git a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java b/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
deleted file mode 100644
index 2dac223..0000000
--- a/lens-cube/src/test/java/org/apache/lens/driver/cube/TestRewriting.java
+++ /dev/null
@@ -1,464 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.driver.cube;
-
-import static org.mockito.Matchers.any;
-
-import java.util.*;
-
-import org.apache.lens.api.LensConf;
-import org.apache.lens.cube.parse.CubeQueryContext;
-import org.apache.lens.cube.parse.CubeQueryRewriter;
-import org.apache.lens.cube.parse.HQLParser;
-import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.driver.MockDriver;
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.metrics.LensMetricsRegistry;
-import org.apache.lens.server.api.query.QueryContext;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.Context;
-import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.HiveParser;
-import org.apache.hadoop.hive.ql.parse.ParseException;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
-
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.testng.PowerMockObjectFactory;
-import org.testng.Assert;
-import org.testng.IObjectFactory;
-import org.testng.annotations.ObjectFactory;
-import org.testng.annotations.Test;
-
-import com.codahale.metrics.MetricRegistry;
-
-/**
- * The Class TestRewriting.
- */
-@PrepareForTest(RewriteUtil.class)
-@PowerMockIgnore({"org.apache.log4j.*", "javax.management.*", "javax.xml.*",
- "com.sun.org.apache.xerces.internal.jaxp.*", "ch.qos.logback.*", "org.slf4j.*", "org.w3c.dom*"})
-public class TestRewriting {
- /**
- * We need a special {@link IObjectFactory}.
- *
- * @return {@link PowerMockObjectFactory}.
- */
- @ObjectFactory
- public IObjectFactory getObjectFactory() {
- return new PowerMockObjectFactory();
- }
- private HiveConf hconf = new HiveConf();
-
- static int i = 0;
- // number of successful queries through mock rewriter
- // we use this number to mock failures after successful queries
- // change the number, if more tests for success needs to be added
- static final int NUM_SUCCESS = 36;
-
- private CubeQueryRewriter getMockedRewriter() throws SemanticException, ParseException, LensException {
- CubeQueryRewriter mockwriter = Mockito.mock(CubeQueryRewriter.class);
- Mockito.when(mockwriter.rewrite(any(String.class))).thenAnswer(new Answer<CubeQueryContext>() {
- @Override
- public CubeQueryContext answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- i++;
- // return query for first NUM_SUCCESS calls and fail later
- if (i <= NUM_SUCCESS) {
- return getMockedCubeContext((String) args[0]);
- } else {
- throw new RuntimeException("Mock fail");
- }
- }
- });
- Mockito.when(mockwriter.rewrite(any(ASTNode.class))).thenAnswer(new Answer<CubeQueryContext>() {
- @Override
- public CubeQueryContext answer(InvocationOnMock invocation) throws Throwable {
- Object[] args = invocation.getArguments();
- return getMockedCubeContext((ASTNode) args[0]);
- }
- });
- return mockwriter;
- }
-
- /**
- * Gets the mocked cube context.
- *
- * @param query the query
- * @return the mocked cube context
- * @throws SemanticException the semantic exception
- * @throws ParseException the parse exception
- */
- private CubeQueryContext getMockedCubeContext(String query)
- throws SemanticException, ParseException, LensException {
- CubeQueryContext context = Mockito.mock(CubeQueryContext.class);
- Mockito.when(context.toHQL()).thenReturn(query.substring(4));
- Mockito.when(context.toAST(any(Context.class))).thenReturn(HQLParser.parseHQL(query.substring(4), hconf));
- return context;
- }
-
- /**
- * Gets the mocked cube context.
- *
- * @param ast the ast
- * @return the mocked cube context
- * @throws SemanticException the semantic exception
- * @throws ParseException the parse exception
- */
- private CubeQueryContext getMockedCubeContext(ASTNode ast) throws SemanticException, ParseException {
- CubeQueryContext context = Mockito.mock(CubeQueryContext.class);
- if (ast.getToken().getType() == HiveParser.TOK_QUERY) {
- if (((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) {
- // remove cube child from AST
- for (int i = 0; i < ast.getChildCount() - 1; i++) {
- ast.setChild(i, ast.getChild(i + 1));
- }
- ast.deleteChild(ast.getChildCount() - 1);
- }
- }
- StringBuilder builder = new StringBuilder();
- HQLParser.toInfixString(ast, builder);
- Mockito.when(context.toHQL()).thenReturn(builder.toString());
- Mockito.when(context.toAST(any(Context.class))).thenReturn(ast);
- return context;
- }
-
- private void runRewrites(Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnableMap) {
- for (LensDriver driver: runnableMap.keySet()) {
- RewriteUtil.DriverRewriterRunnable r = runnableMap.get(driver);
- r.run();
- Assert.assertTrue(r.getDriver() == driver);
-
- // Failure cause should be set only when rewrite fails
- if (r.isSucceeded()) {
- Assert.assertNull(r.getFailureCause(), driver
- + " succeeded but failure cause is set to " + r.getFailureCause());
- Assert.assertNotNull(r.getRewrittenQuery(), driver + " succeeded but rewritten query is not set");
- } else {
- Assert.assertNotNull(r.getFailureCause(), driver + " failed but failure cause is not set");
- Assert.assertNull(r.getRewrittenQuery(),
- driver + " failed but rewritten query is set to " + r.getRewrittenQuery());
- }
-
- }
- }
-
- /**
- * Test cube query.
- *
- * @throws ParseException the parse exception
- * @throws SemanticException the semantic exception
- * @throws LensException the lens exception
- */
- @Test
- public void testCubeQuery() throws ParseException, SemanticException, LensException {
- List<LensDriver> drivers = new ArrayList<LensDriver>();
- MockDriver driver = new MockDriver();
- LensConf lensConf = new LensConf();
- Configuration conf = new Configuration();
- driver.configure(conf);
- drivers.add(driver);
-
- CubeQueryRewriter mockWriter = getMockedRewriter();
- PowerMockito.stub(PowerMockito.method(RewriteUtil.class, "getCubeRewriter")).toReturn(mockWriter);
- String q1 = "select name from table";
- Assert.assertFalse(RewriteUtil.isCubeQuery(q1));
- List<RewriteUtil.CubeQueryInfo> cubeQueries = RewriteUtil.findCubePositions(q1, hconf);
- Assert.assertEquals(cubeQueries.size(), 0);
- QueryContext ctx = new QueryContext(q1, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName());
- driver.configure(conf);
- String q2 = "cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- MetricRegistry reg = LensMetricsRegistry.getStaticRegistry();
-
- Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestRewriting-MockDriver-RewriteUtil-rewriteQuery",
- "lens.MethodMetricGauge.TestRewriting-MockDriver-1-RewriteUtil-rewriteQuery-toHQL")));
- conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
-
- q2 = "insert overwrite directory '/tmp/rewrite' cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "insert overwrite local directory '/tmp/rewrite' cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "insert overwrite local directory '/tmp/example-output' cube select id,name from dim_table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select id,name from dim_table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "explain cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from (cube select name from table) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "insert overwrite directory '/tmp/rewrite' select * from (cube select name from table) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from (cube select name from table)a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from ( cube select name from table ) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from ( cube select name from table where"
- + " (name = 'ABC'||name = 'XYZ')&&(key=100) ) a";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(RewriteUtil.getReplacedQuery(q2), hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from"
- + " table where (name = 'ABC' OR name = 'XYZ') AND (key=100)");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, TestRewriting.class.getSimpleName() + "-multiple");
- q2 = "select * from (cube select name from table) a join (cube select" + " name2 from table2) b";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 2);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- reg = LensMetricsRegistry.getStaticRegistry();
- Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
- "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-1-RewriteUtil-rewriteQuery-toHQL",
- "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-2-RewriteUtil-rewriteQuery-toHQL",
- "lens.MethodMetricGauge.TestRewriting-multiple-MockDriver-RewriteUtil-rewriteQuery")));
- conf.unset(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY);
-
- q2 = "select * from (cube select name from table) a full outer join"
- + " (cube select name2 from table2) b on a.name=b.name2";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 2);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from (cube select name from table) a join (select name2 from table2) b";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from (cube select name from table union all cube select name2 from table2) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 2);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
-
- q2 = "insert overwrite directory '/tmp/rewrite' "
- + "select * from (cube select name from table union all cube select name2 from table2) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 2);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
-
- q2 = "select u.* from (select name from table union all cube select name2 from table2) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select u.* from (select name from table union all cube select name2 from table2)u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name2 from table2");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from (cube select name from table union all cube select name2"
- + " from table2 union all cube select name3 from table3) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 3);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
- Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3");
-
- q2 = "select * from ( cube select name from table union all cube"
- + " select name2 from table2 union all cube select name3 from table3 ) u";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 3);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
- Assert.assertEquals(cubeQueries.get(2).query, "cube select name3 from table3");
-
- q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- Assert.assertEquals(cubeQueries.size(), 2);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
-
- q2 = "select * from (cube select name from table union all cube select" + " name2 from table2) u group by u.name";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 2);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
-
- q2 = "create table temp1 as cube select name from table";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
-
- q2 = "create table temp1 as select * from (cube select name from table union all cube select"
- + " name2 from table2) u group by u.name";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 2);
- Assert.assertEquals(cubeQueries.get(0).query, "cube select name from table");
- Assert.assertEquals(cubeQueries.get(1).query, "cube select name2 from table2");
-
- q2 = "create table temp1 as cube select name from table where"
- + " time_range_in('dt', '2014-06-24-23', '2014-06-25-00')";
- Assert.assertTrue(RewriteUtil.isCubeQuery(q2));
- cubeQueries = RewriteUtil.findCubePositions(q2, hconf);
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- runRewrites(RewriteUtil.rewriteQuery(ctx));
- Assert.assertEquals(cubeQueries.size(), 1);
- Assert.assertEquals(cubeQueries.get(0).query,
- "cube select name from table where time_range_in('dt', '2014-06-24-23', '2014-06-25-00')");
-
- // failing query for second driver
- MockDriver driver2 = new MockDriver();
- driver2.configure(conf);
- drivers.add(driver2);
-
- Assert.assertEquals(drivers.size(), 2);
- Assert.assertTrue(drivers.contains(driver) && drivers.contains(driver2));
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- Map<LensDriver, RewriteUtil.DriverRewriterRunnable> dQueries = RewriteUtil.rewriteQuery(ctx);
- Assert.assertEquals(dQueries.size(), 2);
-
- // Check both driver runnables are present
- Assert.assertNotNull(dQueries.get(driver), driver + " runnable not found");
- Assert.assertEquals(dQueries.get(driver).getDriver(), driver);
-
- Assert.assertNotNull(dQueries.get(driver2), driver2 + " runnable not found");
- Assert.assertEquals(dQueries.get(driver2).getDriver(), driver2);
-
- runRewrites(dQueries);
-
- // We have to verify that query failed for the second driver
- // First driver passes
- Iterator<LensDriver> itr = dQueries.keySet().iterator();
- itr.next();
- LensDriver failedDriver = itr.next();
- Assert.assertFalse(dQueries.get(failedDriver).isSucceeded(), failedDriver + " rewrite should have failed");
- Assert.assertNull(dQueries.get(failedDriver).getRewrittenQuery(),
- failedDriver + " rewritten query should not be set");
- // running again will fail on both drivers
- ctx = new QueryContext(q2, null, lensConf, conf, drivers);
- Map<LensDriver, RewriteUtil.DriverRewriterRunnable> runnables = RewriteUtil.rewriteQuery(ctx);
- runRewrites(runnables);
-
- Assert.assertFalse(runnables.get(driver).isSucceeded());
- Assert.assertNotNull(runnables.get(driver).getFailureCause());
- Assert.assertNull(runnables.get(driver).getRewrittenQuery());
- Assert.assertNotNull(ctx.getDriverRewriteError(driver));
-
- Assert.assertFalse(runnables.get(driver2).isSucceeded());
- Assert.assertNotNull(runnables.get(driver2).getFailureCause());
- Assert.assertNull(runnables.get(driver2).getRewrittenQuery());
- Assert.assertNotNull(ctx.getDriverRewriteError(driver2));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
index 0e0beb8..18f84cd 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/ColumnarSQLRewriter.java
@@ -27,7 +27,7 @@ import org.apache.lens.cube.parse.CubeSemanticAnalyzer;
import org.apache.lens.cube.parse.HQLParser;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.QueryRewriter;
+import org.apache.lens.server.api.query.rewrite.QueryRewriter;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.mutable.MutableInt;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 0f2b740..e048a78 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -44,6 +44,7 @@ import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.query.*;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.rewrite.QueryRewriter;
import org.apache.lens.server.api.user.UserConfigLoader;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index ad27d78..eab42d0 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -892,4 +892,5 @@ public final class LensConfConstants {
public static final String ESTIMATE_POOL_KEEP_ALIVE_MILLIS = SERVER_PFX + "estimate.pool.keepalive.millis";
public static final int DEFAULT_ESTIMATE_POOL_KEEP_ALIVE_MILLIS = 60000; // 1 minute
+ public static final String QUERY_PHASE1_REWRITERS = SERVER_PFX + "query.phase1.rewriters";
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java b/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
index 49b118d..123b6ee 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/error/LensException.java
@@ -124,7 +124,7 @@ public class LensException extends Exception {
* @see Exception#Exception(Throwable)
*/
public LensException(final String errorMsg, final int errorcode, final Throwable cause,
- @NonNull final Object... errorMsgFormattingArgs) {
+ @NonNull final Object... errorMsgFormattingArgs) {
super(errorMsg, cause);
checkArgument(errorcode > 0);
@@ -134,7 +134,7 @@ public class LensException extends Exception {
}
public final void buildLensErrorResponse(final ErrorCollection errorCollection,
- final String apiVersion, final String id) {
+ final String apiVersion, final String id) {
final LensError lensError = errorCollection.getLensError(errorCode);
final LensErrorTO lensErrorTO = buildLensErrorTO(errorCollection, lensError);
@@ -148,7 +148,7 @@ public class LensException extends Exception {
}
protected LensErrorTO buildLensErrorTO(final ErrorCollection errorCollection, final String errorMsg,
- final String stackTrace) {
+ final String stackTrace) {
return LensErrorTO.composedOf(errorCode, errorMsg, stackTrace);
}
@@ -173,7 +173,7 @@ public class LensException extends Exception {
LensException e = (LensException) o;
if (errorCode == e.errorCode && isErrorMsgEqual(e)
- && Arrays.deepEquals(errorMsgFormattingArgs, e.errorMsgFormattingArgs)) {
+ && Arrays.deepEquals(errorMsgFormattingArgs, e.errorMsgFormattingArgs)) {
return true;
}
return false;
@@ -211,4 +211,11 @@ public class LensException extends Exception {
result = result * PRIME + Arrays.deepHashCode(errorMsgFormattingArgs);
return result;
}
+
+ public static LensException wrap(Exception e) {
+ if (e instanceof LensException) {
+ return (LensException) e;
+ }
+ return new LensException(e);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index 0cccaef..1334d9a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -56,6 +56,12 @@ public abstract class AbstractQueryContext implements Serializable {
*/
@Getter
protected String userQuery;
+ /**
+ * The replaced user query.
+ */
+ @Getter
+ @Setter
+ protected String phase1RewrittenQuery;
/**
* The merged Query conf.
@@ -127,6 +133,7 @@ public abstract class AbstractQueryContext implements Serializable {
}
driverContext = new DriverSelectorQueryContext(query, conf, drivers, mergeDriverConf);
userQuery = query;
+ phase1RewrittenQuery = query;
this.lensConf = qconf;
this.conf = conf;
this.submittedUser = user;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java
deleted file mode 100644
index 355a4e9..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRewriter.java
+++ /dev/null
@@ -1,50 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.server.api.error.LensException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * The Interface QueryRewriter.
- */
-public interface QueryRewriter {
-
- /**
- * Rewrite.
- * @param query the query
- * @param queryConf the query configuration
- * @param metastoreConf The metastore configuration. If rewriters requires to access metastore, this configuration
- * needs to passed
- *
- * @return the string
- * @throws LensException the lens exception
- */
- String rewrite(String query, Configuration queryConf, HiveConf metastoreConf) throws LensException;
-
- /**
- * Set conf for the rewriter
- *
- * @param rewriteConf Configuration required for rewriter init
- */
- void init(Configuration rewriteConf);
-}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/Phase1Rewriter.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/Phase1Rewriter.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/Phase1Rewriter.java
new file mode 100644
index 0000000..25b0067
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/Phase1Rewriter.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.query.rewrite;
+
+/**
+ * Phase 1 rewriting is run on the query provided by user to convert it to a valid
+ * and (probably) optimized cube query.
+ */
+public interface Phase1Rewriter extends QueryRewriter {
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/QueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/QueryRewriter.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/QueryRewriter.java
new file mode 100644
index 0000000..e5254ab
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/rewrite/QueryRewriter.java
@@ -0,0 +1,50 @@
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.rewrite;
+
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+/**
+ * The Interface QueryRewriter.
+ */
+public interface QueryRewriter {
+
+ /**
+ * Rewrite.
+ * @param query the query
+ * @param queryConf the query configuration
+ * @param metastoreConf The metastore configuration. If rewriters requires to access metastore, this configuration
+ * needs to passed
+ *
+ * @return the string
+ * @throws LensException the lens exception
+ */
+ String rewrite(String query, Configuration queryConf, HiveConf metastoreConf) throws LensException;
+
+ /**
+ * Set conf for the rewriter
+ *
+ * @param rewriteConf Configuration required for rewriter init
+ */
+ void init(Configuration rewriteConf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index c4f3bed..6ac30a5 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -157,6 +157,16 @@
</dependency>
<dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-testng</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.codahale.metrics</groupId>
<artifactId>metrics-core</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index a778a1f..3511236 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -40,7 +40,6 @@ import org.apache.lens.api.error.ErrorCollection;
import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
import org.apache.lens.api.result.LensErrorTO;
-import org.apache.lens.driver.cube.RewriteUtil;
import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.LensService;
@@ -57,6 +56,8 @@ import org.apache.lens.server.api.query.*;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
+import org.apache.lens.server.rewrite.RewriteUtil;
+import org.apache.lens.server.rewrite.UserQueryToCubeQueryRewriter;
import org.apache.lens.server.session.LensSessionImpl;
import org.apache.lens.server.stats.StatisticsService;
import org.apache.lens.server.user.UserConfigLoaderFactory;
@@ -253,6 +254,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
}
};
+ private UserQueryToCubeQueryRewriter userQueryToCubeQueryRewriter;
/**
@@ -297,7 +299,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
getEventService().addListenerForType(new QueryExecutionStatisticsGenerator(this, getEventService()),
QueryEnded.class);
getEventService().addListenerForType(
- new QueryEndNotifier(this, getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class);
+ new QueryEndNotifier(this, getCliService().getHiveConf(), this.logSegregationContext), QueryEnded.class);
log.info("Registered query result formatter");
}
@@ -489,6 +491,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
public QuerySubmitter(@NonNull final ErrorCollection errorCollection) {
this.errorCollection = errorCollection;
}
+
/*
* (non-Javadoc)
*
@@ -909,6 +912,11 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
super.init(hiveConf);
this.conf = hiveConf;
try {
+ this.userQueryToCubeQueryRewriter = new UserQueryToCubeQueryRewriter(conf);
+ } catch (LensException e) {
+ throw new IllegalStateException("Could not load phase 1 rewriters");
+ }
+ try {
initializeQueryAcceptors();
} catch (LensException e) {
throw new IllegalStateException("Could not load acceptors");
@@ -1078,6 +1086,7 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
MethodMetricsContext parallelCallGauge = MethodMetricsFactory.createMethodGauge(ctx.getConf(), false,
PARALLEL_CALL_GAUGE);
try {
+ userQueryToCubeQueryRewriter.rewrite(ctx);
// Initially we obtain individual runnables for rewrite and estimate calls
// These are mapped against the driver, so that later it becomes easy to chain them
// for each driver.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
new file mode 100644
index 0000000..447d638
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.rewrite;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.lens.cube.parse.CubeQueryContext;
+import org.apache.lens.cube.parse.CubeQueryRewriter;
+import org.apache.lens.cube.parse.HQLParser;
+import org.apache.lens.driver.cube.RewriterPlan;
+import org.apache.lens.server.api.driver.DriverQueryPlan;
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.metrics.MethodMetricsContext;
+import org.apache.lens.server.api.metrics.MethodMetricsFactory;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.log4j.Logger;
+
+import lombok.Getter;
+
+/**
+ * The Class RewriteUtil.
+ */
+public final class RewriteUtil {
+ private RewriteUtil() {
+
+ }
+ public static final Logger LOG = Logger.getLogger(RewriteUtil.class);
+
+ /** The cube pattern. */
+ static Pattern cubePattern = Pattern.compile(".*CUBE(.*)", Pattern.CASE_INSENSITIVE | Pattern.MULTILINE
+ | Pattern.DOTALL);
+
+ /** The matcher. */
+ static Matcher matcher = null;
+
+ /**
+ * The Class CubeQueryInfo.
+ */
+ static class CubeQueryInfo {
+
+ /** The start pos. */
+ int startPos;
+
+ /** The end pos. */
+ int endPos;
+
+ /** The query. */
+ String query;
+
+ /** The cube ast. */
+ ASTNode cubeAST;
+ }
+
+ /**
+ * Find cube positions.
+ *
+ * @param query the query
+ * @return the list
+ * @throws SemanticException the semantic exception
+ * @throws LensException the lensexception
+ */
+ static List<CubeQueryInfo> findCubePositions(String query, HiveConf conf)
+ throws SemanticException, LensException {
+
+ ASTNode ast = HQLParser.parseHQL(query, conf);
+ LOG.debug("User query AST:" + ast.dump());
+ List<CubeQueryInfo> cubeQueries = new ArrayList<CubeQueryInfo>();
+ findCubePositions(ast, cubeQueries, query);
+ for (CubeQueryInfo cqi : cubeQueries) {
+ cqi.query = query.substring(cqi.startPos, cqi.endPos);
+ }
+ return cubeQueries;
+ }
+
+ /**
+ * Find cube positions.
+ *
+ * @param ast the ast
+ * @param cubeQueries the cube queries
+ * @param originalQuery the original query
+ * @throws SemanticException the semantic exception
+ */
+ private static void findCubePositions(ASTNode ast, List<CubeQueryInfo> cubeQueries, String originalQuery)
+ throws SemanticException {
+ int childCount = ast.getChildCount();
+ if (ast.getToken() != null) {
+ if (ast.getChild(0) != null) {
+ LOG.debug("First child:" + ast.getChild(0) + " Type:"
+ + ((ASTNode) ast.getChild(0)).getToken().getType());
+ }
+ if (ast.getToken().getType() == HiveParser.TOK_QUERY
+ && ((ASTNode) ast.getChild(0)).getToken().getType() == HiveParser.KW_CUBE) {
+ LOG.debug("Inside cube clause");
+ CubeQueryInfo cqi = new CubeQueryInfo();
+ cqi.cubeAST = ast;
+ if (ast.getParent() != null) {
+ ASTNode parent = (ASTNode) ast.getParent();
+ cqi.startPos = ast.getCharPositionInLine();
+ int ci = ast.getChildIndex();
+ if (parent.getToken() == null || parent.getToken().getType() == HiveParser.TOK_EXPLAIN
+ || parent.getToken().getType() == HiveParser.TOK_CREATETABLE) {
+ // Not a sub query
+ cqi.endPos = originalQuery.length();
+ } else if (parent.getChildCount() > ci + 1) {
+ if (parent.getToken().getType() == HiveParser.TOK_SUBQUERY) {
+ // less for the next start and for close parenthesis
+ cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine(), ")");
+ } else if (parent.getToken().getType() == HiveParser.TOK_UNION) {
+ // one less for the next start and less the size of string 'UNION ALL'
+ cqi.endPos = getEndPos(originalQuery, parent.getChild(ci + 1).getCharPositionInLine() - 1, "UNION ALL");
+ } else {
+ // Not expected to reach here
+ LOG.warn("Unknown query pattern found with AST:" + ast.dump());
+ throw new SemanticException("Unknown query pattern");
+ }
+ } else {
+ // last child of union all query
+ // one for next AST
+ // and one for the close parenthesis if there are no more unionall
+ // or one for the string 'UNION ALL' if there are more union all
+ LOG.debug("Child of union all");
+ cqi.endPos = getEndPos(originalQuery, parent.getParent().getChild(1).getCharPositionInLine(), ")",
+ "UNION ALL");
+ }
+ }
+ LOG.debug("Adding cqi " + cqi + " query:" + originalQuery.substring(cqi.startPos, cqi.endPos));
+ cubeQueries.add(cqi);
+ } else {
+ for (int childPos = 0; childPos < childCount; ++childPos) {
+ findCubePositions((ASTNode) ast.getChild(childPos), cubeQueries, originalQuery);
+ }
+ }
+ } else {
+ LOG.warn("Null AST!");
+ }
+ }
+
+ /**
+ * Gets the end pos.
+ *
+ * @param query the query
+ * @param backTrackIndex the back track index
+ * @param backTrackStr the back track str
+ * @return the end pos
+ */
+ private static int getEndPos(String query, int backTrackIndex, String... backTrackStr) {
+ if (backTrackStr != null) {
+ String q = query.substring(0, backTrackIndex).toLowerCase();
+ for (int i = 0; i < backTrackStr.length; i++) {
+ if (q.trim().endsWith(backTrackStr[i].toLowerCase())) {
+ backTrackIndex = q.lastIndexOf(backTrackStr[i].toLowerCase());
+ break;
+ }
+ }
+ }
+ while (Character.isSpaceChar(query.charAt(backTrackIndex - 1))) {
+ backTrackIndex--;
+ }
+ return backTrackIndex;
+ }
+
+ /**
+ * Gets the rewriter.
+ *
+ * @param queryConf the query conf
+ * @return the rewriter
+ * @throws SemanticException the semantic exception
+ */
+ static CubeQueryRewriter getCubeRewriter(Configuration queryConf, HiveConf hconf) throws SemanticException {
+ return new CubeQueryRewriter(queryConf, hconf);
+ }
+
+ /**
+ * Replaces new lines with spaces; '&&' with AND; '||' with OR // these two can be removed once HIVE-5326 gets
+ * resolved.
+ *
+ * @param query the query
+ * @return the replaced query
+ */
+ static String getReplacedQuery(final String query) {
+ return query.replaceAll("[\\n\\r]", " ").replaceAll("&&", " AND ").replaceAll("\\|\\|", " OR ").trim();
+ }
+
+ private static final String REWRITE_QUERY_GAUGE = RewriteUtil.class.getSimpleName() + "-rewriteQuery";
+ private static final String TOHQL_GAUGE = RewriteUtil.class.getSimpleName() + "-rewriteQuery-toHQL";
+
+ /**
+ * Rewrite query.
+ *
+ * @param ctx the query context
+ * @return the map
+ * @throws LensException the lens exception
+ */
+ public static Map<LensDriver, DriverRewriterRunnable> rewriteQuery(AbstractQueryContext ctx) throws LensException {
+ try {
+
+ String replacedQuery = getReplacedQuery(ctx.getPhase1RewrittenQuery());
+ Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<LensDriver, DriverRewriterRunnable>();
+ List<RewriteUtil.CubeQueryInfo> cubeQueries = findCubePositions(replacedQuery, ctx.getHiveConf());
+
+ for (LensDriver driver : ctx.getDriverContext().getDrivers()) {
+ runnables.put(driver, new DriverRewriterRunnable(driver, ctx, cubeQueries, replacedQuery));
+ }
+
+ return runnables;
+ } catch (LensException e) {
+
+ /* REST layer has to catch LensException and add additional information to it.
+ To enable REST layer to call methods on LensException instance, we have to propagate
+ LensException to the REST layer. Since there is a Exception e catch here,
+ we need to explicitly catch and rethrow LensException to enable us for the same.*/
+
+ throw e;
+ } catch (Exception e) {
+ throw new LensException("Rewriting failed, cause :" + e.getMessage(), e);
+ }
+ }
+
+ public static DriverQueryPlan getRewriterPlan(DriverRewriterRunnable rewriter) {
+ RewriterPlan plan = new RewriterPlan(rewriter.cubeQueryCtx);
+ return plan;
+ }
+
+ public static class DriverRewriterRunnable implements Runnable {
+ @Getter
+ private final LensDriver driver;
+ private final AbstractQueryContext ctx;
+ private final List<CubeQueryInfo> cubeQueries;
+ private final String replacedQuery;
+ /** Cube query context - set after rewriting */
+ private List<CubeQueryContext> cubeQueryCtx;
+
+ @Getter
+ /** Indicate if rewrite operation succeeded */
+ private boolean succeeded;
+
+ @Getter
+ /** Get cause of rewrite failure if rewrite operation failed */
+ private String failureCause = null;
+
+ @Getter
+ private LensException cause;
+
+ @Getter
+ /** Get eventual rewritten query */
+ private String rewrittenQuery;
+
+ public DriverRewriterRunnable(LensDriver driver,
+ AbstractQueryContext ctx,
+ List<CubeQueryInfo> cubeQueries,
+ String replacedQuery) {
+ this.driver = driver;
+ this.ctx = ctx;
+ this.cubeQueries = cubeQueries;
+ this.replacedQuery = replacedQuery;
+ if (cubeQueries != null) {
+ cubeQueryCtx = new ArrayList<CubeQueryContext>(cubeQueries.size());
+ }
+ }
+
+ @Override
+ public void run() {
+ String lowerCaseQuery = replacedQuery.toLowerCase();
+ if (lowerCaseQuery.startsWith("add") || lowerCaseQuery.startsWith("set")) {
+ rewrittenQuery = replacedQuery;
+ return;
+ }
+
+ MethodMetricsContext rewriteGauge = MethodMetricsFactory
+ .createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_QUERY_GAUGE);
+ StringBuilder builder = new StringBuilder();
+ int start = 0;
+ CubeQueryRewriter rewriter = null;
+ try {
+ if (cubeQueries.size() > 0) {
+ // avoid creating rewriter if there are no cube queries
+ rewriter = getCubeRewriter(ctx.getDriverContext().getDriverConf(driver), ctx.getHiveConf());
+ ctx.setOlapQuery(true);
+ }
+
+ // We have to rewrite each sub cube query which might be present in the original
+ // user query. We are looping through all sub queries here.
+ int qIndex = 1;
+ for (RewriteUtil.CubeQueryInfo cqi : cubeQueries) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Rewriting cube query:" + cqi.query);
+ }
+
+ if (start != cqi.startPos) {
+ builder.append(replacedQuery.substring(start, cqi.startPos));
+ }
+
+ // Parse and rewrite individual cube query
+ CubeQueryContext cqc = rewriter.rewrite(cqi.query);
+ MethodMetricsContext toHQLGauge = MethodMetricsFactory
+ .createMethodGauge(ctx.getDriverConf(driver), true, qIndex + "-" + TOHQL_GAUGE);
+ // toHQL actually generates the rewritten query
+ String hqlQuery = cqc.toHQL();
+ cubeQueryCtx.add(cqc);
+ toHQLGauge.markSuccess();
+ qIndex++;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Rewritten query:" + hqlQuery);
+ }
+
+ builder.append(hqlQuery);
+ start = cqi.endPos;
+ }
+
+ builder.append(replacedQuery.substring(start));
+
+ rewrittenQuery = builder.toString();
+ // set rewriter plan
+ ctx.getDriverContext().setDriverRewriterPlan(driver, getRewriterPlan(this));
+ succeeded = true;
+ ctx.setDriverQuery(driver, rewrittenQuery);
+ LOG.info("Final rewritten query for driver:" + driver + " is: " + rewrittenQuery);
+
+ } catch (final LensException e) {
+
+ this.cause = e;
+ captureExceptionInformation(e);
+ } catch (Exception e) {
+
+ // we are catching all exceptions sothat other drivers can be picked in case of driver bugs
+ captureExceptionInformation(e);
+ } finally {
+ if (rewriter != null) {
+ rewriter.clear();
+ }
+ rewriteGauge.markSuccess();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Rewrite runnable for " + driver;
+ }
+
+ private void captureExceptionInformation(final Exception e) {
+
+ LOG.warn("Driver : " + driver + " Skipped for the query rewriting due to ", e);
+ ctx.setDriverRewriteError(driver, e);
+ failureCause = new StringBuilder(" Driver :")
+ .append(driver.getClass().getName())
+ .append(" Cause :" + e.getLocalizedMessage())
+ .toString();
+ }
+ }
+
+ /**
+ * Checks if is cube query.
+ *
+ * @param query the query
+ * @return true, if is cube query
+ */
+ public static boolean isCubeQuery(String query) {
+ if (matcher == null) {
+ matcher = cubePattern.matcher(query);
+ } else {
+ matcher.reset(query);
+ }
+ return matcher.matches();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java
new file mode 100644
index 0000000..35de4d7
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/UserQueryToCubeQueryRewriter.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.rewrite;
+
+import java.util.List;
+
+import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.rewrite.Phase1Rewriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Rewrite user query to cube query using phase 1 rewriters specified in the server installation
+ * @see Phase1Rewriter
+ * @see LensConfConstants#QUERY_PHASE1_REWRITERS
+ */
+public class UserQueryToCubeQueryRewriter {
+ List<Phase1Rewriter> phase1RewriterList = Lists.newArrayList();
+
+ public UserQueryToCubeQueryRewriter(Configuration conf) throws LensException {
+ try {
+ for (Class clazz : conf.getClasses(LensConfConstants.QUERY_PHASE1_REWRITERS)) {
+ if (!Phase1Rewriter.class.isAssignableFrom(clazz)) {
+ throw new LensException("Class " + clazz.getCanonicalName() + " is not Phase 1 Rewriter");
+ }
+ Phase1Rewriter rewriter = (Phase1Rewriter) clazz.newInstance();
+ rewriter.init(conf);
+ phase1RewriterList.add(rewriter);
+ }
+ } catch (Exception e) {
+ throw LensException.wrap(e);
+ }
+ }
+
+ public String rewriteToCubeQuery(String userQuery, Configuration conf, HiveConf hiveConf) throws LensException {
+ for (Phase1Rewriter rewriter : phase1RewriterList) {
+ userQuery = rewriter.rewrite(userQuery, conf, hiveConf);
+ }
+ return userQuery;
+ }
+
+ public void rewrite(AbstractQueryContext ctx) throws LensException {
+ ctx.setPhase1RewrittenQuery(rewriteToCubeQuery(ctx.getUserQuery(), ctx.getConf(), ctx.getHiveConf()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/689c9a99/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index da00c0d..2b540b8 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -44,6 +44,16 @@
</property>
<property>
+ <name>lens.server.query.phase1.rewriters</name>
+ <value></value>
+ <description>Query phase 1 rewriters. This is to convert user query to cube query. The resulting cube query will
+ be passed for validation and rewriting to hql query.\
+ Use cases will be to use extra intelligence to convert user query to optimized cube query.\
+ Or define shortcuts for certain frequently used queries :)
+ </description>
+ </property>
+
+ <property>
<name>lens.server.servicenames</name>
<value>session,query,metastore,scheduler,quota</value>
<description>These services would be started in the specified order when lens-server starts up</description>
[42/50] [abbrv] incubator-lens git commit: LENS-698 : Fix findbug
error in PopulateSampleMetastore
Posted by sh...@apache.org.
LENS-698 : Fix findbug error in PopulateSampleMetastore
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/701d28bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/701d28bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/701d28bc
Branch: refs/heads/LENS-581
Commit: 701d28bcd97c6e57507845d0f3be3386d7c3406f
Parents: 3ce889f
Author: Yash Sharma <ya...@apache.org>
Authored: Tue Jul 28 12:05:35 2015 +0530
Committer: Raju Bairishetti <ra...@c1mng0pxdty3.corp.inmobi.com>
Committed: Tue Jul 28 12:05:35 2015 +0530
----------------------------------------------------------------------
.../lens/examples/PopulateSampleMetastore.java | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/701d28bc/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
----------------------------------------------------------------------
diff --git a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
index 5ba0ea6..cfe3465 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
@@ -39,6 +39,16 @@ public class PopulateSampleMetastore {
private APIResult result;
private int retCode = 0;
+ private static final Date DATE = new Date(System.currentTimeMillis());
+ private static final SimpleDateFormat FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ private static final String NOW_TIME = FORMAT.format(DATE);
+
+ private static final String INSERT_QUERY = "INSERT INTO "
+ + "mydb_sales_aggr_continuous_fact (order_time, delivery_time, customer_id, "
+ + "product_id, promotion_id, customer_city_id, production_city_id, delivery_city_id, unit_sales, "
+ + "store_sales, store_cost, max_line_item_price, max_line_item_discount) values "
+ + "('" + NOW_TIME + "','" + NOW_TIME + "',2,2,1,2,2,2,1,8,2,10,2)";
+
public PopulateSampleMetastore() throws JAXBException {
metaClient = new LensMetadataClient(LensClientSingletonWrapper.instance().getClient().getConnection());
}
@@ -107,13 +117,7 @@ public class PopulateSampleMetastore {
con.setAutoCommit(true);
Statement statement = con.createStatement();
try {
- Date date = new Date(System.currentTimeMillis());
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- String nowTime = format.format(date);
- statement.execute("INSERT INTO mydb_sales_aggr_continuous_fact (order_time, delivery_time, customer_id, "
- + "product_id, promotion_id, customer_city_id, production_city_id, delivery_city_id, unit_sales, "
- + "store_sales, store_cost, max_line_item_price, max_line_item_discount) values "
- + "('" + nowTime + "','" + nowTime + "',2,2,1,2,2,2,1,8,2,10,2)");
+ statement.execute(INSERT_QUERY);
} finally {
statement.close();