You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2016/05/03 19:50:44 UTC
[1/5] drill git commit: Updated instructions for Linux build of the
Drill Native Client
Repository: drill
Updated Branches:
refs/heads/master b8f6ebc65 -> 2d9f9abb4
Updated instructions for Linux build of the Drill Native Client
1. CMake 3.0 is required, rather than CMake 2.8
2. Added some info that is helpful for the case where you build Boost the hard way.
This closes #492
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/94b8aec6
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/94b8aec6
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/94b8aec6
Branch: refs/heads/master
Commit: 94b8aec6414b006780a967ace4a29b36ac8b867b
Parents: b8f6ebc
Author: Patrick Wong <pw...@maprtech.com>
Authored: Mon May 2 18:30:34 2016 -0700
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue May 3 10:50:08 2016 -0700
----------------------------------------------------------------------
contrib/native/client/readme.linux | 17 +++++++++++++----
1 file changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/94b8aec6/contrib/native/client/readme.linux
----------------------------------------------------------------------
diff --git a/contrib/native/client/readme.linux b/contrib/native/client/readme.linux
index c2f9ba2..3e2336e 100644
--- a/contrib/native/client/readme.linux
+++ b/contrib/native/client/readme.linux
@@ -24,8 +24,8 @@ Install Prerequisites
0) Install development tools
$>yum groupinstall 'Development Tools'
-1) CMAKE 2.8
- $> yum install cmake28
+1) CMAKE 3.0
+ $> yum install cmake3
2.1) Download protobuf 2.5 from :
http://rpm.pbone.net/index.php3/stat/4/idpl/23552166/dir/centos_6/com/protobuf-2.5.0-16.1.x86_64.rpm.html
@@ -73,13 +73,22 @@ Install Prerequisites
OR
Download and build using boost build.
See this link for how to build: http://www.boost.org/doc/libs/1_53_0/more/getting_started/unix-variants.html#prepare-to-use-a-boost-library-binary
-
+ After building boost the hard way, you will need to do some symlinks for compatibility:
+ cd /usr/lib
+ ln -svf libboost_regex.so libboost_regex-mt.so
+ ln -svf libboost_system.so libboost_system-mt.so
+ ln -svf libboost_filesystem.so libboost_filesystem-mt.so
+ ln -svf libboost_date_time.so libboost_date_time-mt.so
+ ln -svf libboost_regex.a libboost_regex-mt.a
+ ln -svf libboost_system.a libboost_system-mt.a
+ ln -svf libboost_filesystem.a libboost_filesystem-mt.a
+ ln -svf libboost_date_time.a libboost_date_time-mt.a
Build drill client
-------------------
$> cd DRILL_DIR/contrib/native/client
$> mkdir build
- $> cd build && cmake28 -G "Unix Makefiles" -D CMAKE_BUILD_TYPE=Debug ..
+ $> cd build && cmake3 -G "Unix Makefiles" -D CMAKE_BUILD_TYPE=Debug ..
$> make
Test
[2/5] drill git commit: DRILL-2100: Added deleting temporary spill
directories when query is finished.
Posted by pa...@apache.org.
DRILL-2100: Added deleting temporary spill directories when query is finished.
This closes #454
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/38e1016c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/38e1016c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/38e1016c
Branch: refs/heads/master
Commit: 38e1016c49786acaacb153ee37784b3ce3023eb5
Parents: 1a89a7f
Author: Vitalii Diravka <vi...@gmail.com>
Authored: Mon Mar 28 18:05:22 2016 +0000
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue May 3 10:50:09 2016 -0700
----------------------------------------------------------------------
.../physical/impl/xsort/ExternalSortBatch.java | 31 ++++++++++++++++++--
1 file changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/38e1016c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
index 0ee518e..32df705 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/xsort/ExternalSortBatch.java
@@ -22,8 +22,10 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Sets;
import org.apache.calcite.rel.RelFieldCollation.Direction;
import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.config.DrillConfig;
@@ -72,6 +74,7 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractContainerVector;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import com.google.common.base.Joiner;
import com.google.common.base.Stopwatch;
@@ -116,6 +119,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
private boolean first = true;
private int targetRecordCount;
private final String fileName;
+ private Set<Path> currSpillDirs = Sets.newTreeSet();
private int firstSpillBatchCount = 0;
private int peakNumBatches = -1;
@@ -158,7 +162,7 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
copierAllocator = oAllocator.newChildAllocator(oAllocator.getName() + ":copier",
PriorityQueueCopier.INITIAL_ALLOCATION, PriorityQueueCopier.MAX_ALLOCATION);
FragmentHandle handle = context.getHandle();
- fileName = String.format("%s/major_fragment_%s/minor_fragment_%s/operator_%s", QueryIdHelper.getQueryId(handle.getQueryId()),
+ fileName = String.format("%s_majorfragment%s_minorfragment%s_operator%s", QueryIdHelper.getQueryId(handle.getQueryId()),
handle.getMajorFragmentId(), handle.getMinorFragmentId(), popConfig.getOperatorId());
}
@@ -223,7 +227,19 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
if (mSorter != null) {
mSorter.clear();
}
-
+ for(Iterator iter = this.currSpillDirs.iterator(); iter.hasNext(); iter.remove()) {
+ Path path = (Path)iter.next();
+ try {
+ if (fs != null && path != null && fs.exists(path)) {
+ if (fs.delete(path, true)) {
+ fs.cancelDeleteOnExit(path);
+ }
+ }
+ } catch (IOException e) {
+ // since this is meant to be used in a batches's cleanup, we don't propagate the exception
+ logger.warn("Unable to delete spill directory " + path, e);
+ }
+ }
}
}
@@ -554,7 +570,16 @@ public class ExternalSortBatch extends AbstractRecordBatch<ExternalSort> {
c1.buildSchema(BatchSchema.SelectionVectorMode.NONE);
c1.setRecordCount(count);
- String outputFile = Joiner.on("/").join(dirs.next(), fileName, spillCount++);
+ String spillDir = dirs.next();
+ Path currSpillPath = new Path(Joiner.on("/").join(spillDir, fileName));
+ currSpillDirs.add(currSpillPath);
+ String outputFile = Joiner.on("/").join(currSpillPath, spillCount++);
+ try {
+ fs.deleteOnExit(currSpillPath);
+ } catch (IOException e) {
+ // since this is meant to be used in a batches's spilling, we don't propagate the exception
+ logger.warn("Unable to mark spill directory " + currSpillPath + " for deleting on exit", e);
+ }
stats.setLongStat(Metric.SPILL_COUNT, spillCount);
BatchGroup newGroup = new BatchGroup(c1, fs, outputFile, oContext);
try (AutoCloseable a = AutoCloseables.all(batchGroupList)) {
[4/5] drill git commit: DRILL-4584: JDBC/ODBC Client IP in Drill
audit logs - the format of added field in log files is
"remoteAddress":"192.168.121.1:58984"
Posted by pa...@apache.org.
DRILL-4584: JDBC/ODBC Client IP in Drill audit logs - the format of added field in log files is "remoteAddress":"192.168.121.1:58984"
This closes #475
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2d9f9abb
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2d9f9abb
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2d9f9abb
Branch: refs/heads/master
Commit: 2d9f9abb4c47d08f8462599c8d6076a61a1708fe
Parents: a6a85ab
Author: Vitalii Diravka <vi...@gmail.com>
Authored: Fri Apr 8 11:01:37 2016 +0300
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue May 3 10:50:09 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/drill/exec/work/foreman/Foreman.java | 3 ++-
.../org/apache/drill/exec/work/foreman/LoggedQuery.java | 10 +++++++++-
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/2d9f9abb/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index 1464cad..29c7971 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -687,7 +687,8 @@ public class Foreman implements Runnable {
new Date(queryContext.getQueryContextInfo().getQueryStartTime()),
new Date(System.currentTimeMillis()),
state,
- queryContext.getSession().getCredentials().getUserName());
+ queryContext.getSession().getCredentials().getUserName(),
+ initiatingClient.getChannel().remoteAddress());
queryLogger.info(MAPPER.writeValueAsString(q));
} catch (Exception e) {
logger.error("Failure while recording query information to query log.", e);
http://git-wip-us.apache.org/repos/asf/drill/blob/2d9f9abb/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/LoggedQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/LoggedQuery.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/LoggedQuery.java
index f856d53..4248595 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/LoggedQuery.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/LoggedQuery.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.work.foreman;
+import java.net.SocketAddress;
import java.util.Date;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
@@ -33,9 +34,11 @@ public class LoggedQuery {
private final Date finish;
private final QueryState outcome;
private final String username;
+ private final SocketAddress remoteAddress;
+
public LoggedQuery(String queryId, String schema, String queryText, Date start, Date finish, QueryState outcome,
- String username) {
+ String username, SocketAddress remoteAddress) {
super();
this.queryId = queryId;
this.schema = schema;
@@ -44,6 +47,7 @@ public class LoggedQuery {
this.finish = finish;
this.outcome = outcome;
this.username = username;
+ this.remoteAddress = remoteAddress;
}
@JsonProperty("id")
@@ -77,4 +81,8 @@ public class LoggedQuery {
return username;
}
+ public String getRemoteAddress() {
+ return remoteAddress.toString().replace("/","");
+ }
+
}
[5/5] drill git commit: DRILL-4571: Add link to local Drill logs from
the web UI
Posted by pa...@apache.org.
DRILL-4571: Add link to local Drill logs from the web UI
This closes #472
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1a89a7fe
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1a89a7fe
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1a89a7fe
Branch: refs/heads/master
Commit: 1a89a7fe56533ecea8f7a7c9be6a3699925f3c96
Parents: 94b8aec
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Thu Mar 31 18:43:25 2016 +0300
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue May 3 10:50:09 2016 -0700
----------------------------------------------------------------------
distribution/src/resources/logback.xml | 2 +-
.../org/apache/drill/exec/ExecConstants.java | 5 +
.../server/options/SystemOptionManager.java | 3 +-
.../drill/exec/server/rest/DrillRestServer.java | 1 +
.../drill/exec/server/rest/LogsResources.java | 211 +++++++++++++++++++
.../server/rest/ViewableWithPermissions.java | 1 +
.../src/main/resources/rest/generic.ftl | 3 +
.../src/main/resources/rest/logs/list.ftl | 55 +++++
.../src/main/resources/rest/logs/log.ftl | 37 ++++
9 files changed, 316 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/distribution/src/resources/logback.xml
----------------------------------------------------------------------
diff --git a/distribution/src/resources/logback.xml b/distribution/src/resources/logback.xml
index 350383a..fb53dfc 100644
--- a/distribution/src/resources/logback.xml
+++ b/distribution/src/resources/logback.xml
@@ -44,7 +44,7 @@
<maxFileSize>100MB</maxFileSize>
</triggeringPolicy>
<encoder>
- <pattern>%msg</pattern>
+ <pattern>%msg%n</pattern>
</encoder>
</appender>
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index 7f216f0..17fbb7b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -303,4 +303,9 @@ public interface ExecConstants {
StringValidator IMPERSONATION_POLICY_VALIDATOR =
new InboundImpersonationManager.InboundImpersonationPolicyValidator(IMPERSONATION_POLICIES_KEY, "[]");
+ /**
+ * Web settings
+ */
+ String WEB_LOGS_MAX_LINES = "web.logs.max_lines";
+ OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE, 10000);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index db78108..c35ed0e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -135,7 +135,8 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
ExecConstants.ENABLE_WINDOW_FUNCTIONS_VALIDATOR,
ClassTransformer.SCALAR_REPLACEMENT_VALIDATOR,
ExecConstants.ENABLE_NEW_TEXT_READER,
- ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST
+ ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST,
+ ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR
};
final Map<String, OptionValidator> tmp = new HashMap<>();
for (final OptionValidator validator : validators) {
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
index ceecdb4..0401d58 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/DrillRestServer.java
@@ -55,6 +55,7 @@ public class DrillRestServer extends ResourceConfig {
register(QueryResources.class);
register(MetricsResources.class);
register(ThreadsResources.class);
+ register(LogsResources.class);
register(FreemarkerMvcFeature.class);
register(MultiPartFeature.class);
property(ServerProperties.METAINF_SERVICES_LOOKUP_DISABLE, true);
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
new file mode 100644
index 0000000..8a89d41
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/LogsResources.java
@@ -0,0 +1,211 @@
+/**
+ * ****************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ****************************************************************************
+ */
+package org.apache.drill.exec.server.rest;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.collect.Sets;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.work.WorkManager;
+import org.glassfish.jersey.server.mvc.Viewable;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+import javax.annotation.security.RolesAllowed;
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.SecurityContext;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.drill.exec.server.rest.auth.DrillUserPrincipal.ADMIN_ROLE;
+
+@Path("/")
+@RolesAllowed(ADMIN_ROLE)
+public class LogsResources {
+
+ @Inject DrillRestServer.UserAuthEnabled authEnabled;
+ @Inject SecurityContext sc;
+ @Inject WorkManager work;
+
+ private static final FileFilter file_filter = new FileFilter() {
+ @Override
+ public boolean accept(File file) {
+ return file.isFile();
+ }
+ };
+ private static final DateTimeFormatter format = DateTimeFormat.forPattern("MM/dd/yyyy HH:mm:ss");
+
+
+ @GET
+ @Path("/logs")
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getLogs() {
+ Set<Log> logs = getLogsJSON();
+ return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/list.ftl", sc, logs);
+ }
+
+ @GET
+ @Path("/logs.json")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<Log> getLogsJSON() {
+ Set<Log> logs = Sets.newTreeSet();
+ File[] files = getLogFolder().listFiles(file_filter);
+
+ for (File file : files) {
+ logs.add(new Log(file.getName(), file.length(), file.lastModified()));
+ }
+
+ return logs;
+ }
+
+ @GET
+ @Path("/log/{name}/content")
+ @Produces(MediaType.TEXT_HTML)
+ public Viewable getLog(@PathParam("name") String name) throws IOException {
+ LogContent content = getLogJSON(name);
+ return ViewableWithPermissions.create(authEnabled.get(), "/rest/logs/log.ftl", sc, content);
+ }
+
+ @GET
+ @Path("/log/{name}/content.json")
+ @Produces(MediaType.APPLICATION_JSON)
+ public LogContent getLogJSON(@PathParam("name") final String name) throws IOException {
+ File file = getFileByName(getLogFolder(), name);
+
+ final int maxLines = work.getContext().getOptionManager().getOption(ExecConstants.WEB_LOGS_MAX_LINES).num_val.intValue();
+
+ try (BufferedReader br = new BufferedReader(new FileReader(file))) {
+ Map<String, String> cache = new LinkedHashMap<String, String>(maxLines, .75f, true) {
+ @Override
+ protected boolean removeEldestEntry(Map.Entry<String, String> eldest) {
+ return size() > maxLines;
+ }
+ };
+
+ String line;
+ while ((line = br.readLine()) != null) {
+ cache.put(line, null);
+ }
+
+ return new LogContent(file.getName(), cache.keySet(), maxLines);
+ }
+ }
+
+ @GET
+ @Path("/log/{name}/download")
+ @Produces(MediaType.TEXT_PLAIN)
+ public Response getFullLog(@PathParam("name") final String name) {
+ File file = getFileByName(getLogFolder(), name);
+ Response.ResponseBuilder response = Response.ok(file);
+ response.header("Content-Disposition", String.format("attachment;filename\"%s\"", name));
+ return response.build();
+ }
+
+ private File getLogFolder() {
+ return new File(System.getenv("DRILL_LOG_DIR"));
+ }
+
+ private File getFileByName(File folder, final String name) {
+ File[] files = folder.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String fileName) {
+ return fileName.equals(name);
+ }
+ });
+ if (files.length == 0) {
+ throw new DrillRuntimeException (name + " doesn't exist");
+ }
+ return files[0];
+ }
+
+
+ @XmlRootElement
+ public class Log implements Comparable<Log> {
+
+ private String name;
+ private long size;
+ private DateTime lastModified;
+
+ @JsonCreator
+ public Log (@JsonProperty("name") String name, @JsonProperty("size") long size, @JsonProperty("lastModified") long lastModified) {
+ this.name = name;
+ this.size = size;
+ this.lastModified = new DateTime(lastModified);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getSize() {
+ return Math.ceil(size / 1024d) + " KB";
+ }
+
+ public String getLastModified() {
+ return lastModified.toString(format);
+ }
+
+ @Override
+ public int compareTo(Log log) {
+ return this.getName().compareTo(log.getName());
+ }
+ }
+
+ @XmlRootElement
+ public class LogContent {
+ private String name;
+ private Collection<String> lines;
+ private int maxLines;
+
+ @JsonCreator
+ public LogContent (@JsonProperty("name") String name, @JsonProperty("lines") Collection<String> lines, @JsonProperty("maxLines") int maxLines) {
+ this.name = name;
+ this.lines = lines;
+ this.maxLines = maxLines;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Collection<String> getLines() { return lines; }
+
+ public int getMaxLines() { return maxLines; }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
index b2a0fae..73019aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/ViewableWithPermissions.java
@@ -78,6 +78,7 @@ public class ViewableWithPermissions extends Viewable {
.put("showStorage", isAdmin)
.put("showOptions", isAdmin)
.put("showThreads", isAdmin)
+ .put("showLogs", isAdmin)
.put("showLogin", authEnabled && showControls && !isUserLoggedIn)
.put("showLogout", authEnabled && showControls && isUserLoggedIn)
.put("loggedInUserName", authEnabled && showControls &&
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/generic.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/generic.ftl b/exec/java-exec/src/main/resources/rest/generic.ftl
index b3e249e..60869e7 100644
--- a/exec/java-exec/src/main/resources/rest/generic.ftl
+++ b/exec/java-exec/src/main/resources/rest/generic.ftl
@@ -64,6 +64,9 @@
<#if showThreads == true>
<li><a href="/threads">Threads</a></li>
</#if>
+ <#if showLogs == true>
+ <li><a href="/logs">Logs</a></li>
+ </#if>
</ul>
</#if>
<ul class="nav navbar-nav navbar-right">
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/list.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/logs/list.ftl b/exec/java-exec/src/main/resources/rest/logs/list.ftl
new file mode 100644
index 0000000..3d836df
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/logs/list.ftl
@@ -0,0 +1,55 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+<a href="/queries">back</a><br/>
+<div class="page-header">
+</div>
+
+<#if (model?size > 0)>
+<div class="table-responsive">
+ <table class="table table-hover">
+ <thead>
+ <td>Name</td>
+ <td>Size</td>
+ <td>Last Modified</td>
+ </thead>
+ <tbody>
+ <#list model as log>
+ <tr>
+ <td>
+ <a href="/log/${log.getName()}/content">
+ <div style="height:100%;width:100%;white-space:pre-line">${log.getName()}</div>
+ </a>
+ </td>
+ <td>
+ <div style="height:100%;width:100%;white-space:pre-line">${log.getSize()}</div>
+ </td>
+ <td>
+ <div style="height:100%;width:100%;white-space:pre-line">${log.getLastModified()}</div>
+ </td>
+ </tr>
+ </#list>
+ </tbody>
+ </table>
+</div>
+<#else>
+<div id="message" class="alert alert-info">
+ <strong>No logs are available.</strong>
+</div>
+</#if>
+</#macro>
+
+<@page_html/>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/1a89a7fe/exec/java-exec/src/main/resources/rest/logs/log.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/logs/log.ftl b/exec/java-exec/src/main/resources/rest/logs/log.ftl
new file mode 100644
index 0000000..b09b57a
--- /dev/null
+++ b/exec/java-exec/src/main/resources/rest/logs/log.ftl
@@ -0,0 +1,37 @@
+<#-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<#include "*/generic.ftl">
+<#macro page_head>
+</#macro>
+
+<#macro page_body>
+<a href="/logs">back</a><br/>
+<div class="page-header">
+</div>
+<h3>${model.getName()} <span class="badge alert-info">(last ${model.getMaxLines()} lines)</span></h3>
+<p>
+ <a href="/log/${model.getName()}/download">Download Full Log</a>
+</p>
+ <#if (model.getLines()?size > 0)>
+ <pre>
+ <#list model.getLines() as line>
+${line}
+ </#list>
+ </pre>
+ <#else>
+ <div id="message" class="alert alert-info">
+ <strong>Log is empty.</strong>
+ </div>
+ </#if>
+</#macro>
+
+<@page_html/>
\ No newline at end of file
[3/5] drill git commit: DRILL-3894: Upgrade functions MaxDir,
MinDir... Optional filename parameter - added implementations of functions
MAXDIR, IMAXDIR, MINDIR, IMINDIR with one (schema) without filename argument;
- added UTest for results comparison of
Posted by pa...@apache.org.
DRILL-3894: Upgrade functions MaxDir, MinDir... Optional filename parameter - added implementations of functions MAXDIR, IMAXDIR, MINDIR, IMINDIR with one (schema) without filename argument; - added UTest for results comparison of using Query Directory Functions with one and two arguments.
This closes #467
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a6a85ab6
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a6a85ab6
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a6a85ab6
Branch: refs/heads/master
Commit: a6a85ab66360cac81ab4777cec20292470ac483d
Parents: 38e1016
Author: Vitalii Diravka <vi...@gmail.com>
Authored: Tue Apr 5 15:07:29 2016 +0000
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue May 3 10:50:09 2016 -0700
----------------------------------------------------------------------
.../codegen/templates/DirectoryExplorers.java | 23 ++++++++++++++++++
.../exec/planner/TestDirectoryExplorerUDFs.java | 25 ++++++++++++++++++++
2 files changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a85ab6/exec/java-exec/src/main/codegen/templates/DirectoryExplorers.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/DirectoryExplorers.java b/exec/java-exec/src/main/codegen/templates/DirectoryExplorers.java
index 655ff81..a47a541 100644
--- a/exec/java-exec/src/main/codegen/templates/DirectoryExplorers.java
+++ b/exec/java-exec/src/main/codegen/templates/DirectoryExplorers.java
@@ -43,6 +43,11 @@ public class DirectoryExplorers {
{ "name" : "\"imaxdir\"", "functionClassName" : "IMaxDir", "comparison" : "compareToIgnoreCase(curr) < 0", "goal" : "maximum", "comparisonType" : "case-insensitive"},
{ "name" : "\"mindir\"", "functionClassName" : "MinDir", "comparison" : "compareTo(curr) > 0", "goal" : "minimum", "comparisonType" : "case-sensitive"},
{ "name" : "\"imindir\"", "functionClassName" : "IMinDir", "comparison" : "compareToIgnoreCase(curr) > 0", "goal" : "minimum", "comparisonType" : "case-insensitive"}
+
+ { "name" : "\"maxdir\"", "functionClassName" : "MaxDirTwoArg", "comparison" : "compareTo(curr) < 0", "goal" : "maximum", "comparisonType" : "case-sensitive"},
+ { "name" : "\"imaxdir\"", "functionClassName" : "IMaxDirTwoArg", "comparison" : "compareToIgnoreCase(curr) < 0", "goal" : "maximum", "comparisonType" : "case-insensitive"},
+ { "name" : "\"mindir\"", "functionClassName" : "MinDirTwoArg", "comparison" : "compareTo(curr) > 0", "goal" : "minimum", "comparisonType" : "case-sensitive"},
+ { "name" : "\"imindir\"", "functionClassName" : "IMinDirTwoArg", "comparison" : "compareToIgnoreCase(curr) > 0", "goal" : "minimum", "comparisonType" : "case-insensitive"}
] as dirAggrProps>
@@ -50,7 +55,9 @@ public class DirectoryExplorers {
public static class ${dirAggrProps.functionClassName} implements DrillSimpleFunc {
@Param VarCharHolder schema;
+ <#if dirAggrProps.functionClassName?ends_with("TwoArg")>
@Param VarCharHolder table;
+ </#if>
@Output VarCharHolder out;
@Inject DrillBuf buffer;
@Inject org.apache.drill.exec.store.PartitionExplorer partitionExplorer;
@@ -63,23 +70,39 @@ public class DirectoryExplorers {
try {
subPartitions = partitionExplorer.getSubPartitions(
org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getStringFromVarCharHolder(schema),
+ <#if dirAggrProps.functionClassName?ends_with("TwoArg")>
org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getStringFromVarCharHolder(table),
+ <#else>
+ ".",
+ </#if>
new java.util.ArrayList<String>(),
new java.util.ArrayList<String>());
} catch (org.apache.drill.exec.store.PartitionNotFoundException e) {
throw new RuntimeException(
+ <#if dirAggrProps.functionClassName?ends_with("TwoArg")>
String.format("Error in %s function: Table %s does not exist in schema %s ",
+ <#else>
+ String.format("Error in %s function: Schema/table %s does not exist ",
+ </#if>
${dirAggrProps.name},
+ <#if dirAggrProps.functionClassName?ends_with("TwoArg")>
org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getStringFromVarCharHolder(table),
+ </#if>
org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getStringFromVarCharHolder(schema))
);
}
java.util.Iterator partitionIterator = subPartitions.iterator();
if (!partitionIterator.hasNext()) {
throw new RuntimeException(
+ <#if dirAggrProps.functionClassName?ends_with("TwoArg")>
String.format("Error in %s function: Table %s in schema %s does not contain sub-partitions.",
+ <#else>
+ String.format("Error in %s function: Schema/table %s does not contain sub-partitions.",
+ </#if>
${dirAggrProps.name},
+ <#if dirAggrProps.functionClassName?ends_with("TwoArg")>
org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getStringFromVarCharHolder(table),
+ </#if>
org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.getStringFromVarCharHolder(schema)
)
);
http://git-wip-us.apache.org/repos/asf/drill/blob/a6a85ab6/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
index c3427f6..a5916a5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/planner/TestDirectoryExplorerUDFs.java
@@ -24,7 +24,9 @@ import com.google.common.collect.ImmutableMap;
import org.apache.drill.PlanTestBase;
import org.apache.drill.common.exceptions.UserRemoteException;
import org.apache.drill.exec.fn.interp.TestConstantFolding;
+import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.util.TestUtilities;
import org.apache.drill.exec.util.Text;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -163,4 +165,27 @@ public class TestDirectoryExplorerUDFs extends PlanTestBase {
test("set `planner.enable_constant_folding` = true;");
}
}
+
+ @Test
+ public void testOneArgQueryDirFunctions() throws Exception {
+ //Initially update the location of dfs_test.tmp workspace with "path" temp directory just for use in this UTest
+ final StoragePluginRegistry pluginRegistry = getDrillbitContext().getStorage();
+ try {
+ TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, path);
+
+ //Results comparison of using Query Directory Functions (MAXDIR, IMAXDIR, MINDIR, IMINDIR) with one and two arguments
+ String queryWithTwoArgFunc = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs.root','" + path + "')";
+ String queryWithOneArgFunc = "select * from dfs.`" + path + "/*/*.csv` where dir0 = %s('dfs_test.tmp')";
+ for (ConstantFoldingTestConfig config : tests) {
+ testBuilder()
+ .sqlQuery(String.format(queryWithOneArgFunc, config.funcName))
+ .unOrdered()
+ .sqlBaselineQuery(String.format(queryWithTwoArgFunc, config.funcName))
+ .go();
+ }
+ } finally {
+ TestUtilities.updateDfsTestTmpSchemaLocation(pluginRegistry, getDfsTestTmpSchemaLocation());
+ }
+ }
+
}