You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2016/11/01 20:29:48 UTC
[03/15] drill git commit: DRILL-3243: Added CSG mods. Fixed field
names. Removed old test files Added Parse_url and parse_query() functions Fix
unit test
DRILL-3243: Added CSG mods. Fixed field names.
Removed old test files
Added Parse_url and parse_query() functions
Fix unit test
This closes #607
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4a82bc13
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4a82bc13
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4a82bc13
Branch: refs/heads/master
Commit: 4a82bc137865393f6f88b31805ad7c62cc9ca20b
Parents: 46c0f2a
Author: cgivre <cg...@gmail.com>
Authored: Wed Oct 5 23:29:26 2016 -0400
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue Nov 1 10:42:22 2016 -0700
----------------------------------------------------------------------
.../exec/expr/fn/impl/ParseQueryFunction.java | 87 ++++
.../exec/expr/fn/impl/ParseUrlFunction.java | 153 ++++++
.../exec/store/httpd/HttpdFormatPlugin.java | 487 -------------------
.../exec/store/httpd/HttpdLogFormatPlugin.java | 3 +-
.../drill/exec/store/httpd/HttpdLogRecord.java | 2 +-
.../drill/exec/store/httpd/HttpdParser.java | 123 ++++-
.../drill/exec/store/httpd/HttpdParserTest.java | 48 ++
.../dfs/TestFormatPluginOptionExtractor.java | 3 +
.../drill/exec/store/httpd/HttpdParserTest.java | 48 --
.../store/httpd/TestHttpdLogFormatPlugin.java | 97 ----
.../drill/exec/store/httpd/TestHttpdPlugin.java | 31 --
11 files changed, 411 insertions(+), 671 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java
new file mode 100644
index 0000000..7dce1fc
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseQueryFunction.java
@@ -0,0 +1,87 @@
+package org.apache.drill.exec.expr.fn.impl;
+
+//*
+
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
+
+import javax.inject.Inject;
+
+/* Copyright 2001-2004 The Apache Software Foundation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+@FunctionTemplate(
+ name="parse_query",
+ scope= FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = FunctionTemplate.NullHandling.NULL_IF_NULL
+)
+
+public class ParseQueryFunction implements DrillSimpleFunc {
+
+ @Param
+ NullableVarCharHolder input;
+
+ @Output
+ BaseWriter.ComplexWriter outWriter;
+
+ @Inject
+ DrillBuf outBuffer;
+
+ public void setup() {
+ }
+
+ public void eval() {
+
+ org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter queryMapWriter = outWriter.rootAsMap();
+
+ String queryString = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer);
+
+ if( queryString.isEmpty() || queryString.equals("null")){
+ queryString = "";
+ }
+
+ String firstLetter = queryString.substring(0, 1);
+
+ //If the first character is a &, it doesn't split properly. This checks to see if the first character is an & and if so, removes it.
+ if(firstLetter.equals("&")){
+ queryString = queryString.substring(1);
+ }
+
+ String[] arguments = queryString.split("&");
+
+ for (int i = 0; i < arguments.length; i++) {
+ String[] queryParts = arguments[i].split("=");
+
+ org.apache.drill.exec.expr.holders.VarCharHolder rowHolder = new org.apache.drill.exec.expr.holders.VarCharHolder();
+
+ byte[] rowStringBytes = queryParts[1].getBytes();
+
+ outBuffer.reallocIfNeeded(rowStringBytes.length);
+ outBuffer.setBytes(0, rowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = rowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ queryMapWriter.varChar(queryParts[0]).write(rowHolder);
+
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java
new file mode 100644
index 0000000..fa339d4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ParseUrlFunction.java
@@ -0,0 +1,153 @@
+package org.apache.drill.exec.expr.fn.impl;
+
+/*
+ * Copyright 2001-2004 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter;
+
+import javax.inject.Inject;
+
+@FunctionTemplate(
+ name="parse_url",
+ scope= FunctionTemplate.FunctionScope.SIMPLE,
+ nulls = FunctionTemplate.NullHandling.NULL_IF_NULL
+)
+
+public class ParseUrlFunction implements DrillSimpleFunc {
+
+ @Param VarCharHolder input;
+
+ @Output BaseWriter.ComplexWriter outWriter;
+
+ @Inject DrillBuf outBuffer;
+
+ public void setup() {}
+
+ public void eval() {
+
+ org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter urlMapWriter = outWriter.rootAsMap();
+
+ String urlString = org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers.toStringFromUTF8(input.start, input.end, input.buffer);
+
+ try {
+ java.net.URL aURL = new java.net.URL(urlString);
+
+ String protocol = aURL.getProtocol();
+ String authority = aURL.getAuthority();
+ String host = aURL.getHost();
+ java.lang.Integer port = aURL.getPort();
+ String path = aURL.getPath();
+ String query = aURL.getQuery();
+ String filename = aURL.getFile();
+ String ref = aURL.getRef();
+
+ org.apache.drill.exec.expr.holders.VarCharHolder rowHolder = new org.apache.drill.exec.expr.holders.VarCharHolder();
+
+ byte[] rowStringBytes = protocol.getBytes();
+
+ outBuffer.reallocIfNeeded(rowStringBytes.length);
+ outBuffer.setBytes(0, rowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = rowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ urlMapWriter.varChar("protocol").write(rowHolder);
+
+
+ byte[] authRowStringBytes = authority.getBytes();
+
+ outBuffer.reallocIfNeeded(authRowStringBytes.length);
+ outBuffer.setBytes(0, authRowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = authRowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ urlMapWriter.varChar("authority").write(rowHolder);
+
+
+ byte[] hostRowStringBytes = host.getBytes();
+
+ outBuffer.reallocIfNeeded(hostRowStringBytes.length);
+ outBuffer.setBytes(0, hostRowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = hostRowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ urlMapWriter.varChar("host").write(rowHolder);
+
+
+ byte[] pathRowStringBytes = path.getBytes();
+
+ outBuffer.reallocIfNeeded(pathRowStringBytes.length);
+ outBuffer.setBytes(0, pathRowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = pathRowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ urlMapWriter.varChar("path").write(rowHolder);
+
+
+ byte[] queryRowStringBytes = query.getBytes();
+
+ outBuffer.reallocIfNeeded(queryRowStringBytes.length);
+ outBuffer.setBytes(0, queryRowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = queryRowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ urlMapWriter.varChar("query").write(rowHolder);
+
+
+ byte[] filenameRowStringBytes = filename.getBytes();
+
+ outBuffer.reallocIfNeeded(filenameRowStringBytes.length);
+ outBuffer.setBytes(0, filenameRowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = filenameRowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ urlMapWriter.varChar("filename").write(rowHolder);
+
+
+ byte[] refRowStringBytes = ref.getBytes();
+
+ outBuffer.reallocIfNeeded(refRowStringBytes.length);
+ outBuffer.setBytes(0, refRowStringBytes);
+
+ rowHolder.start = 0;
+ rowHolder.end = refRowStringBytes.length;
+ rowHolder.buffer = outBuffer;
+
+ urlMapWriter.varChar("ref").write(rowHolder);
+
+ org.apache.drill.exec.expr.holders.IntHolder intHolder = new org.apache.drill.exec.expr.holders.IntHolder();
+ intHolder.value = port;
+ urlMapWriter.integer("port").write(intHolder);
+ }
+ catch (Exception e ) {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java
deleted file mode 100644
index 7b8dc0e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdFormatPlugin.java
+++ /dev/null
@@ -1,487 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import io.netty.buffer.DrillBuf;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-import nl.basjes.parse.core.Parser;
-import nl.basjes.parse.core.exceptions.DissectionFailure;
-import nl.basjes.parse.core.exceptions.InvalidDissectorException;
-import nl.basjes.parse.core.exceptions.MissingDissectorsException;
-import nl.basjes.parse.httpdlog.ApacheHttpdLogFormatDissector;
-import nl.basjes.parse.httpdlog.dissectors.HttpFirstLineDissector;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.exceptions.UserException;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.FormatPluginConfig;
-import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.ops.OperatorContext;
-import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.AbstractRecordReader;
-import org.apache.drill.exec.store.RecordWriter;
-import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.store.dfs.easy.EasyFormatPlugin;
-import org.apache.drill.exec.store.dfs.easy.EasyWriter;
-import org.apache.drill.exec.store.dfs.easy.FileWork;
-import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.drill.exec.vector.complex.writer.BigIntWriter;
-import org.apache.drill.exec.vector.complex.writer.Float8Writer;
-import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.FileSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.LineRecordReader;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.TextInputFormat;
-
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class HttpdFormatPlugin extends EasyFormatPlugin<HttpdFormatPlugin.HttpdLogFormatConfig> {
-
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HttpdFormatPlugin.class);
-
- private static final String DEFAULT_EXTENSION = "httpd";
-
- public HttpdFormatPlugin(
- String name,
- DrillbitContext context,
- Configuration fsConf,
- StoragePluginConfig storageConfig,
- HttpdLogFormatConfig formatConfig) {
- super(name, context, fsConf, storageConfig, formatConfig, true, false, true, true,
- Lists.newArrayList(DEFAULT_EXTENSION), DEFAULT_EXTENSION);
- }
-
- @JsonTypeName("httpd")
- public static class HttpdLogFormatConfig implements FormatPluginConfig {
- public String format;
- }
-
- private class RecordReader extends AbstractRecordReader {
-
- private final DrillFileSystem fs;
- private final FileWork work;
- private final FragmentContext fragmentContext;
-
- private ComplexWriter writer;
- private Parser<ComplexWriterFacade> parser;
- private LineRecordReader lineReader;
- private LongWritable lineNumber;
- private ComplexWriterFacade record;
- private DrillBuf managedBuffer;
-
- public RecordReader(FragmentContext context, DrillFileSystem fs, FileWork work) {
- this.fs = fs;
- this.work = work;
- fragmentContext = context;
- managedBuffer = context.getManagedBuffer();
- }
-
- @Override
- public void setup(OperatorContext context, OutputMutator output) throws ExecutionSetupException {
-
- try {
-
- parser = new PartiallyDissectedParser<ComplexWriterFacade>(
- ComplexWriterFacade.class,
- HttpdFormatPlugin.this.getConfig().format);
- writer = new VectorContainerWriter(output);
- record = new ComplexWriterFacade(writer);
- record.addAsParseTarget(parser);
-
- final Path path = fs.makeQualified(new Path(work.getPath()));
- FileSplit split = new FileSplit(path, work.getStart(), work.getLength(), new String[] { "" });
- TextInputFormat inputFormat = new TextInputFormat();
- JobConf job = new JobConf(fs.getConf());
- job.setInt("io.file.buffer.size", fragmentContext.getConfig()
- .getInt(ExecConstants.TEXT_LINE_READER_BUFFER_SIZE));
- job.setInputFormat(inputFormat.getClass());
- lineReader = (LineRecordReader) inputFormat.getRecordReader(split, job, Reporter.NULL);
- lineNumber = lineReader.createKey();
- } catch (Exception e) {
- throw handleAndGenerate("Failure in creating record reader", e);
- }
-
-
- }
-
- private DrillBuf buf(int size) {
- if (managedBuffer.capacity() < size) {
- managedBuffer = managedBuffer.reallocIfNeeded(size);
- }
- return managedBuffer;
- }
-
- protected RuntimeException handleAndGenerate(String s, Exception e) {
- throw UserException.dataReadError(e)
- .message(s + "\n%s", e.getMessage())
- .addContext("Path", work.getPath())
- .addContext("Split Start", work.getStart())
- .addContext("Split Length", work.getLength())
- .addContext("Local Line Number", lineNumber.get())
- .build(logger);
- }
-
- @Override
- public int next() {
- try {
- final Text currentText = lineReader.createValue();
-
- writer.allocate();
- writer.reset();
- int recordCount = 0;
-
- for (; recordCount < 4095 && lineReader.next(lineNumber, currentText); recordCount++) {
- writer.setPosition(recordCount);
- parser.parse(record, currentText.toString());
- }
-
- writer.setValueCount(recordCount);
- return recordCount;
- } catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException | IOException e) {
- throw handleAndGenerate("Failure while reading httpd log record.", e);
- }
- }
-
- @Override
- public void cleanup() {
-
- try {
- if (lineReader != null) {
- lineReader.close();
- }
- } catch (IOException e) {
- logger.warn("Failure while closing Httpd reader.", e);
- }
- }
-
- /**
- * Maps Httpd Log Libraries calls to Drills ComplexWriter interface.
- */
- public class ComplexWriterFacade {
- private final ComplexWriter writer;
- private final Map<String, VarCharWriter> stringWriters = Maps.newHashMap();
- private final Map<String, BigIntWriter> longWriters = Maps.newHashMap();
- private final Map<String, Float8Writer> doubleWriters = Maps.newHashMap();
-
- private ComplexWriterFacade(ComplexWriter writer) {
- this.writer = writer;
- }
-
- @SuppressWarnings("unused")
- public void set(final String name, final String value) {
- if (value != null) {
- final byte[] stringBytes = value.getBytes(Charsets.UTF_8);
- final DrillBuf stringBuffer = buf(stringBytes.length);
- stringBuffer.clear();
- stringBuffer.writeBytes(stringBytes);
- final VarCharWriter writer = stringWriters.get(name);
- if (writer != null) {
- writer.writeVarChar(0, stringBytes.length, stringBuffer);
- } else {
- logger.warn("Dropped string. Name: {}, Value: {}", name, value);
- }
- }
- }
-
- @SuppressWarnings("unused")
- public void set(String name, Long value) {
- if (value != null) {
- longWriters.get(name).writeBigInt(value);
- }
- }
-
- @SuppressWarnings("unused")
- public void set(String name, Double value) {
- if (value != null) {
- doubleWriters.get(name).writeFloat8(value);
- }
- }
-
- private void add(Parser<ComplexWriterFacade> parser, String path, VarCharWriter writer)
- throws NoSuchMethodException,
- SecurityException {
- stringWriters.put(path, writer);
- parser.addParseTarget(
- ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, String.class }),
- path);
- }
-
- @SuppressWarnings("unused")
- private void add(Parser<ComplexWriterFacade> parser, String path, Float8Writer writer)
- throws NoSuchMethodException,
- SecurityException {
- doubleWriters.put(path, writer);
- parser.addParseTarget(
- ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, Double.class }),
- path);
- }
-
- private void add(Parser<ComplexWriterFacade> parser, String path, BigIntWriter writer)
- throws NoSuchMethodException,
- SecurityException {
- longWriters.put(path, writer);
- parser.addParseTarget(
- ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, Long.class }),
- path);
- }
-
- public void addAsParseTarget(Parser<ComplexWriterFacade> parser) {
- try {
-
- for (final String path : parser.getPossiblePaths()) {
- switch (path) {
- case "IP:connection.client.ip":
- add(parser, path, writer.rootAsMap().map("client").varChar("ip"));
- break;
- case "IP:connection.client.peerip":
- add(parser, path, writer.rootAsMap().map("client").varChar("peer_ip"));
- break;
- case "IP:connection.server.ip":
- add(parser, path, writer.rootAsMap().map("server").varChar("ip"));
- break;
- case "BYTES:response.body.bytes":
- add(parser, path, writer.rootAsMap().map("response").bigInt("bytes"));
- break;
- case "BYTES:response.body.bytesclf":
- add(parser, path, writer.rootAsMap().map("response").bigInt("bytes"));
- break;
- case "HTTP.COOKIE:request.cookies.":
- add(parser, path, writer.rootAsMap().map("request").varChar("cookies"));
- break;
- case "MICROSECONDS:server.process.time":
- add(parser, path, writer.rootAsMap().map("response").bigInt("process_time"));
- break;
- case "FILENAME:server.filename":
- add(parser, path, writer.rootAsMap().map("response").varChar("filename"));
- break;
- case "IP:connection.client.host":
- add(parser, path, writer.rootAsMap().map("client").varChar("host"));
- break;
- case "PROTOCOL:request.protocol":
- add(parser, path, writer.rootAsMap().map("request").varChar("protocol"));
- break;
- case "HTTP.HEADER:request.header.":
- add(parser, path, writer.rootAsMap().map("request").varChar("header"));
- break;
- case "NUMBER:connection.keepalivecount":
- add(parser, path, writer.rootAsMap().map("client").bigInt("keepalivecount"));
- break;
- case "NUMBER:connection.client.logname":
- add(parser, path, writer.rootAsMap().map("request").bigInt("logname"));
- break;
- case "STRING:request.errorlogid":
- add(parser, path, writer.rootAsMap().map("request").varChar("errorlogid"));
- break;
- case "HTTP.METHOD:request.method":
- add(parser, path, writer.rootAsMap().map("request").varChar("method"));
- break;
- case "PORT:request.server.port.canonical":
- add(parser, path, writer.rootAsMap().map("server").bigInt("canonical_port"));
- break;
- case "PORT:connection.server.port.canonical":
- add(parser, path, writer.rootAsMap().map("server").bigInt("canonical_port"));
- break;
- case "PORT:connection.client.port":
- add(parser, path, writer.rootAsMap().map("client").bigInt("port"));
- break;
- case "NUBMER:connection.server.child.processid":
- add(parser, path, writer.rootAsMap().map("server").bigInt("process_id"));
- break;
- case "NUMBER:connection.server.child.threadid":
- add(parser, path, writer.rootAsMap().map("server").bigInt("thread_id"));
- break;
- case "STRING:connection.server.child.hexthreadid":
- add(parser, path, writer.rootAsMap().map("connection").varChar("hex_thread_id"));
- break;
- case "HTTP.QUERYSTRING:request.querystring":
- add(parser, path, writer.rootAsMap().map("").varChar(""));
- break;
- case "HTTP.FIRSTLINE:request.firstline":
- add(parser, path, writer.rootAsMap().map("").varChar(""));
- break;
- case "STRING:request.handler":
- add(parser, path, writer.rootAsMap().map("request").varChar("handler"));
- break;
- case "STRING:request.status.original":
- add(parser, path, writer.rootAsMap().map("request").varChar("status_original"));
- break;
- case "STRING:request.status.last":
- add(parser, path, writer.rootAsMap().map("request").varChar("status_last"));
- break;
- case "TIME.STAMP:request.receive.time":
- add(parser, path, writer.rootAsMap().map("request").varChar("timestamp"));
- break;
- case "TIME.EPOCH:request.receive.time.begin.msec":
- add(parser, path, writer.rootAsMap().map("request").bigInt("begin_msec"));
- break;
- case "TIME.EPOCH:request.receive.time.end.msec":
- add(parser, path, writer.rootAsMap().map("request").bigInt("end_msec"));
- break;
- case "TIME.EPOCH.USEC:request.receive.time.begin.usec":
- add(parser, path, writer.rootAsMap().map("request").bigInt("begin_usec"));
- break;
- case "TIME.EPOCH.USEC:request.receive.time.end.usec":
- add(parser, path, writer.rootAsMap().map("request").bigInt("end_usec"));
- break;
- case "TIME.EPOCH:request.receive.time.begin.msec_frac":
- add(parser, path, writer.rootAsMap().map("request").bigInt("begin_msec_frac"));
- break;
- case "TIME.EPOCH:request.receive.time.end.msec_frac":
- add(parser, path, writer.rootAsMap().map("request").varChar("end_msec_frac"));
- break;
- case "TIME.EPOCH.USEC_FRAC:request.receive.time.begin.usec_frac":
- add(parser, path, writer.rootAsMap().map("request").varChar("begin_usec_frac"));
- break;
- case "TIME.EPOCH.USEC_FRAC:request.receive.time.end.usec_frac":
- add(parser, path, writer.rootAsMap().map("request").varChar("end_usec_frac"));
- break;
- case "SECONDS:response.server.processing.time":
- add(parser, path, writer.rootAsMap().map("response").varChar("processing_time"));
- break;
- case "STRING:connection.client.user":
- add(parser, path, writer.rootAsMap().map("client").varChar("user"));
- break;
- case "URI:request.urlpath":
- add(parser, path, writer.rootAsMap().map("request").varChar("url"));
- break;
- case "STRING:connection.server.name.canonical":
- add(parser, path, writer.rootAsMap().map("server").varChar("canonical_name"));
- break;
- case "STRING:connection.server.name":
- add(parser, path, writer.rootAsMap().map("server").varChar("name"));
- break;
- case "HTTP.CONNECTSTATUS:response.connection.status":
- add(parser, path, writer.rootAsMap().map("response").varChar("connection_status"));
- break;
- case "BYTES:request.bytes":
- add(parser, path, writer.rootAsMap().map("request").varChar("bytes"));
- break;
- case "BYTES:response.bytes":
- add(parser, path, writer.rootAsMap().map("response").bigInt("bytes"));
- break;
- case "HTTP.COOKIES:request.cookies":
- add(parser, path, writer.rootAsMap().map("request").varChar("cookies"));
- break;
- case "HTTP.SETCOOKIES:response.cookies":
- add(parser, path, writer.rootAsMap().map("response").varChar("cookies"));
- break;
- case "HTTP.USERAGENT:request.user-agent":
- add(parser, path, writer.rootAsMap().map("request").varChar("useragent"));
- break;
- case "HTTP.URI:request.referer":
- add(parser, path, writer.rootAsMap().map("request").varChar("referer"));
- break;
- case "HTTP.METHOD:method":
- add(parser, path, writer.rootAsMap().map("request").varChar("method"));
- break;
- case "HTTP.URI:uri":
- add(parser, path, writer.rootAsMap().map("request").varChar("uri"));
- break;
- case "HTTP.PROTOCOL:protocol":
- add(parser, path, writer.rootAsMap().map("request").varChar("protocol"));
- break;
- case "HTTP.PROTOCOL.VERSION:protocol.version":
- add(parser, path, writer.rootAsMap().map("request").varChar("protocol_version"));
- break;
- case "HTTP.METHOD:request.firstline.method":
- add(parser, path, writer.rootAsMap().map("request").varChar("method"));
- break;
- case "HTTP.URI:request.firstline.uri":
- add(parser, path, writer.rootAsMap().map("request").varChar("uri"));
- break;
- case "HTTP.PROTOCOL:request.firstline.protocol":
- add(parser, path, writer.rootAsMap().map("request").varChar("protocol"));
- break;
- case "HTTP.PROTOCOL.VERSION:request.firstline.protocol.version":
- add(parser, path, writer.rootAsMap().map("request").varChar("protocol_version"));
- break;
- default:
-
- // if we don't know what to do, just write the raw value.
- parser.addParseTarget(
- ComplexWriterFacade.class.getMethod("set", new Class[] { String.class, String.class }),
- path);
- final String noPeriodPath = path.replace(".", "_");
- stringWriters.put(path, writer.rootAsMap().varChar(noPeriodPath));
- break;
-
- }
- }
-
-
- } catch (MissingDissectorsException | SecurityException | NoSuchMethodException | InvalidDissectorException e) {
- throw handleAndGenerate("Failure while setting up log mappings.", e);
- }
- }
- }
- }
-
- @Override
- public boolean supportsPushDown() {
- return true;
- }
-
-
- @Override
- public RecordReader getRecordReader(FragmentContext context, DrillFileSystem dfs,
- FileWork fileWork, List<SchemaPath> columns) throws ExecutionSetupException {
- return new RecordReader(context, dfs, fileWork);
- }
-
- @Override
- public RecordWriter getRecordWriter(FragmentContext context, EasyWriter writer) throws IOException {
- throw new UnsupportedOperationException("Drill doesn't currently support writing to HTTPD logs.");
- }
-
- @Override
- public int getReaderOperatorType() {
- return -1;
- }
-
- @Override
- public int getWriterOperatorType() {
- return -1;
- }
-
- private class PartiallyDissectedParser<RECORD> extends Parser<RECORD> {
- public PartiallyDissectedParser(Class<RECORD> clazz, final String logformat) {
- super(clazz);
-
- addDissector(new ApacheHttpdLogFormatDissector(logformat));
- addDissector(new HttpFirstLineDissector());
- setRootType(ApacheHttpdLogFormatDissector.INPUT_TYPE);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
index 7a83d45..a6dc7d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
* file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
@@ -243,4 +244,4 @@ public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.
public int getWriterOperatorType() {
return -1;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
index 03f70c1..2775285 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
@@ -296,4 +296,4 @@ public class HttpdLogRecord {
strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
index 114a7f4..a896638 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
@@ -18,9 +18,6 @@ package org.apache.drill.exec.store.httpd;
import com.google.common.base.Preconditions;
import com.google.common.collect.Maps;
import io.netty.buffer.DrillBuf;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
import nl.basjes.parse.core.Casts;
import nl.basjes.parse.core.Parser;
import nl.basjes.parse.core.exceptions.DissectionFailure;
@@ -31,6 +28,11 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
public class HttpdParser {
private static final Logger LOG = LoggerFactory.getLogger(HttpdParser.class);
@@ -41,6 +43,105 @@ public class HttpdParser {
private final Parser<HttpdLogRecord> parser;
private final HttpdLogRecord record;
+ public static final HashMap<String, String> LOGFIELDS = new HashMap<String, String>();
+ static
+ {
+ LOGFIELDS.put("request_receive_time_weekyear__utc", "TIME_YEAR:request_receive_time_weekyear__utc");
+ LOGFIELDS.put("request_referer_ref", "HTTP_REF:request_referer_ref");
+ LOGFIELDS.put("request_referer_protocol", "HTTP_PROTOCOL:request_referer_protocol");
+ LOGFIELDS.put("request_receive_time_timezone", "TIME_ZONE:request_receive_time_timezone");
+ LOGFIELDS.put("connection_client_host", "IP:connection_client_host");
+ LOGFIELDS.put("connection_client_ip", "IP:connection_client_ip");
+ LOGFIELDS.put("connection_client_peerip", "IP:connection_client_peerip");
+ LOGFIELDS.put("connection_server_ip", "IP:connection_server_ip");
+ LOGFIELDS.put("request_receive_time_day", "TIME_DAY:request_receive_time_day");
+ LOGFIELDS.put("request_receive_time_minute__utc", "TIME_MINUTE:request_receive_time_minute__utc");
+ LOGFIELDS.put("request_referer_query_$", "STRING:request_referer_query_$");
+ LOGFIELDS.put("request_receive_time_millisecond__utc", "TIME_MILLISECOND:request_receive_time_millisecond__utc");
+ LOGFIELDS.put("request_firstline_uri_port", "HTTP_PORT:request_firstline_uri_port");
+ LOGFIELDS.put("request_referer_userinfo", "HTTP_USERINFO:request_referer_userinfo");
+ LOGFIELDS.put("request_receive_time_second__utc", "TIME_SECOND:request_receive_time_second__utc");
+ LOGFIELDS.put("request_firstline_uri_protocol", "HTTP_PROTOCOL:request_firstline_uri_protocol");
+ LOGFIELDS.put("request_receive_time_month", "TIME_MONTH:request_receive_time_month");
+ LOGFIELDS.put("request_firstline_uri_query", "HTTP_QUERYSTRING:request_firstline_uri_query");
+ LOGFIELDS.put("request_firstline_uri_path", "HTTP_PATH:request_firstline_uri_path");
+ LOGFIELDS.put("request_receive_time_hour__utc", "TIME_HOUR:request_receive_time_hour__utc");
+ LOGFIELDS.put("request_receive_time_monthname", "TIME_MONTHNAME:request_receive_time_monthname");
+ LOGFIELDS.put("request_receive_time_year__utc", "TIME_YEAR:request_receive_time_year__utc");
+ LOGFIELDS.put("request_receive_time_second", "TIME_SECOND:request_receive_time_second");
+ LOGFIELDS.put("request_referer", "HTTP_URI:request_referer");
+ LOGFIELDS.put("request_receive_time_monthname__utc", "TIME_MONTHNAME:request_receive_time_monthname__utc");
+ LOGFIELDS.put("request_referer_path", "HTTP_PATH:request_referer_path");
+ LOGFIELDS.put("request_receive_time_weekyear", "TIME_YEAR:request_receive_time_weekyear");
+ LOGFIELDS.put("request_firstline_protocol", "HTTP_PROTOCOL:request_firstline_protocol");
+ LOGFIELDS.put("request_referer_port", "HTTP_PORT:request_referer_port");
+ LOGFIELDS.put("request_receive_time_minute", "TIME_MINUTE:request_receive_time_minute");
+ LOGFIELDS.put("request_status_last", "STRING:request_status_last");
+ LOGFIELDS.put("request_receive_time_hour", "TIME_HOUR:request_receive_time_hour");
+ LOGFIELDS.put("request_firstline_protocol_version", "HTTP_PROTOCOL_VERSION:request_firstline_protocol_version");
+ LOGFIELDS.put("request_receive_time", "TIME_STAMP:request_receive_time");
+ LOGFIELDS.put("request_firstline_method", "HTTP_METHOD:request_firstline_method");
+ LOGFIELDS.put("request_receive_time_epoch", "TIME_EPOCH:request_receive_time_epoch");
+ LOGFIELDS.put("request_receive_time_weekofweekyear", "TIME_WEEK:request_receive_time_weekofweekyear");
+ LOGFIELDS.put("request_firstline_uri_host", "HTTP_HOST:request_firstline_uri_host");
+ LOGFIELDS.put("request_referer_query", "HTTP_QUERYSTRING:request_referer_query");
+ LOGFIELDS.put("request_firstline_uri_userinfo", "HTTP_USERINFO:request_firstline_uri_userinfo");
+ LOGFIELDS.put("response_body_bytes", "BYTES:response_body_bytes");
+ LOGFIELDS.put("response_body_bytesclf", "BYTES:response_body_bytesclf");
+ LOGFIELDS.put("request_referer_host", "HTTP_HOST:request_referer_host");
+ LOGFIELDS.put("request_receive_time_weekofweekyear__utc", "TIME_WEEK:request_receive_time_weekofweekyear__utc");
+ LOGFIELDS.put("request_firstline_uri", "HTTP_URI:request_firstline_uri");
+ LOGFIELDS.put("request_firstline_uri_ref", "HTTP_REF:request_firstline_uri_ref");
+ LOGFIELDS.put("request_receive_time_year", "TIME_YEAR:request_receive_time_year");
+ LOGFIELDS.put("request_firstline", "HTTP_FIRSTLINE:request_firstline");
+ LOGFIELDS.put("request_user-agent", "HTTP_USERAGENT:request_user-agent");
+ LOGFIELDS.put("request_cookies", "HTTP_COOKIE:request_cookies");
+ LOGFIELDS.put("server_process_time", "MICROSECONDS:server_process_time");
+ LOGFIELDS.put("request_cookies_$", "HTTP_COOKIE:request_cookies_$");
+ LOGFIELDS.put("server_environment_$", "VARIABLE:server_environment_$");
+ LOGFIELDS.put("server_filename", "FILENAME:server_filename");
+ LOGFIELDS.put("request_protocol", "PROTOCOL:request_protocol");
+ LOGFIELDS.put("request_header_", "HTTP_HEADER:request_header_");
+ LOGFIELDS.put("connection_keepalivecount", "NUMBER:connection_keepalivecount");
+ LOGFIELDS.put("connection_client_logname", "NUMBER:connection_client_logname");
+ LOGFIELDS.put("request_errorlogid", "STRING:request_errorlogid");
+ LOGFIELDS.put("request_method", "HTTP_METHOD:request_method");
+ LOGFIELDS.put("server_module_note_$", "STRING:server_module_note_$");
+ LOGFIELDS.put("response_header_$", "HTTP_HEADER:response_header_$");
+ LOGFIELDS.put("request_server_port_canonical", "PORT:request_server_port_canonical");
+ LOGFIELDS.put("connection_server_port_canonical", "PORT:connection_server_port_canonical");
+ LOGFIELDS.put("connection_server_port", "PORT:connection_server_port");
+ LOGFIELDS.put("connection_client_port", "PORT:connection_client_port");
+ LOGFIELDS.put("connection_server_child_processid", "NUMBER:connection_server_child_processid");
+ LOGFIELDS.put("connection_server_child_threadid", "NUMBER:connection_server_child_threadid");
+ LOGFIELDS.put("connection_server_child_hexthreadid", "NUMBER:connection_server_child_hexthreadid");
+ LOGFIELDS.put("request_querystring", "HTTP_QUERYSTRING:request_querystring");
+ LOGFIELDS.put("request_handler", "STRING:request_handler");
+ LOGFIELDS.put("request_status_original", "STRING:request_status_original");
+ LOGFIELDS.put("request_status_last", "STRING:request_status_last");
+ LOGFIELDS.put("request_receive_time_begin_msec", "TIME_EPOCH:request_receive_time_begin_msec");
+ LOGFIELDS.put("request_receive_time_end_msec", "TIME_EPOCH:request_receive_time_end_msec");
+ LOGFIELDS.put("request_receive_time_begin_usec", "TIME_EPOCH_USEC:request_receive_time_begin_usec");
+ LOGFIELDS.put("request_receive_time_begin_usec", "TIME_EPOCH_USEC:request_receive_time_begin_usec");
+ LOGFIELDS.put("request_receive_time_end_usec", "TIME_EPOCH_USEC:request_receive_time_end_usec");
+ LOGFIELDS.put("request_receive_time_begin_msec_frac", "TIME_EPOCH:request_receive_time_begin_msec_frac");
+ LOGFIELDS.put("request_receive_time_begin_msec_frac", "TIME_EPOCH:request_receive_time_begin_msec_frac");
+ LOGFIELDS.put("request_receive_time_end_msec_frac", "TIME_EPOCH:request_receive_time_end_msec_frac");
+ LOGFIELDS.put("request_receive_time_begin_usec_frac", "TIME_EPOCH_USEC_FRAC:request_receive_time_begin_usec_frac");
+ LOGFIELDS.put("request_receive_time_begin_usec_frac", "TIME_EPOCH_USEC_FRAC:request.receive.time.begin.usec_frac");
+ LOGFIELDS.put("request_receive_time_end_usec_frac", "TIME_EPOCH_USEC_FRAC:request_receive_time_end_usec_frac");
+ LOGFIELDS.put("response_server_processing_time", "SECONDS:response_server_processing_time");
+ LOGFIELDS.put("connection_client_user", "STRING:connection_client_user");
+ LOGFIELDS.put("request_urlpath", "URI:request_urlpath");
+ LOGFIELDS.put("connection_server_name_canonical", "STRING:connection_server_name_canonical");
+ LOGFIELDS.put("connection_server_name", "STRING:connection_server_name");
+ LOGFIELDS.put("response_connection_status", "HTTP_CONNECTSTATUS:response_connection_status");
+ LOGFIELDS.put("request_bytes", "BYTES:request_bytes");
+ LOGFIELDS.put("response_bytes", "BYTES:response_bytes");
+ }
+
+ //Map map = Collections.synchronizedMap(LOGFIELDS);
+
public HttpdParser(final MapWriter mapWriter, final DrillBuf managedBuffer, final String logFormat,
final String timestampFormat, final Map<String, String> fieldMapping)
throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
@@ -96,7 +197,10 @@ public class HttpdParser {
* @return
*/
public static String parserFormattedFieldName(final String drillFieldName) {
- return drillFieldName.replace(SAFE_WILDCARD, PARSER_WILDCARD).replaceAll(SAFE_SEPARATOR, ".").replaceAll("\\.\\.", "_");
+ String tempFieldName;
+ tempFieldName = LOGFIELDS.get(drillFieldName);
+ return tempFieldName.replace(SAFE_WILDCARD, PARSER_WILDCARD).replaceAll(SAFE_SEPARATOR, ".").replaceAll("\\.\\.", "_");
+
}
/**
@@ -108,7 +212,14 @@ public class HttpdParser {
* @return
*/
public static String drillFormattedFieldName(final String parserFieldName) {
- return parserFieldName.replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR);
+
+ if (parserFieldName.contains(":") ) {
+ String[] fieldPart= parserFieldName.split(":");
+ return fieldPart[1].replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR);
+ }
+ else{
+ return parserFieldName.replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR);
+ }
}
private void setupParser(final MapWriter mapWriter, final String logFormat, final Map<String, String> fieldMapping)
@@ -168,4 +279,4 @@ public class HttpdParser {
record.addField(parser, mapWriter, casts, entry.getValue(), entry.getKey());
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
new file mode 100644
index 0000000..b82b1ee
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.httpd;
+
+import io.netty.buffer.DrillBuf;
+import java.util.Map;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpdParserTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HttpdParserTest.class);
+
+ private void runTest(String logFormat, String logLine) throws Exception {
+ MapWriter mapWriter = null;
+ DrillBuf managedBuffer = null;
+ Map<String, String> configuredFields = null;
+ HttpdParser parser = new HttpdParser(mapWriter, managedBuffer, logFormat, null, configuredFields);
+ parser.parse(logLine);
+ }
+
+// @Test
+ public void testFirstPattern() throws Exception {
+ LOG.info("testFirstPattern");
+// final String format = "common";
+// final String format = "%h %l %u %t \"%r\" %>s %b";
+ final String format = "%h %t \"%r\" %>s %b \"%{Referer}i\"";
+ final String line = "127.0.0.1 [31/Dec/2012:23:49:41 +0100] "
+ + "\"GET /foo HTTP/1.1\" 200 "
+ + "1213 \"http://localhost/index.php?mies=wim\"";
+ runTest(format, line);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
index c341295..2cbc09a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java
@@ -61,6 +61,9 @@ public class TestFormatPluginOptionExtractor {
case "avro":
assertEquals(d.typeName, "(type: String)", d.presentParams());
break;
+ case "httpd":
+ assertEquals("(type: String, logFormat: String, timestampFormat: String)", d.presentParams());
+ break;
default:
fail("add validation for format plugin type " + d.typeName);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
deleted file mode 100644
index 961d9a6..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2015 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import io.netty.buffer.DrillBuf;
-import java.util.Map;
-import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HttpdParserTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(HttpdParserTest.class);
-
- private void runTest(String logFormat, String logLine) throws Exception {
- MapWriter mapWriter = null;
- DrillBuf managedBuffer = null;
- Map<String, String> configuredFields = null;
- HttpdParser parser = new HttpdParser(mapWriter, managedBuffer, logFormat, null, configuredFields);
- parser.parse(logLine);
- }
-
-// @Test
- public void testFirstPattern() throws Exception {
- LOG.info("testFirstPattern");
-// final String format = "common";
-// final String format = "%h %l %u %t \"%r\" %>s %b";
- final String format = "%h %t \"%r\" %>s %b \"%{Referer}i\"";
- final String line = "127.0.0.1 [31/Dec/2012:23:49:41 +0100] "
- + "\"GET /foo HTTP/1.1\" 200 "
- + "1213 \"http://localhost/index.php?mies=wim\"";
- runTest(format, line);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
deleted file mode 100644
index 6c2eaf8..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
- * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
- * License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import java.util.List;
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import static org.junit.Assert.*;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestHttpdLogFormatPlugin extends BaseTestQuery {
-
- private static final Logger LOG = LoggerFactory.getLogger(TestHttpdLogFormatPlugin.class);
-
- /**
- * This test covers the test bootstrap-storage-plugins.json section of httpd.
- *
- * Indirectly this validates the HttpdLogFormatPlugin.HttpdLogFormatConfig deserializing properly.
- *
- * @throws Exception
- */
- @Test
- public void testDfsTestBootstrap_star() throws Exception {
- test("select * from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`");
- }
-
- /**
- * This test covers the test bootstrap-storage-plugins.json section of httpd.
- *
- * Indirectly this validates the HttpdLogFormatPlugin.HttpdLogFormatConfig deserializing properly.
- *
- * @throws Exception
- */
- @Test
- public void testDfsTestBootstrap_notstar() throws Exception {
- test("select `TIME_STAMP:request_receive_time`, `HTTP_METHOD:request_firstline_method`, `STRING:request_status_last`, `BYTES:response_body_bytesclf` \n"
- + "from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`");
- }
-
- /**
- * This test covers the main bootstrap-storage-plugins.json section of httpd.
- *
- * @throws Exception
- */
- @Test
- public void testDfsBootstrap_star() throws Exception {
- test("select * from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
- }
-
- /**
- * This test covers the main bootstrap-storage-plugins.json section of httpd.
- *
- * @throws Exception
- */
- @Test
- public void testDfsBootstrap_wildcard() throws Exception {
- test("select `STRING:request_referer_query_$` from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
- }
-
- /**
- * This test covers the main bootstrap-storage-plugins.json section of httpd.
- *
- * @throws Exception
- */
- @Test
- public void testDfsBootstrap_underscore() throws Exception {
- test("select `TIME_DAY:request_receive_time_day__utc` from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
- }
-
- @Test
- public void testGroupBy_1() throws Exception {
- final List<QueryDataBatch> actualResults = testSqlWithResults(
- "select `HTTP_METHOD:request_firstline_method` as http_method, `STRING:request_status_last` as status_code, sum(`BYTES:response_body_bytesclf`) as total_bytes \n"
- + "from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`\n"
- + "group by `HTTP_METHOD:request_firstline_method`, `STRING:request_status_last`"
- );
-
- final TestResultSet expectedResultSet = new TestResultSet();
- expectedResultSet.addRow("GET", "200", "46551");
- expectedResultSet.addRow("POST", "302", "18186");
-
- TestResultSet actualResultSet = new TestResultSet(actualResults);
- assertTrue(expectedResultSet.equals(actualResultSet));
- }
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4a82bc13/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java
deleted file mode 100644
index ce1f685..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdPlugin.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.httpd;
-
-import org.apache.drill.BaseTestQuery;
-import org.junit.Test;
-
-public class TestHttpdPlugin extends BaseTestQuery {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHttpdPlugin.class);
-
- @Test
- public void tryBasicQuery() throws Exception {
- // test("select * from cp.`store/httpd/example1.httpd`");
- test("select * from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/example1.httpd`");
- }
-}