You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2016/11/01 20:29:47 UTC
[02/15] drill git commit: DRILL-3423: Adding HTTPd Log Parsing
functionality including full pushdown,
type remapping and wildcard support. Pushed through the requested columns for
push down to the parser. Added more tests to cover a few more use cases.
DRILL-3423: Adding HTTPd Log Parsing functionality including full pushdown, type remapping and wildcard support.
Pushed through the requested columns for push down to the parser. Added more tests to cover a few more use cases. Ensured that user query fields are now completely consistent with returned values.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/46c0f2a4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/46c0f2a4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/46c0f2a4
Branch: refs/heads/master
Commit: 46c0f2a4135450417dfebf52f11538f8926fd467
Parents: 818f945
Author: Jim Scott <ji...@13ways.com>
Authored: Tue Nov 3 15:43:54 2015 -0600
Committer: Parth Chandra <pa...@apache.org>
Committed: Tue Nov 1 10:42:22 2016 -0700
----------------------------------------------------------------------
exec/java-exec/pom.xml | 3 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 1 +
.../exec/store/httpd/HttpdLogFormatPlugin.java | 246 +++++++++++++++
.../drill/exec/store/httpd/HttpdLogRecord.java | 299 +++++++++++++++++++
.../drill/exec/store/httpd/HttpdParser.java | 171 +++++++++++
.../resources/bootstrap-storage-plugins.json | 5 +-
.../java/org/apache/drill/BaseTestQuery.java | 76 ++++-
.../drill/exec/store/httpd/HttpdParserTest.java | 48 +++
.../store/httpd/TestHttpdLogFormatPlugin.java | 97 ++++++
.../drill/exec/store/text/TestTextColumn.java | 103 ++-----
.../resources/bootstrap-storage-plugins.json | 4 +-
.../resources/store/httpd/dfs-bootstrap.httpd | 2 +
.../store/httpd/dfs-test-bootstrap-test.httpd | 5 +
pom.xml | 1 +
14 files changed, 976 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index e9b07be..6bf77c0 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -456,9 +456,8 @@
<dependency>
<groupId>nl.basjes.parse.httpdlog</groupId>
<artifactId>httpdlog-parser</artifactId>
- <version>2.1.1</version>
+ <version>2.4</version>
</dependency>
-
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index c7f9cf8..c09e009 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -117,6 +117,7 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
/** Method indicates whether or not this format could also be in a compression container (for example: csv.gz versus csv).
* If this format uses its own internal compression scheme, such as Parquet does, then this should return false.
+ * @return <code>true</code> if it is compressible
*/
public boolean isCompressible() {
return compressible;
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
new file mode 100644
index 0000000..7a83d45
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogFormatPlugin.java
@@ -0,0 +1,246 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.drill.exec.store.httpd;
+
+import java.io.IOException;
+import java.util.List;
+
+import nl.basjes.parse.core.exceptions.DissectionFailure;
+import nl.basjes.parse.core.exceptions.InvalidDissectorException;
+import nl.basjes.parse.core.exceptions.MissingDissectorsException;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.exceptions.UserException;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.FormatPluginConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.OperatorContext;
+import org.apache.drill.exec.physical.impl.OutputMutator;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.AbstractRecordReader;
+import org.apache.drill.exec.store.RecordWriter;
+import org.apache.drill.exec.store.dfs.DrillFileSystem;
+import org.apache.drill.exec.store.dfs.easy.EasyFormatPlugin;
+import org.apache.drill.exec.store.dfs.easy.EasyWriter;
+import org.apache.drill.exec.store.dfs.easy.FileWork;
+import org.apache.drill.exec.vector.complex.impl.VectorContainerWriter;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.LineRecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.TextInputFormat;
+
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import java.util.Map;
+import org.apache.drill.exec.store.RecordReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpdLogFormatPlugin extends EasyFormatPlugin<HttpdLogFormatPlugin.HttpdLogFormatConfig> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HttpdLogFormatPlugin.class);
+ private static final String PLUGIN_EXTENSION = "httpd";
+ private static final int VECTOR_MEMORY_ALLOCATION = 4095;
+
+ public HttpdLogFormatPlugin(final String name, final DrillbitContext context, final Configuration fsConf,
+ final StoragePluginConfig storageConfig, final HttpdLogFormatConfig formatConfig) {
+
+ super(name, context, fsConf, storageConfig, formatConfig, true, false, true, true,
+ Lists.newArrayList(PLUGIN_EXTENSION), PLUGIN_EXTENSION);
+ }
+
+ /**
+ * This class is a POJO to hold the configuration for the HttpdLogFormat Parser. This is automatically
+ * serialized/deserialized from JSON format.
+ */
+ @JsonTypeName(PLUGIN_EXTENSION)
+ public static class HttpdLogFormatConfig implements FormatPluginConfig {
+
+ private String logFormat;
+ private String timestampFormat;
+
+ /**
+ * @return the logFormat
+ */
+ public String getLogFormat() {
+ return logFormat;
+ }
+
+ /**
+ * @return the timestampFormat
+ */
+ public String getTimestampFormat() {
+ return timestampFormat;
+ }
+ }
+
+ /**
+ * This class performs the work for the plugin. This is where all logic goes to read records. In this case httpd logs
+ * are lines terminated with a new line character.
+ */
+ private class HttpdLogRecordReader extends AbstractRecordReader {
+
+ private final DrillFileSystem fs;
+ private final FileWork work;
+ private final FragmentContext fragmentContext;
+ private ComplexWriter writer;
+ private HttpdParser parser;
+ private LineRecordReader lineReader;
+ private LongWritable lineNumber;
+
+ public HttpdLogRecordReader(final FragmentContext context, final DrillFileSystem fs, final FileWork work, final List<SchemaPath> columns) {
+ this.fs = fs;
+ this.work = work;
+ this.fragmentContext = context;
+ setColumns(columns);
+ }
+
+ /**
+ * The query fields passed in are formatted in a way that Drill requires. Those must be cleaned up to work with the
+ * parser.
+ *
+ * @return Map<DrillFieldNames, ParserFieldNames>
+ */
+ private Map<String, String> makeParserFields() {
+ final Map<String, String> fieldMapping = Maps.newHashMap();
+ for (final SchemaPath sp : getColumns()) {
+ final String drillField = sp.getRootSegment().getPath();
+ final String parserField = HttpdParser.parserFormattedFieldName(drillField);
+ fieldMapping.put(drillField, parserField);
+ }
+ return fieldMapping;
+ }
+
+ @Override
+ public void setup(final OperatorContext context, final OutputMutator output) throws ExecutionSetupException {
+ try {
+ /**
+ * Extract the list of field names for the parser to use if it is NOT a star query. If it is a star query just
+ * pass through an empty map, because the parser is going to have to build all possibilities.
+ */
+ final Map<String, String> fieldMapping = !isStarQuery() ? makeParserFields() : null;
+ writer = new VectorContainerWriter(output);
+ parser = new HttpdParser(writer.rootAsMap(), context.getManagedBuffer(),
+ HttpdLogFormatPlugin.this.getConfig().getLogFormat(),
+ HttpdLogFormatPlugin.this.getConfig().getTimestampFormat(),
+ fieldMapping);
+
+ final Path path = fs.makeQualified(new Path(work.getPath()));
+ FileSplit split = new FileSplit(path, work.getStart(), work.getLength(), new String[]{""});
+ TextInputFormat inputFormat = new TextInputFormat();
+ JobConf job = new JobConf(fs.getConf());
+ job.setInt("io.file.buffer.size", fragmentContext.getConfig().getInt(ExecConstants.TEXT_LINE_READER_BUFFER_SIZE));
+ job.setInputFormat(inputFormat.getClass());
+ lineReader = (LineRecordReader) inputFormat.getRecordReader(split, job, Reporter.NULL);
+ lineNumber = lineReader.createKey();
+ }
+ catch (NoSuchMethodException | MissingDissectorsException | InvalidDissectorException e) {
+ throw handleAndGenerate("Failure creating HttpdParser", e);
+ }
+ catch (IOException e) {
+ throw handleAndGenerate("Failure creating HttpdRecordReader", e);
+ }
+ }
+
+ private RuntimeException handleAndGenerate(final String s, final Exception e) {
+ throw UserException.dataReadError(e)
+ .message(s + "\n%s", e.getMessage())
+ .addContext("Path", work.getPath())
+ .addContext("Split Start", work.getStart())
+ .addContext("Split Length", work.getLength())
+ .addContext("Local Line Number", lineNumber.get())
+ .build(LOG);
+ }
+
+ /**
+ * This record reader is given a batch of records (lines) to read. Next acts upon a batch of records.
+ *
+ * @return Number of records in this batch.
+ */
+ @Override
+ public int next() {
+ try {
+ final Text line = lineReader.createValue();
+
+ writer.allocate();
+ writer.reset();
+
+ int recordCount = 0;
+ while (recordCount < VECTOR_MEMORY_ALLOCATION && lineReader.next(lineNumber, line)) {
+ writer.setPosition(recordCount);
+ parser.parse(line.toString());
+ recordCount++;
+ }
+ writer.setValueCount(recordCount);
+
+ return recordCount;
+ }
+ catch (DissectionFailure | InvalidDissectorException | MissingDissectorsException | IOException e) {
+ throw handleAndGenerate("Failure while parsing log record.", e);
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ try {
+ if (lineReader != null) {
+ lineReader.close();
+ }
+ }
+ catch (IOException e) {
+ LOG.warn("Failure while closing Httpd reader.", e);
+ }
+ }
+
+ }
+
+ /**
+ * This plugin supports pushing down into the parser. Only fields specifically asked for within the configuration will
+ * be parsed. If no fields are asked for then all possible fields will be returned.
+ *
+ * @return true
+ */
+ @Override
+ public boolean supportsPushDown() {
+ return true;
+ }
+
+ @Override
+ public RecordReader getRecordReader(final FragmentContext context, final DrillFileSystem dfs, final FileWork fileWork, final List<SchemaPath> columns, final String userName) throws ExecutionSetupException {
+ return new HttpdLogRecordReader(context, dfs, fileWork, columns);
+ }
+
+ @Override
+ public RecordWriter getRecordWriter(final FragmentContext context, final EasyWriter writer) throws IOException {
+ throw new UnsupportedOperationException("Drill doesn't currently support writing HTTPd logs");
+ }
+
+ @Override
+ public int getReaderOperatorType() {
+ return -1;
+ }
+
+ @Override
+ public int getWriterOperatorType() {
+ return -1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
new file mode 100644
index 0000000..03f70c1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdLogRecord.java
@@ -0,0 +1,299 @@
+/*
+ * Copyright 2015 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.httpd;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Maps;
+import io.netty.buffer.DrillBuf;
+import java.util.EnumSet;
+import java.util.Map;
+import nl.basjes.parse.core.Casts;
+import nl.basjes.parse.core.Parser;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.drill.exec.vector.complex.writer.BigIntWriter;
+import org.apache.drill.exec.vector.complex.writer.Float8Writer;
+import org.apache.drill.exec.vector.complex.writer.VarCharWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpdLogRecord {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HttpdLogRecord.class);
+ private final Map<String, VarCharWriter> strings = Maps.newHashMap();
+ private final Map<String, BigIntWriter> longs = Maps.newHashMap();
+ private final Map<String, Float8Writer> doubles = Maps.newHashMap();
+ private final Map<String, MapWriter> wildcards = Maps.newHashMap();
+ private final Map<String, String> cleanExtensions = Maps.newHashMap();
+ private final Map<String, MapWriter> startedWildcards = Maps.newHashMap();
+ private final Map<String, MapWriter> wildcardWriters = Maps.newHashMap();
+ private DrillBuf managedBuffer;
+
+ public HttpdLogRecord(final DrillBuf managedBuffer) {
+ this.managedBuffer = managedBuffer;
+ }
+
+ /**
+ * Call this method after a record has been parsed. This finished the lifecycle of any maps that were written and
+ * removes all the entries for the next record to be able to work.
+ */
+ public void finishRecord() {
+ for (MapWriter writer : wildcardWriters.values()) {
+ writer.end();
+ }
+ wildcardWriters.clear();
+ startedWildcards.clear();
+ }
+
+ private DrillBuf buf(final int size) {
+ if (managedBuffer.capacity() < size) {
+ managedBuffer = managedBuffer.reallocIfNeeded(size);
+ }
+ return managedBuffer;
+ }
+
+ private void writeString(final VarCharWriter writer, final String value) {
+ final byte[] stringBytes = value.getBytes(Charsets.UTF_8);
+ final DrillBuf stringBuffer = buf(stringBytes.length);
+ stringBuffer.clear();
+ stringBuffer.writeBytes(stringBytes);
+ writer.writeVarChar(0, stringBytes.length, stringBuffer);
+ }
+
+ /**
+ * This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get
+ * called when the value of a log field is a String data type.
+ *
+ * @param field name of field
+ * @param value value of field
+ */
+ @SuppressWarnings("unused")
+ public void set(final String field, final String value) {
+ if (value != null) {
+ final VarCharWriter w = strings.get(field);
+ if (w != null) {
+ LOG.trace("Parsed field: {}, as string: {}", field, value);
+ writeString(w, value);
+ }
+ else {
+ LOG.warn("No 'string' writer found for field: {}", field);
+ }
+ }
+ }
+
+ /**
+ * This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get
+ * called when the value of a log field is a Long data type.
+ *
+ * @param field name of field
+ * @param value value of field
+ */
+ @SuppressWarnings("unused")
+ public void set(final String field, final Long value) {
+ if (value != null) {
+ final BigIntWriter w = longs.get(field);
+ if (w != null) {
+ LOG.trace("Parsed field: {}, as long: {}", field, value);
+ w.writeBigInt(value);
+ }
+ else {
+ LOG.warn("No 'long' writer found for field: {}", field);
+ }
+ }
+ }
+
+ /**
+ * This method is referenced and called via reflection. This is added as a parsing target for the parser. It will get
+ * called when the value of a log field is a Double data type.
+ *
+ * @param field name of field
+ * @param value value of field
+ */
+ @SuppressWarnings("unused")
+ public void set(final String field, final Double value) {
+ if (value != null) {
+ final Float8Writer w = doubles.get(field);
+ if (w != null) {
+ LOG.trace("Parsed field: {}, as double: {}", field, value);
+ w.writeFloat8(value);
+ }
+ else {
+ LOG.warn("No 'double' writer found for field: {}", field);
+ }
+ }
+ }
+
+ /**
+ * This method is referenced and called via reflection. When the parser processes a field like:
+ * HTTP.URI:request.firstline.uri.query.* where star is an arbitrary field that the parser found this method will be
+ * invoked. <br>
+ *
+ * @param field name of field
+ * @param value value of field
+ */
+ @SuppressWarnings("unused")
+ public void setWildcard(final String field, final String value) {
+ if (value != null) {
+ final MapWriter mapWriter = getWildcardWriter(field);
+ LOG.trace("Parsed wildcard field: {}, as string: {}", field, value);
+ final VarCharWriter w = mapWriter.varChar(cleanExtensions.get(field));
+ writeString(w, value);
+ }
+ }
+
+ /**
+ * This method is referenced and called via reflection. When the parser processes a field like:
+ * HTTP.URI:request.firstline.uri.query.* where star is an arbitrary field that the parser found this method will be
+ * invoked. <br>
+ *
+ * @param field name of field
+ * @param value value of field
+ */
+ @SuppressWarnings("unused")
+ public void setWildcard(final String field, final Long value) {
+ if (value != null) {
+ final MapWriter mapWriter = getWildcardWriter(field);
+ LOG.trace("Parsed wildcard field: {}, as long: {}", field, value);
+ final BigIntWriter w = mapWriter.bigInt(cleanExtensions.get(field));
+ w.writeBigInt(value);
+ }
+ }
+
+ /**
+ * This method is referenced and called via reflection. When the parser processes a field like:
+ * HTTP.URI:request.firstline.uri.query.* where star is an arbitrary field that the parser found this method will be
+ * invoked. <br>
+ *
+ * @param field name of field
+ * @param value value of field
+ */
+ @SuppressWarnings("unused")
+ public void setWildcard(final String field, final Double value) {
+ if (value != null) {
+ final MapWriter mapWriter = getWildcardWriter(field);
+ LOG.trace("Parsed wildcard field: {}, as double: {}", field, value);
+ final Float8Writer w = mapWriter.float8(cleanExtensions.get(field));
+ w.writeFloat8(value);
+ }
+ }
+
+ /**
+ * For a configuration like HTTP.URI:request.firstline.uri.query.*, a writer was created with name
+ * HTTP.URI:request.firstline.uri.query, we traverse the list of wildcard writers to see which one is the root of the
+ * name of the field passed in like HTTP.URI:request.firstline.uri.query.old. This is writer entry that is needed.
+ *
+ * @param field like HTTP.URI:request.firstline.uri.query.old where 'old' is one of many different parameter names.
+ * @return the writer to be used for this field.
+ */
+ private MapWriter getWildcardWriter(final String field) {
+ MapWriter writer = startedWildcards.get(field);
+ if (writer == null) {
+ for (Map.Entry<String, MapWriter> entry : wildcards.entrySet()) {
+ final String root = entry.getKey();
+ if (field.startsWith(root)) {
+ writer = entry.getValue();
+
+ /**
+ * In order to save some time, store the cleaned version of the field extension. It is possible it will have
+ * unsafe characters in it.
+ */
+ if (!cleanExtensions.containsKey(field)) {
+ final String extension = field.substring(root.length() + 1, field.length());
+ final String cleanExtension = HttpdParser.drillFormattedFieldName(extension);
+ cleanExtensions.put(field, cleanExtension);
+ LOG.debug("Added extension: field='{}' with cleanExtension='{}'", field, cleanExtension);
+ }
+
+ /**
+ * We already know we have the writer, but if we have put this writer in the started list, do NOT call start
+ * again.
+ */
+ if (!wildcardWriters.containsKey(root)) {
+ /**
+ * Start and store this root map writer for later retrieval.
+ */
+ LOG.debug("Starting new wildcard field writer: {}", field);
+ writer.start();
+ startedWildcards.put(field, writer);
+ wildcardWriters.put(root, writer);
+ }
+
+ /**
+ * Break out of the for loop when we find a root writer that matches the field.
+ */
+ break;
+ }
+ }
+ }
+
+ return writer;
+ }
+
+ public Map<String, VarCharWriter> getStrings() {
+ return strings;
+ }
+
+ public Map<String, BigIntWriter> getLongs() {
+ return longs;
+ }
+
+ public Map<String, Float8Writer> getDoubles() {
+ return doubles;
+ }
+
+ /**
+ * This record will be used with a single parser. For each field that is to be parsed a setter will be called. It
+ * registers a setter method for each field being parsed. It also builds the data writers to hold the data beings
+ * parsed.
+ *
+ * @param parser
+ * @param mapWriter
+ * @param type
+ * @param parserFieldName
+ * @param drillFieldName
+ * @throws NoSuchMethodException
+ */
+ public void addField(final Parser<HttpdLogRecord> parser, final MapWriter mapWriter, final EnumSet<Casts> type, final String parserFieldName, final String drillFieldName) throws NoSuchMethodException {
+ final boolean hasWildcard = parserFieldName.endsWith(HttpdParser.PARSER_WILDCARD);
+
+ /**
+ * This is a dynamic way to map the setter for each specified field type. <br/>
+ * e.g. a TIME.STAMP may map to a LONG while a referrer may map to a STRING
+ */
+ if (hasWildcard) {
+ final String cleanName = parserFieldName.substring(0, parserFieldName.length() - HttpdParser.PARSER_WILDCARD.length());
+ LOG.debug("Adding WILDCARD parse target: {} as {}, with field name: {}", parserFieldName, cleanName, drillFieldName);
+ parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, String.class), parserFieldName);
+ parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Double.class), parserFieldName);
+ parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Long.class), parserFieldName);
+ wildcards.put(cleanName, mapWriter.map(drillFieldName));
+ }
+ else if (type.contains(Casts.DOUBLE)) {
+ LOG.debug("Adding DOUBLE parse target: {}, with field name: {}", parserFieldName, drillFieldName);
+ parser.addParseTarget(this.getClass().getMethod("set", String.class, Double.class), parserFieldName);
+ doubles.put(parserFieldName, mapWriter.float8(drillFieldName));
+ }
+ else if (type.contains(Casts.LONG)) {
+ LOG.debug("Adding LONG parse target: {}, with field name: {}", parserFieldName, drillFieldName);
+ parser.addParseTarget(this.getClass().getMethod("set", String.class, Long.class), parserFieldName);
+ longs.put(parserFieldName, mapWriter.bigInt(drillFieldName));
+ }
+ else {
+ LOG.debug("Adding STRING parse target: {}, with field name: {}", parserFieldName, drillFieldName);
+ parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName);
+ strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
new file mode 100644
index 0000000..114a7f4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/httpd/HttpdParser.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2015 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.httpd;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import io.netty.buffer.DrillBuf;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import nl.basjes.parse.core.Casts;
+import nl.basjes.parse.core.Parser;
+import nl.basjes.parse.core.exceptions.DissectionFailure;
+import nl.basjes.parse.core.exceptions.InvalidDissectorException;
+import nl.basjes.parse.core.exceptions.MissingDissectorsException;
+import nl.basjes.parse.httpdlog.HttpdLoglineParser;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpdParser {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HttpdParser.class);
+ public static final String PARSER_WILDCARD = ".*";
+ public static final String SAFE_WILDCARD = "_$";
+ public static final String SAFE_SEPARATOR = "_";
+ public static final String REMAPPING_FLAG = "#";
+ private final Parser<HttpdLogRecord> parser;
+ private final HttpdLogRecord record;
+
+ public HttpdParser(final MapWriter mapWriter, final DrillBuf managedBuffer, final String logFormat,
+ final String timestampFormat, final Map<String, String> fieldMapping)
+ throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
+
+ Preconditions.checkArgument(logFormat != null && !logFormat.trim().isEmpty(), "logFormat cannot be null or empty");
+
+ this.record = new HttpdLogRecord(managedBuffer);
+ this.parser = new HttpdLoglineParser<>(HttpdLogRecord.class, logFormat, timestampFormat);
+
+ setupParser(mapWriter, logFormat, fieldMapping);
+
+ if (timestampFormat != null && !timestampFormat.trim().isEmpty()) {
+ LOG.info("Custom timestamp format has been specified. This is an informational note only as custom timestamps is rather unusual.");
+ }
+ if (logFormat.contains("\n")) {
+ LOG.info("Specified logformat is a multiline log format: {}", logFormat);
+ }
+ }
+
+ /**
+ * We do not expose the underlying parser or the record which is used to manage the writers.
+ *
+ * @param line log line to tear apart.
+ *
+ * @throws DissectionFailure
+ * @throws InvalidDissectorException
+ * @throws MissingDissectorsException
+ */
+ public void parse(final String line) throws DissectionFailure, InvalidDissectorException, MissingDissectorsException {
+ parser.parse(record, line);
+ record.finishRecord();
+ }
+
+ /**
+ * In order to define a type remapping the format of the field configuration will look like: <br/>
+ * HTTP.URI:request.firstline.uri.query.[parameter name] <br/>
+ *
+ * @param parser Add type remapping to this parser instance.
+ * @param fieldName request.firstline.uri.query.[parameter_name]
+ * @param fieldType HTTP.URI, etc..
+ */
+ private void addTypeRemapping(final Parser<HttpdLogRecord> parser, final String fieldName, final String fieldType) {
+ LOG.debug("Adding type remapping - fieldName: {}, fieldType: {}", fieldName, fieldType);
+ parser.addTypeRemapping(fieldName, fieldType);
+ }
+
+ /**
+ * The parser deals with dots unlike Drill wanting underscores request_referer. For the sake of simplicity we are
+ * going replace the dots. The resultant output field will look like: request.referer.<br>
+ * Additionally, wild cards will get replaced with .*
+ *
+ * @param drillFieldName name to be cleansed.
+ * @return
+ */
+ public static String parserFormattedFieldName(final String drillFieldName) {
+ return drillFieldName.replace(SAFE_WILDCARD, PARSER_WILDCARD).replaceAll(SAFE_SEPARATOR, ".").replaceAll("\\.\\.", "_");
+ }
+
+ /**
+ * Drill cannot deal with fields with dots in them like request.referer. For the sake of simplicity we are going
+ * ensure the field name is cleansed. The resultant output field will look like: request_referer.<br>
+ * Additionally, wild cards will get replaced with _$
+ *
+ * @param parserFieldName name to be cleansed.
+ * @return
+ */
+ public static String drillFormattedFieldName(final String parserFieldName) {
+ return parserFieldName.replaceAll("_", "__").replace(PARSER_WILDCARD, SAFE_WILDCARD).replaceAll("\\.", SAFE_SEPARATOR);
+ }
+
+ private void setupParser(final MapWriter mapWriter, final String logFormat, final Map<String, String> fieldMapping)
+ throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
+
+ /**
+ * If the user has selected fields, then we will use them to configure the parser because this would be the most
+ * efficient way to parse the log.
+ */
+ final Map<String, String> requestedPaths;
+ final List<String> allParserPaths = parser.getPossiblePaths();
+ if (fieldMapping != null && !fieldMapping.isEmpty()) {
+ LOG.debug("Using fields defined by user.");
+ requestedPaths = fieldMapping;
+ }
+ else {
+ /**
+ * Use all possible paths that the parser has determined from the specified log format.
+ */
+ LOG.debug("No fields defined by user, defaulting to all possible fields.");
+ requestedPaths = Maps.newHashMap();
+ for (final String parserPath : allParserPaths) {
+ requestedPaths.put(drillFormattedFieldName(parserPath), parserPath);
+ }
+ }
+
+ /**
+ * By adding the parse target to the dummy instance we activate it for use. Which we can then use to find out which
+ * paths cast to which native data types. After we are done figuring this information out, we throw this away
+ * because this will be the slowest parsing path possible for the specified format.
+ */
+ Parser<Object> dummy = new HttpdLoglineParser<>(Object.class, logFormat);
+ dummy.addParseTarget(String.class.getMethod("indexOf", String.class), allParserPaths);
+
+ for (final Map.Entry<String, String> entry : requestedPaths.entrySet()) {
+ final EnumSet<Casts> casts;
+
+ /**
+ * Check the field specified by the user to see if it is supposed to be remapped.
+ */
+ if (entry.getValue().startsWith(REMAPPING_FLAG)) {
+ /**
+ * Because this field is being remapped we need to replace the field name that the parser uses.
+ */
+ entry.setValue(entry.getValue().substring(REMAPPING_FLAG.length()));
+
+ final String[] pieces = entry.getValue().split(":");
+ addTypeRemapping(parser, pieces[1], pieces[0]);
+
+ casts = Casts.STRING_ONLY;
+ }
+ else {
+ casts = dummy.getCasts(entry.getValue());
+ }
+
+ LOG.debug("Setting up drill field: {}, parser field: {}, which casts as: {}", entry.getKey(), entry.getValue(), casts);
+ record.addField(parser, mapWriter, casts, entry.getValue(), entry.getKey());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
index dab4ada..b5485d2 100644
--- a/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/main/resources/bootstrap-storage-plugins.json
@@ -29,9 +29,10 @@
extensions: [ "tsv" ],
delimiter: "\t"
},
- "httpd" :{
+ "httpd" : {
type: "httpd",
- format: "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\""
+ logFormat: "%h %t \"%r\" %>s %b \"%{Referer}i\""
+ /* timestampFormat: "dd/MMM/yyyy:HH:mm:ss ZZ" */
},
"parquet" : {
type: "parquet"
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index e528d0e..93916e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -18,6 +18,7 @@
package org.apache.drill;
import static org.hamcrest.core.StringContains.containsString;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
@@ -65,6 +66,10 @@ import org.junit.runner.Description;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.io.Resources;
+import java.util.ArrayList;
+import java.util.Arrays;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
public class BaseTestQuery extends ExecTest {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseTestQuery.class);
@@ -302,7 +307,7 @@ public class BaseTestQuery extends ExecTest {
Preconditions.checkArgument(query instanceof String, "Expected a string as input query");
query = QueryTestUtil.normalizeQuery((String)query);
return client.runQuery(type, (String)query);
- }
+ }
}
public static List<QueryDataBatch> testPreparedStatement(PreparedStatementHandle handle) throws Exception {
@@ -384,9 +389,9 @@ public class BaseTestQuery extends ExecTest {
} catch (AssertionError e) {
e.addSuppressed(actualException);
throw e;
- }
}
}
+ }
/**
* Utility method which tests given query produces a {@link UserException}
@@ -501,4 +506,69 @@ public class BaseTestQuery extends ExecTest {
return formattedResults.toString();
}
-}
+
+
+ public class TestResultSet {
+
+ private final List<List<String>> rows;
+
+ public TestResultSet() {
+ rows = new ArrayList<>();
+ }
+
+ public TestResultSet(List<QueryDataBatch> batches) throws SchemaChangeException {
+ rows = new ArrayList<>();
+ convert(batches);
+ }
+
+ public void addRow(String... cells) {
+ List<String> newRow = Arrays.asList(cells);
+ rows.add(newRow);
+ }
+
+ public int size() {
+ return rows.size();
+ }
+
+ @Override public boolean equals(Object o) {
+ boolean result = false;
+
+ if (this == o) {
+ result = true;
+ } else if (o instanceof TestResultSet) {
+ TestResultSet that = (TestResultSet) o;
+ assertEquals(this.size(), that.size());
+ for (int i = 0; i < this.rows.size(); i++) {
+ assertEquals(this.rows.get(i).size(), that.rows.get(i).size());
+ for (int j = 0; j < this.rows.get(i).size(); ++j) {
+ assertEquals(this.rows.get(i).get(j), that.rows.get(i).get(j));
+ }
+ }
+ result = true;
+ }
+
+ return result;
+ }
+
+ private void convert(List<QueryDataBatch> batches) throws SchemaChangeException {
+ RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+ for (QueryDataBatch batch : batches) {
+ int rc = batch.getHeader().getRowCount();
+ if (batch.getData() != null) {
+ loader.load(batch.getHeader().getDef(), batch.getData());
+ for (int i = 0; i < rc; ++i) {
+ List<String> newRow = new ArrayList<>();
+ rows.add(newRow);
+ for (VectorWrapper<?> vw : loader) {
+ ValueVector.Accessor accessor = vw.getValueVector().getAccessor();
+ Object o = accessor.getObject(i);
+ newRow.add(o == null ? null : o.toString());
+ }
+ }
+ }
+ loader.clear();
+ batch.release();
+ }
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
new file mode 100644
index 0000000..961d9a6
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/HttpdParserTest.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2015 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.httpd;
+
+import io.netty.buffer.DrillBuf;
+import java.util.Map;
+import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpdParserTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HttpdParserTest.class);
+
+ private void runTest(String logFormat, String logLine) throws Exception {
+ MapWriter mapWriter = null;
+ DrillBuf managedBuffer = null;
+ Map<String, String> configuredFields = null;
+ HttpdParser parser = new HttpdParser(mapWriter, managedBuffer, logFormat, null, configuredFields);
+ parser.parse(logLine);
+ }
+
+// @Test
+ public void testFirstPattern() throws Exception {
+ LOG.info("testFirstPattern");
+// final String format = "common";
+// final String format = "%h %l %u %t \"%r\" %>s %b";
+ final String format = "%h %t \"%r\" %>s %b \"%{Referer}i\"";
+ final String line = "127.0.0.1 [31/Dec/2012:23:49:41 +0100] "
+ + "\"GET /foo HTTP/1.1\" 200 "
+ + "1213 \"http://localhost/index.php?mies=wim\"";
+ runTest(format, line);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
new file mode 100644
index 0000000..6c2eaf8
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/httpd/TestHttpdLogFormatPlugin.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package org.apache.drill.exec.store.httpd;
+
+import java.util.List;
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.rpc.user.QueryDataBatch;
+import static org.junit.Assert.*;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestHttpdLogFormatPlugin extends BaseTestQuery {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestHttpdLogFormatPlugin.class);
+
+ /**
+ * This test covers the test bootstrap-storage-plugins.json section of httpd.
+ *
+ * Indirectly this validates the HttpdLogFormatPlugin.HttpdLogFormatConfig deserializing properly.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDfsTestBootstrap_star() throws Exception {
+ test("select * from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`");
+ }
+
+ /**
+ * This test covers the test bootstrap-storage-plugins.json section of httpd.
+ *
+ * Indirectly this validates the HttpdLogFormatPlugin.HttpdLogFormatConfig deserializing properly.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDfsTestBootstrap_notstar() throws Exception {
+ test("select `TIME_STAMP:request_receive_time`, `HTTP_METHOD:request_firstline_method`, `STRING:request_status_last`, `BYTES:response_body_bytesclf` \n"
+ + "from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`");
+ }
+
+ /**
+ * This test covers the main bootstrap-storage-plugins.json section of httpd.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDfsBootstrap_star() throws Exception {
+ test("select * from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
+ }
+
+ /**
+ * This test covers the main bootstrap-storage-plugins.json section of httpd.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDfsBootstrap_wildcard() throws Exception {
+ test("select `STRING:request_referer_query_$` from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
+ }
+
+ /**
+ * This test covers the main bootstrap-storage-plugins.json section of httpd.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testDfsBootstrap_underscore() throws Exception {
+ test("select `TIME_DAY:request_receive_time_day__utc` from dfs.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-bootstrap.httpd`");
+ }
+
+ @Test
+ public void testGroupBy_1() throws Exception {
+ final List<QueryDataBatch> actualResults = testSqlWithResults(
+ "select `HTTP_METHOD:request_firstline_method` as http_method, `STRING:request_status_last` as status_code, sum(`BYTES:response_body_bytesclf`) as total_bytes \n"
+ + "from dfs_test.`${WORKING_PATH}/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd`\n"
+ + "group by `HTTP_METHOD:request_firstline_method`, `STRING:request_status_last`"
+ );
+
+ final TestResultSet expectedResultSet = new TestResultSet();
+ expectedResultSet.addRow("GET", "200", "46551");
+ expectedResultSet.addRow("POST", "302", "18186");
+
+ TestResultSet actualResultSet = new TestResultSet(actualResults);
+ assertTrue(expectedResultSet.equals(actualResultSet));
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
index 882033a..1ff6818 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/text/TestTextColumn.java
@@ -1,107 +1,58 @@
/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
+ * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
*/
package org.apache.drill.exec.store.text;
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import org.apache.drill.BaseTestQuery;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.rpc.user.QueryDataBatch;
-import org.apache.drill.exec.vector.ValueVector;
import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestTextColumn extends BaseTestQuery {
-public class TestTextColumn extends BaseTestQuery{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTextColumn.class);
@Test
- public void testCsvColumnSelection() throws Exception{
+ public void testCsvColumnSelection() throws Exception {
test("select columns[0] as region_id, columns[1] as country from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/regions.csv`");
}
@Test
public void testDefaultDelimiterColumnSelection() throws Exception {
- List<QueryDataBatch> batches = testSqlWithResults("SELECT columns[0] as entire_row " +
- "from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.txt`");
+ List<QueryDataBatch> actualResults = testSqlWithResults("SELECT columns[0] as entire_row "
+ + "from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.txt`");
- List<List<String>> expectedOutput = Arrays.asList(
- Arrays.asList("a, b,\",\"c\",\"d,, \\n e"),
- Arrays.asList("d, e,\",\"f\",\"g,, \\n h"),
- Arrays.asList("g, h,\",\"i\",\"j,, \\n k"));
+ final TestResultSet expectedResultSet = new TestResultSet();
+ expectedResultSet.addRow("a, b,\",\"c\",\"d,, \\n e");
+ expectedResultSet.addRow("d, e,\",\"f\",\"g,, \\n h");
+ expectedResultSet.addRow("g, h,\",\"i\",\"j,, \\n k");
- List<List<String>> actualOutput = getOutput(batches);
- System.out.println(actualOutput);
- validateOutput(expectedOutput, actualOutput);
+ TestResultSet actualResultSet = new TestResultSet(actualResults);
+ assertTrue(expectedResultSet.equals(actualResultSet));
}
@Test
public void testCsvColumnSelectionCommasInsideQuotes() throws Exception {
- List<QueryDataBatch> batches = testSqlWithResults("SELECT columns[0] as col1, columns[1] as col2, columns[2] as col3," +
- "columns[3] as col4 from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.csv`");
+ List<QueryDataBatch> actualResults = testSqlWithResults("SELECT columns[0] as col1, columns[1] as col2, columns[2] as col3,"
+ + "columns[3] as col4 from dfs_test.`[WORKING_PATH]/src/test/resources/store/text/data/letters.csv`");
- List<List<String>> expectedOutput = Arrays.asList(
- Arrays.asList("a, b,", "c", "d,, \\n e","f\\\"g"),
- Arrays.asList("d, e,", "f", "g,, \\n h","i\\\"j"),
- Arrays.asList("g, h,", "i", "j,, \\n k","l\\\"m"));
-
- List<List<String>> actualOutput = getOutput(batches);
- validateOutput(expectedOutput, actualOutput);
- }
+ final TestResultSet expectedResultSet = new TestResultSet();
+ expectedResultSet.addRow("a, b,", "c", "d,, \\n e", "f\\\"g");
+ expectedResultSet.addRow("d, e,", "f", "g,, \\n h", "i\\\"j");
+ expectedResultSet.addRow("g, h,", "i", "j,, \\n k", "l\\\"m");
- private List<List<String>> getOutput(List<QueryDataBatch> batches) throws SchemaChangeException {
- List<List<String>> output = new ArrayList<>();
- RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
- int last = 0;
- for(QueryDataBatch batch : batches) {
- int rows = batch.getHeader().getRowCount();
- if(batch.getData() != null) {
- loader.load(batch.getHeader().getDef(), batch.getData());
- // TODO: Clean: DRILL-2933: That load(...) no longer throws
- // SchemaChangeException, so check/clean throws clause above.
- for (int i = 0; i < rows; ++i) {
- output.add(new ArrayList<String>());
- for (VectorWrapper<?> vw: loader) {
- ValueVector.Accessor accessor = vw.getValueVector().getAccessor();
- Object o = accessor.getObject(i);
- output.get(last).add(o == null ? null: o.toString());
- }
- ++last;
- }
- }
- loader.clear();
- batch.release();
- }
- return output;
+ TestResultSet actualResultSet = new TestResultSet(actualResults);
+ assertTrue(expectedResultSet.equals(actualResultSet));
}
-
- private void validateOutput(List<List<String>> expected, List<List<String>> actual) {
- assertEquals(expected.size(), actual.size());
- for (int i = 0 ; i < expected.size(); ++i) {
- assertEquals(expected.get(i).size(), actual.get(i).size());
- for (int j = 0; j < expected.get(i).size(); ++j) {
- assertEquals(expected.get(i).get(j), actual.get(i).get(j));
- }
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json b/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
index 630db6b..7b977e2 100644
--- a/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
+++ b/exec/java-exec/src/test/resources/bootstrap-storage-plugins.json
@@ -40,9 +40,9 @@
"json" : {
type: "json"
},
- "httpd" :{
+ "httpd" : {
type: "httpd",
- format: "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\""
+ logFormat: "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\""
},
"txt" : {
type : "text",
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/test/resources/store/httpd/dfs-bootstrap.httpd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/httpd/dfs-bootstrap.httpd b/exec/java-exec/src/test/resources/store/httpd/dfs-bootstrap.httpd
new file mode 100644
index 0000000..fd12566
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/httpd/dfs-bootstrap.httpd
@@ -0,0 +1,2 @@
+127.0.0.1 [31/Dec/2012:23:49:41 +0100] "GET /foo HTTP/1.1" 200 1213 "http://localhost/index.php?mies=wim&blue=red"
+127.0.0.1 [31/Dec/2012:23:49:41 +0100] "GET /foo HTTP/1.1" 200 1213 "http://localhost/index.php?mies=wim&test=true"
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/exec/java-exec/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd b/exec/java-exec/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd
new file mode 100644
index 0000000..d48fa12
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/httpd/dfs-test-bootstrap-test.httpd
@@ -0,0 +1,5 @@
+195.154.46.135 - - [25/Oct/2015:04:11:25 +0100] "GET /linux/doing-pxe-without-dhcp-control HTTP/1.1" 200 24323 "http://howto.basjes.nl/" "Mozilla/5.0 (Windows NT 5.1; rv:35.0) Gecko/20100101 Firefox/35.0"
+23.95.237.180 - - [25/Oct/2015:04:11:26 +0100] "GET /join_form HTTP/1.0" 200 11114 "http://howto.basjes.nl/" "Mozilla/5.0 (Windows NT 5.1; rv:35.0) Gecko/20100101 Firefox/35.0"
+23.95.237.180 - - [25/Oct/2015:04:11:27 +0100] "POST /join_form HTTP/1.1" 302 9093 "http://howto.basjes.nl/join_form" "Mozilla/5.0 (Windows NT 5.1; rv:35.0) Gecko/20100101 Firefox/35.0"
+158.222.5.157 - - [25/Oct/2015:04:24:31 +0100] "GET /join_form HTTP/1.0" 200 11114 "http://howto.basjes.nl/" "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0 AlexaToolbar/alxf-2.21"
+158.222.5.157 - - [25/Oct/2015:04:24:32 +0100] "POST /join_form HTTP/1.1" 302 9093 "http://howto.basjes.nl/join_form" "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0 AlexaToolbar/alxf-2.21"
http://git-wip-us.apache.org/repos/asf/drill/blob/46c0f2a4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index dcb1b3d..e094c4b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -223,6 +223,7 @@
<exclude>**/*.linux</exclude>
<exclude>**/client/build/**</exclude>
<exclude>**/*.tbl</exclude>
+ <exclude>**/*.httpd</exclude>
<!-- TODO DRILL-4336: try to avoid the need to add this -->
<exclude>dependency-reduced-pom.xml</exclude>
</excludes>