You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2017/06/25 01:32:23 UTC
[1/6] drill git commit: DRILL-3867: Metadata Caching : Moving a
directory which contains a cache file causes subsequent queries to fail -
change absolute to relative path in the parquet metadata cache files;
- add converting of the relative paths in the
Repository: drill
Updated Branches:
refs/heads/master 90f43bff7 -> 9cf6faa7a
DRILL-3867: Metadata Caching : Moving a directory which contains a cache file causes subsequent queries to fail - change absolute to relative path in the parquet metadata cache files; - add converting of the relative paths in the metadata to absolute ones. - test case when table is moved to other place after creating meta cache files.
Changes according to the review
Minor changes according to the review
close apache/drill#824
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/964a9473
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/964a9473
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/964a9473
Branch: refs/heads/master
Commit: 964a947315c31571954c1f4f56ac3336ac7bbcda
Parents: 90f43bf
Author: Vitalii Diravka <vi...@gmail.com>
Authored: Fri Apr 14 18:57:13 2017 +0000
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sat Jun 24 09:16:01 2017 -0700
----------------------------------------------------------------------
.../drill/exec/store/parquet/Metadata.java | 159 ++++++++++++++++---
.../exec/store/parquet/ParquetGroupScan.java | 15 +-
.../java/org/apache/drill/BaseTestQuery.java | 58 +++++++
.../TestCorruptParquetDateCorrection.java | 82 ++--------
.../store/parquet/TestParquetMetadataCache.java | 53 ++++++-
...ies_with_absolute_paths.requires_replace.txt | 3 +
...ble_with_absolute_paths.requires_replace.txt | 108 +++++++++++++
..._with_absolute_paths_t1.requires_replace.txt | 76 +++++++++
..._with_absolute_paths_t2.requires_replace.txt | 76 +++++++++
9 files changed, 530 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
index d85d6f1..0a4ce60 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/Metadata.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -67,12 +67,10 @@ import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.JsonDeserializer;
import com.fasterxml.jackson.databind.JsonSerializer;
import com.fasterxml.jackson.databind.KeyDeserializer;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.fasterxml.jackson.module.afterburner.AfterburnerModule;
@@ -159,10 +157,14 @@ public class Metadata {
}
/**
- * Create the parquet metadata file for the directory at the given path, and for any subdirectories
+ * Create the parquet metadata files for the directory at the given path and for any subdirectories.
+ * Metadata cache files written to the disk contain relative paths. Returned Pair of metadata contains absolute paths.
*
- * @param path
- * @throws IOException
+ * @param path to the directory of the parquet table
+ * @return Pair of parquet metadata. The left one is a parquet metadata for the table. The right one of the Pair is
+ * a metadata for all subdirectories (if they are present and there are no any parquet files in the
+ * {@code path} directory).
+ * @throws IOException if parquet metadata can't be serialized and written to the json file
*/
private Pair<ParquetTableMetadata_v3, ParquetTableMetadataDirs>
createMetaFilesRecursively(final String path) throws IOException {
@@ -207,16 +209,21 @@ public class Metadata {
}
parquetTableMetadata.columnTypeInfo.putAll(columnTypeInfoSet);
- for (String oldname : OLD_METADATA_FILENAMES) {
- fs.delete(new Path(p, oldname), false);
+ for (String oldName : OLD_METADATA_FILENAMES) {
+ fs.delete(new Path(p, oldName), false);
}
- writeFile(parquetTableMetadata, new Path(p, METADATA_FILENAME));
+ // relative paths in the metadata are only necessary for meta cache files.
+ ParquetTableMetadata_v3 metadataTableWithRelativePaths =
+ MetadataPathUtils.createMetadataWithRelativePaths(parquetTableMetadata, path);
+ writeFile(metadataTableWithRelativePaths, new Path(p, METADATA_FILENAME));
if (directoryList.size() > 0 && childFiles.size() == 0) {
- ParquetTableMetadataDirs parquetTableMetadataDirs = new ParquetTableMetadataDirs(directoryList);
- writeFile(parquetTableMetadataDirs, new Path(p, METADATA_DIRECTORIES_FILENAME));
+ ParquetTableMetadataDirs parquetTableMetadataDirsRelativePaths =
+ new ParquetTableMetadataDirs(metadataTableWithRelativePaths.directories);
+ writeFile(parquetTableMetadataDirsRelativePaths, new Path(p, METADATA_DIRECTORIES_FILENAME));
logger.info("Creating metadata files recursively took {} ms", timer.elapsed(TimeUnit.MILLISECONDS));
timer.stop();
+ ParquetTableMetadataDirs parquetTableMetadataDirs = new ParquetTableMetadataDirs(directoryList);
return Pair.of(parquetTableMetadata, parquetTableMetadataDirs);
}
List<String> emptyDirList = Lists.newArrayList();
@@ -264,8 +271,10 @@ public class Metadata {
/**
* Get a list of file metadata for a list of parquet files
*
- * @param fileStatuses
- * @return
+ * @param parquetTableMetadata_v3 can store column schema info from all the files and row groups
+ * @param fileStatuses list of the parquet files statuses
+ *
+ * @return list of the parquet file metadata with absolute paths
* @throws IOException
*/
private List<ParquetFileMetadata_v3> getParquetFileMetadata_v3(
@@ -372,10 +381,6 @@ public class Metadata {
/**
* Get the metadata for a single file
- *
- * @param file
- * @return
- * @throws IOException
*/
private ParquetFileMetadata_v3 getParquetFileMetadata_v3(ParquetTableMetadata_v3 parquetTableMetadata,
FileStatus file) throws IOException {
@@ -537,7 +542,8 @@ public class Metadata {
MetadataContext metaContext) throws IOException {
Stopwatch timer = Stopwatch.createStarted();
Path p = new Path(path);
- Path parentDir = p.getParent(); // parent directory of the metadata file
+ Path parentDir = Path.getPathWithoutSchemeAndAuthority(p.getParent()); // parent directory of the metadata file
+ String parentDirString = parentDir.toUri().toString(); // string representation for parent directory of the metadata file
ObjectMapper mapper = new ObjectMapper();
final SimpleModule serialModule = new SimpleModule();
@@ -557,13 +563,14 @@ public class Metadata {
boolean newMetadata = false;
if (metaContext != null) {
- alreadyCheckedModification = metaContext.getStatus(parentDir.toString());
+ alreadyCheckedModification = metaContext.getStatus(parentDirString);
}
if (dirsOnly) {
parquetTableMetadataDirs = mapper.readValue(is, ParquetTableMetadataDirs.class);
logger.info("Took {} ms to read directories from directory cache file", timer.elapsed(TimeUnit.MILLISECONDS));
timer.stop();
+ parquetTableMetadataDirs.updateRelativePaths(parentDirString);
if (!alreadyCheckedModification && tableModified(parquetTableMetadataDirs.getDirectories(), p, parentDir, metaContext)) {
parquetTableMetadataDirs =
(createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(p.getParent()).toString())).getRight();
@@ -573,6 +580,9 @@ public class Metadata {
parquetTableMetadata = mapper.readValue(is, ParquetTableMetadataBase.class);
logger.info("Took {} ms to read metadata from cache file", timer.elapsed(TimeUnit.MILLISECONDS));
timer.stop();
+ if (parquetTableMetadata instanceof ParquetTableMetadata_v3) {
+ ((ParquetTableMetadata_v3) parquetTableMetadata).updateRelativePaths(parentDirString);
+ }
if (!alreadyCheckedModification && tableModified(parquetTableMetadata.getDirectories(), p, parentDir, metaContext)) {
parquetTableMetadata =
(createMetaFilesRecursively(Path.getPathWithoutSchemeAndAuthority(p.getParent()).toString())).getLeft();
@@ -748,6 +758,12 @@ public class Metadata {
return directories;
}
+ /** If directories list contains relative paths, update it to absolute ones
+ * @param baseDir base parent directory
+ */
+ @JsonIgnore public void updateRelativePaths(String baseDir) {
+ this.directories = MetadataPathUtils.convertToAbsolutePaths(directories, baseDir);
+ }
}
@JsonTypeName("v1")
@@ -1413,6 +1429,18 @@ public class Metadata {
return directories;
}
+ /**
+ * If directories list and file metadata list contain relative paths, update it to absolute ones
+ * @param baseDir base parent directory
+ */
+ @JsonIgnore public void updateRelativePaths(String baseDir) {
+ // update directories paths to absolute ones
+ this.directories = MetadataPathUtils.convertToAbsolutePaths(directories, baseDir);
+
+ // update files paths to absolute ones
+ this.files = MetadataPathUtils.convertToFilesWithAbsolutePaths(files, baseDir);
+ }
+
@JsonIgnore @Override public List<? extends ParquetFileMetadata> getFiles() {
return files;
}
@@ -1736,5 +1764,98 @@ public class Metadata {
}
+ /**
+ * Util class that contains helper methods for converting paths in the table and directory metadata structures
+ */
+ private static class MetadataPathUtils {
+
+ /**
+ * Helper method that converts a list of relative paths to absolute ones
+ *
+ * @param paths list of relative paths
+ * @param baseDir base parent directory
+ * @return list of absolute paths
+ */
+ private static List<String> convertToAbsolutePaths(List<String> paths, String baseDir) {
+ if (!paths.isEmpty()) {
+ List<String> absolutePaths = Lists.newArrayList();
+ for (String relativePath : paths) {
+ String absolutePath = (new Path(relativePath).isAbsolute()) ? relativePath
+ : new Path(baseDir, relativePath).toUri().toString();
+ absolutePaths.add(absolutePath);
+ }
+ return absolutePaths;
+ }
+ return paths;
+ }
+
+ /**
+ * Convert a list of files with relative paths to files with absolute ones
+ *
+ * @param files list of files with relative paths
+ * @param baseDir base parent directory
+ * @return list of files with absolute paths
+ */
+ private static List<ParquetFileMetadata_v3> convertToFilesWithAbsolutePaths(
+ List<ParquetFileMetadata_v3> files, String baseDir) {
+ if (!files.isEmpty()) {
+ List<ParquetFileMetadata_v3> filesWithAbsolutePaths = Lists.newArrayList();
+ for (ParquetFileMetadata_v3 file : files) {
+ Path relativePath = new Path(file.getPath());
+ // create a new file if old one contains a relative path, otherwise use an old file
+ ParquetFileMetadata_v3 fileWithAbsolutePath = (relativePath.isAbsolute()) ? file
+ : new ParquetFileMetadata_v3(new Path(baseDir, relativePath).toUri().toString(), file.length, file.rowGroups);
+ filesWithAbsolutePaths.add(fileWithAbsolutePath);
+ }
+ return filesWithAbsolutePaths;
+ }
+ return files;
+ }
+
+ /**
+ * Creates a new parquet table metadata from the {@code tableMetadataWithAbsolutePaths} parquet table.
+ * A new parquet table will contain relative paths for the files and directories.
+ *
+ * @param tableMetadataWithAbsolutePaths parquet table metadata with absolute paths for the files and directories
+ * @param baseDir base parent directory
+ * @return parquet table metadata with relative paths for the files and directories
+ */
+ private static ParquetTableMetadata_v3 createMetadataWithRelativePaths(
+ ParquetTableMetadata_v3 tableMetadataWithAbsolutePaths, String baseDir) {
+ List<String> directoriesWithRelativePaths = Lists.newArrayList();
+ for (String directory : tableMetadataWithAbsolutePaths.getDirectories()) {
+ directoriesWithRelativePaths.add(relativize(baseDir, directory)) ;
+ }
+ List<ParquetFileMetadata_v3> filesWithRelativePaths = Lists.newArrayList();
+ for (ParquetFileMetadata_v3 file : tableMetadataWithAbsolutePaths.files) {
+ filesWithRelativePaths.add(new ParquetFileMetadata_v3(
+ relativize(baseDir, file.getPath()), file.length, file.rowGroups));
+ }
+ return new ParquetTableMetadata_v3(tableMetadataWithAbsolutePaths, filesWithRelativePaths,
+ directoriesWithRelativePaths, DrillVersionInfo.getVersion());
+ }
+
+ /**
+ * Constructs relative path from child full path and base path. Or return child path if the last one is already relative
+ *
+ * @param childPath full absolute path
+ * @param baseDir base path (the part of the Path, which should be cut off from child path)
+ * @return relative path
+ */
+ private static String relativize(String baseDir, String childPath) {
+ Path fullPathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(childPath));
+ Path basePathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(baseDir));
+
+ // Since hadoop Path hasn't relativize() we use uri.relativize() to get relative path
+ Path relativeFilePath = new Path(basePathWithoutSchemeAndAuthority.toUri()
+ .relativize(fullPathWithoutSchemeAndAuthority.toUri()));
+ if (relativeFilePath.isAbsolute()) {
+ throw new IllegalStateException(String.format("Path %s is not a subpath of %s.",
+ basePathWithoutSchemeAndAuthority.toUri().toString(), fullPathWithoutSchemeAndAuthority.toUri().toString()));
+ }
+ return relativeFilePath.toUri().toString();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 71e681b..3d9cfb3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -26,7 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.avro.generic.GenericData;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.ErrorCollector;
@@ -44,12 +43,10 @@ import org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.stat.ParquetFilterPredicate;
-import org.apache.drill.exec.ops.OptimizerRulesContext;
import org.apache.drill.exec.ops.UdfUtilities;
import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.PhysicalOperatorSetupException;
import org.apache.drill.exec.physical.base.AbstractFileGroupScan;
-import org.apache.drill.exec.physical.base.FileGroupScan;
import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.base.ScanStats;
@@ -58,7 +55,6 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.store.ImplicitColumnExplorer;
-import org.apache.drill.exec.store.ParquetOutputRecordWriter;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
import org.apache.drill.exec.store.dfs.DrillPathFilter;
@@ -677,15 +673,16 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
} else {
// we need to expand the files from fileStatuses
for (FileStatus status : fileStatuses) {
+ Path cacheFileRoot = status.getPath();
if (status.isDirectory()) {
//TODO [DRILL-4496] read the metadata cache files in parallel
- final Path metaPath = new Path(status.getPath(), Metadata.METADATA_FILENAME);
+ final Path metaPath = new Path(cacheFileRoot, Metadata.METADATA_FILENAME);
final Metadata.ParquetTableMetadataBase metadata = Metadata.readBlockMeta(fs, metaPath.toString(), selection.getMetaContext(), formatConfig);
for (Metadata.ParquetFileMetadata file : metadata.getFiles()) {
fileSet.add(file.getPath());
}
} else {
- final Path path = Path.getPathWithoutSchemeAndAuthority(status.getPath());
+ final Path path = Path.getPathWithoutSchemeAndAuthority(cacheFileRoot);
fileSet.add(path.toString());
}
}
@@ -718,9 +715,9 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
}
private void init(MetadataContext metaContext) throws IOException {
+ Path metaPath = null;
if (entries.size() == 1 && parquetTableMetadata == null) {
Path p = Path.getPathWithoutSchemeAndAuthority(new Path(entries.get(0).getPath()));
- Path metaPath = null;
if (fs.isDirectory(p)) {
// Using the metadata file makes sense when querying a directory; otherwise
// if querying a single file we can look up the metadata directly from the file
@@ -734,7 +731,7 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
}
} else {
Path p = Path.getPathWithoutSchemeAndAuthority(new Path(selectionRoot));
- Path metaPath = new Path(p, Metadata.METADATA_FILENAME);
+ metaPath = new Path(p, Metadata.METADATA_FILENAME);
if (fs.isDirectory(new Path(selectionRoot)) && fs.exists(metaPath)) {
usedMetadataCache = true;
if (parquetTableMetadata == null) {
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 4401b9f..e3ce9fc 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -36,6 +36,7 @@ import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.scanner.ClassPathScanner;
import org.apache.drill.common.scanner.persistence.ScanResult;
+import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ExecTest;
import org.apache.drill.exec.client.DrillClient;
@@ -58,6 +59,10 @@ import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.util.TestUtilities;
import org.apache.drill.exec.util.VectorUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.rules.TestRule;
@@ -69,6 +74,9 @@ import com.google.common.base.Preconditions;
import com.google.common.io.Resources;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.ValueVector;
@@ -120,6 +128,8 @@ public class BaseTestQuery extends ExecTest {
private static ScanResult classpathScan;
+ private static FileSystem fs;
+
@BeforeClass
public static void setupDefaultTestCluster() throws Exception {
config = DrillConfig.create(TEST_CONFIGURATIONS);
@@ -128,6 +138,9 @@ public class BaseTestQuery extends ExecTest {
// turns on the verbose errors in tests
// sever side stacktraces are added to the message before sending back to the client
test("ALTER SESSION SET `exec.errors.verbose` = true");
+ Configuration conf = new Configuration();
+ conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
+ fs = FileSystem.get(conf);
}
protected static void updateTestCluster(int newDrillbitCount, DrillConfig newConfig) {
@@ -593,4 +606,49 @@ public class BaseTestQuery extends ExecTest {
}
}
}
+
+ private static String replaceWorkingPathInString(String orig) {
+ return orig.replaceAll(Pattern.quote("[WORKING_PATH]"), Matcher.quoteReplacement(TestTools.getWorkingPath()));
+ }
+
+ protected static void copyDirectoryIntoTempSpace(String resourcesDir) throws IOException {
+ copyDirectoryIntoTempSpace(resourcesDir, null);
+ }
+
+ protected static void copyDirectoryIntoTempSpace(String resourcesDir, String destinationSubDir) throws IOException {
+ Path destination = destinationSubDir != null ? new Path(getDfsTestTmpSchemaLocation(), destinationSubDir)
+ : new Path(getDfsTestTmpSchemaLocation());
+ fs.copyFromLocalFile(
+ new Path(replaceWorkingPathInString(resourcesDir)),
+ destination);
+ }
+
+ /**
+ * Old metadata cache files include full paths to the files that have been scanned.
+ * <p>
+ * There is no way to generate a metadata cache file with absolute paths that
+ * will be guaranteed to be available on an arbitrary test machine.
+ * <p>
+ * To enable testing older metadata cache files, they were generated manually
+ * using older drill versions, and the absolute path up to the folder where
+ * the metadata cache file appeared was manually replaced with the string
+ * REPLACED_IN_TEST. Here the file is re-written into the given temporary
+ * location after the REPLACED_IN_TEST string has been replaced by the actual
+ * location generated during this run of the tests.
+ *
+ * @param srcFileOnClassPath the source path of metadata cache file, which should be replaced
+ * @param destFolderInTmp the parent folder name of the metadata cache file
+ * @param metaFileName the name of metadata cache file depending on the type of the metadata
+ * @throws IOException if a create or write errors occur
+ */
+ protected static void copyMetaDataCacheToTempReplacingInternalPaths(String srcFileOnClassPath, String destFolderInTmp,
+ String metaFileName) throws IOException {
+ String metadataFileContents = getFile(srcFileOnClassPath);
+ Path rootMeta = new Path(dfsTestTmpSchemaLocation, destFolderInTmp);
+ Path newMetaCache = new Path(rootMeta, metaFileName);
+ FSDataOutputStream outSteam = fs.create(newMetaCache);
+ outSteam.writeBytes(metadataFileContents.replace("REPLACED_IN_TEST", dfsTestTmpSchemaLocation));
+ outSteam.close();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
index 0c98eee..df13799 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,20 +21,13 @@ import static java.lang.String.format;
import org.apache.drill.PlanTestBase;
import org.apache.drill.TestBuilder;
-import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.ExecConstants;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.drill.exec.store.parquet.Metadata;
import org.apache.hadoop.fs.Path;
import org.joda.time.DateTime;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
/**
* Tests for compatibility reading old parquet files after date corruption
* issue was fixed in DRILL-4203.
@@ -106,21 +99,13 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
private static final String PARTITIONED_1_2_FOLDER = "partitioned_with_corruption_4203_1_2";
private static final String MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER = "mixed_partitioned";
- private static FileSystem fs;
- private static Path path;
@BeforeClass
public static void initFs() throws Exception {
- Configuration conf = new Configuration();
- conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
- fs = FileSystem.get(conf);
- path = new Path(getDfsTestTmpSchemaLocation());
-
- // Move files into temp directory, rewrite the metadata cache file to contain the appropriate absolute
- // path
+ // Move files into temp directory, rewrite the metadata cache file to contain the appropriate absolute path
copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_2_PATH);
copyMetaDataCacheToTempReplacingInternalPaths("parquet/4203_corrupt_dates/drill.parquet.metadata_1_2.requires_replace.txt",
- PARTITIONED_1_2_FOLDER);
+ PARTITIONED_1_2_FOLDER, Metadata.METADATA_FILENAME);
copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_2_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
copyDirectoryIntoTempSpace(CORRECT_PARTITIONED_DATES_1_9_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_4_0_PATH, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
@@ -334,7 +319,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
@Test
public void testReadOldMetadataCacheFile() throws Exception {
// for sanity, try reading all partitions without a filter
- String query = format("select date_col from dfs.`%s`", new Path(path, PARTITIONED_1_2_FOLDER));
+ String query = format("select date_col from dfs.`%s`", new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER));
TestBuilder builder = testBuilder()
.sqlQuery(query)
.unOrdered()
@@ -347,7 +332,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
@Test
public void testReadOldMetadataCacheFileWithPruning() throws Exception {
String query = format("select date_col from dfs.`%s` where date_col = date '1970-01-01'",
- new Path(path, PARTITIONED_1_2_FOLDER));
+ new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER));
// verify that pruning is actually taking place
testPlanMatchingPatterns(query, new String[]{"numFiles=1", "usedMetadataFile=true"}, null);
@@ -365,7 +350,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
// for sanity, try reading all partitions without a filter
TestBuilder builder = testBuilder()
.sqlQuery("select date_col from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))",
- new Path(path, PARTITIONED_1_2_FOLDER))
+ new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER))
.unOrdered()
.baselineColumns("date_col");
addCorruptedDateBaselineValues(builder);
@@ -373,7 +358,7 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
String query = format("select date_col from table(dfs.`%s` (type => 'parquet', " +
"autoCorrectCorruptDates => false)) where date_col = cast('15334-03-17' as date)",
- new Path(path, PARTITIONED_1_2_FOLDER));
+ new Path(getDfsTestTmpSchemaLocation(), PARTITIONED_1_2_FOLDER));
// verify that pruning is actually taking place
testPlanMatchingPatterns(query, new String[]{"numFiles=1", "usedMetadataFile=true"}, null);
@@ -388,9 +373,10 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
@Test
public void testReadNewMetadataCacheFileOverOldAndNewFiles() throws Exception {
- String table = format("dfs.`%s`", new Path(path, MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER));
- copyMetaDataCacheToTempReplacingInternalPaths("parquet/4203_corrupt_dates/" +
- "mixed_version_partitioned_metadata.requires_replace.txt", MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
+ String table = format("dfs.`%s`", new Path(getDfsTestTmpSchemaLocation(), MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER));
+ copyMetaDataCacheToTempReplacingInternalPaths(
+ "parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt",
+ MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER, Metadata.METADATA_FILENAME);
// for sanity, try reading all partitions without a filter
TestBuilder builder = testBuilder()
.sqlQuery("select date_col from " + table)
@@ -488,48 +474,4 @@ public class TestCorruptParquetDateCorrection extends PlanTestBase {
}
}
- private static String replaceWorkingPathInString(String orig) {
- return orig.replaceAll(Pattern.quote("[WORKING_PATH]"), Matcher.quoteReplacement(TestTools.getWorkingPath()));
- }
-
- private static void copyDirectoryIntoTempSpace(String resourcesDir) throws IOException {
- copyDirectoryIntoTempSpace(resourcesDir, null);
- }
-
- private static void copyDirectoryIntoTempSpace(String resourcesDir, String destinationSubDir) throws IOException {
- Path destination = path;
- if (destinationSubDir != null) {
- destination = new Path(path, destinationSubDir);
- }
- fs.copyFromLocalFile(
- new Path(replaceWorkingPathInString(resourcesDir)),
- destination);
- }
-
- /**
- * Metadata cache files include full paths to the files that have been scanned.
- *
- * There is no way to generate a metadata cache file with absolute paths that
- * will be guaranteed to be available on an arbitrary test machine.
- *
- * To enable testing older metadata cache files, they were generated manually
- * using older drill versions, and the absolute path up to the folder where
- * the metadata cache file appeared was manually replaced with the string
- * REPLACED_IN_TEST. Here the file is re-written into the given temporary
- * location after the REPLACED_IN_TEST string has been replaced by the actual
- * location generated during this run of the tests.
- *
- * @param srcFileOnClassPath
- * @param destFolderInTmp
- * @throws IOException
- */
- private static void copyMetaDataCacheToTempReplacingInternalPaths(String srcFileOnClassPath, String destFolderInTmp)
- throws IOException {
- String metadataFileContents = getFile(srcFileOnClassPath);
- Path newMetaCache = new Path(new Path(path, destFolderInTmp), ".drill.parquet_metadata");
- FSDataOutputStream outSteam = fs.create(newMetaCache);
- outSteam.writeBytes(metadataFileContents.replace("REPLACED_IN_TEST", path.toString()));
- outSteam.close();
- }
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
index e199ba5..dff2e86 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -30,12 +30,15 @@ import java.io.File;
import java.nio.file.Files;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
public class TestParquetMetadataCache extends PlanTestBase {
private static final String WORKING_PATH = TestTools.getWorkingPath();
private static final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources";
private static final String tableName1 = "parquetTable1";
private static final String tableName2 = "parquetTable2";
+ private static final String RELATIVE_PATHS_METADATA = "relative_paths_metadata";
@BeforeClass
@@ -398,10 +401,56 @@ public class TestParquetMetadataCache extends PlanTestBase {
}
+ @Test // DRILL-3867
+ public void testMoveCache() throws Exception {
+ final String tableName = "nation_move";
+ final String newTableName = "nation_moved";
+ try {
+ test("use dfs_test.tmp");
+ test("create table `%s/t1` as select * from cp.`tpch/nation.parquet`", tableName);
+ test("create table `%s/t2` as select * from cp.`tpch/nation.parquet`", tableName);
+ test("refresh table metadata %s", tableName);
+ checkForMetadataFile(tableName);
+ File srcFile = new File(getDfsTestTmpSchemaLocation(), tableName);
+ File dstFile = new File(getDfsTestTmpSchemaLocation(), newTableName);
+ FileUtils.moveDirectory(srcFile, dstFile);
+ assertFalse("Cache file was not moved successfully", srcFile.exists());
+ int rowCount = testSql(String.format("select * from %s", newTableName));
+ assertEquals("An incorrect result was obtained while querying a table with metadata cache files", 50, rowCount);
+ } finally {
+ test("drop table if exists %s", newTableName);
+ }
+ }
+
+ @Test
+ public void testMetadataCacheAbsolutePaths() throws Exception {
+ try {
+ test("use dfs_test.tmp");
+ final String relative_path_metadata_t1 = RELATIVE_PATHS_METADATA + "/t1";
+ final String relative_path_metadata_t2 = RELATIVE_PATHS_METADATA + "/t2";
+ test("create table `%s` as select * from cp.`tpch/nation.parquet`", relative_path_metadata_t1);
+ test("create table `%s` as select * from cp.`tpch/nation.parquet`", relative_path_metadata_t2);
+ copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
+ "metadata_directories_with_absolute_paths.requires_replace.txt", RELATIVE_PATHS_METADATA, Metadata.METADATA_DIRECTORIES_FILENAME);
+ copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
+ "metadata_table_with_absolute_paths.requires_replace.txt", RELATIVE_PATHS_METADATA, Metadata.METADATA_FILENAME);
+ copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
+ "metadata_table_with_absolute_paths_t1.requires_replace.txt", relative_path_metadata_t1, Metadata.METADATA_FILENAME);
+ copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" +
+ "metadata_table_with_absolute_paths_t2.requires_replace.txt", relative_path_metadata_t2, Metadata.METADATA_FILENAME);
+
+ int rowCount = testSql(String.format("select * from %s", RELATIVE_PATHS_METADATA));
+ assertEquals("An incorrect result was obtained while querying a table with metadata cache files", 50, rowCount);
+ } finally {
+ test("drop table if exists %s", RELATIVE_PATHS_METADATA);
+ }
+ }
+
private void checkForMetadataFile(String table) throws Exception {
String tmpDir = getDfsTestTmpSchemaLocation();
String metaFile = Joiner.on("/").join(tmpDir, table, Metadata.METADATA_FILENAME);
- Assert.assertTrue(Files.exists(new File(metaFile).toPath()));
+ assertTrue(String.format("There is no metadata cache file for the %s table", table),
+ Files.exists(new File(metaFile).toPath()));
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt
new file mode 100644
index 0000000..887988d
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_directories_with_absolute_paths.requires_replace.txt
@@ -0,0 +1,3 @@
+{
+ "directories" : [ "file:REPLACED_IN_TEST/relative_paths_metadata/t1", "file:REPLACED_IN_TEST/relative_paths_metadata/t2" ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt
new file mode 100644
index 0000000..78a822e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths.requires_replace.txt
@@ -0,0 +1,108 @@
+{
+ "metadata_version" : "v3",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "REPLACED_IN_TEST/relative_paths_metadata/t1/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ }, {
+ "path" : "REPLACED_IN_TEST/relative_paths_metadata/t2/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ "file:REPLACED_IN_TEST/relative_paths_metadata/t1", "file:REPLACED_IN_TEST/relative_paths_metadata/t2" ],
+ "drillVersion" : "1.11.0"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt
new file mode 100644
index 0000000..7ce10e7
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t1.requires_replace.txt
@@ -0,0 +1,76 @@
+{
+ "metadata_version" : "v3",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "REPLACED_IN_TEST/relative_paths_metadata/t1/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ ],
+ "drillVersion" : "1.11.0"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/964a9473/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt
new file mode 100644
index 0000000..c984f58
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/metadata_with_absolute_path/metadata_table_with_absolute_paths_t2.requires_replace.txt
@@ -0,0 +1,76 @@
+{
+ "metadata_version" : "v3",
+ "columnTypeInfo" : {
+ "n_name" : {
+ "name" : [ "n_name" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_nationkey" : {
+ "name" : [ "n_nationkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_regionkey" : {
+ "name" : [ "n_regionkey" ],
+ "primitiveType" : "INT32",
+ "originalType" : null,
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ },
+ "n_comment" : {
+ "name" : [ "n_comment" ],
+ "primitiveType" : "BINARY",
+ "originalType" : "UTF8",
+ "precision" : 0,
+ "scale" : 0,
+ "repetitionLevel" : 0,
+ "definitionLevel" : 0
+ }
+ },
+ "files" : [ {
+ "path" : "REPLACED_IN_TEST/relative_paths_metadata/t2/0_0_0.parquet",
+ "length" : 2424,
+ "rowGroups" : [ {
+ "start" : 4,
+ "length" : 1802,
+ "rowCount" : 25,
+ "hostAffinity" : {
+ "localhost" : 1.0
+ },
+ "columns" : [ {
+ "name" : [ "n_nationkey" ],
+ "minValue" : 0,
+ "maxValue" : 24,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_name" ],
+ "minValue" : "ALGERIA",
+ "maxValue" : "VIETNAM",
+ "nulls" : 0
+ }, {
+ "name" : [ "n_regionkey" ],
+ "minValue" : 0,
+ "maxValue" : 4,
+ "nulls" : 0
+ }, {
+ "name" : [ "n_comment" ],
+ "minValue" : " haggle. carefully final deposits detect slyly agai",
+ "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be",
+ "nulls" : 0
+ } ]
+ } ]
+ } ],
+ "directories" : [ ],
+ "drillVersion" : "1.11.0"
+}
\ No newline at end of file
[5/6] drill git commit: DRILL-5599: Notify StatusHandler that batch
sending has failed even if channel is still open
Posted by am...@apache.org.
DRILL-5599: Notify StatusHandler that batch sending has failed even if channel is still open
close #857
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7e6571aa
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7e6571aa
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7e6571aa
Branch: refs/heads/master
Commit: 7e6571aa5d4c58185dbfa131de99354ea7dc6b4e
Parents: dd55b5c
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Tue Jun 20 12:18:27 2017 +0300
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sat Jun 24 09:42:34 2017 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/rpc/RequestIdMap.java | 21 +++++++++++---------
1 file changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/7e6571aa/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RequestIdMap.java
----------------------------------------------------------------------
diff --git a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RequestIdMap.java b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RequestIdMap.java
index a9c3012..804834c 100644
--- a/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RequestIdMap.java
+++ b/exec/rpc/src/main/java/org/apache/drill/exec/rpc/RequestIdMap.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -47,7 +47,7 @@ class RequestIdMap {
private final IntObjectHashMap<RpcOutcome<?>> map;
public RequestIdMap() {
- map = new IntObjectHashMap<RpcOutcome<?>>();
+ map = new IntObjectHashMap<>();
}
void channelClosed(Throwable ex) {
@@ -82,7 +82,7 @@ class RequestIdMap {
public <V> ChannelListenerWithCoordinationId createNewRpcListener(RpcOutcomeListener<V> handler, Class<V> clazz,
RemoteConnection connection) {
final int i = lastCoordinationId.incrementAndGet();
- final RpcListener<V> future = new RpcListener<V>(handler, clazz, i, connection);
+ final RpcListener<V> future = new RpcListener<>(handler, clazz, i, connection);
final Object old;
synchronized (map) {
Preconditions.checkArgument(isOpen.get(),
@@ -111,13 +111,16 @@ class RequestIdMap {
@Override
public void operationComplete(ChannelFuture future) throws Exception {
-
if (!future.isSuccess()) {
- removeFromMap(coordinationId);
- if (future.channel().isActive()) {
- throw new RpcException("Future failed");
- } else {
- setException(new ChannelClosedException());
+ try {
+ removeFromMap(coordinationId);
+ } finally {
+ final Throwable cause = future.cause();
+ if (future.channel().isActive()) {
+ setException(cause == null ? new RpcException("Unknown ChannelFuture operation failure") : cause);
+ } else {
+ setException(cause == null ? new ChannelClosedException() : new ChannelClosedException(cause));
+ }
}
}
}
[6/6] drill git commit: DRILL-5587: Validate Parquet blockSize and
pageSize configured with SYSTEM/SESSION option
Posted by am...@apache.org.
DRILL-5587: Validate Parquet blockSize and pageSize configured with SYSTEM/SESSION option
close #852
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9cf6faa7
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9cf6faa7
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9cf6faa7
Branch: refs/heads/master
Commit: 9cf6faa7aa834c7ba654ce956c8b523ff3464658
Parents: 7e6571a
Author: Padma Penumarthy <pp...@yahoo.com>
Authored: Thu Jun 15 11:43:04 2017 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sat Jun 24 09:46:20 2017 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/drill/exec/ExecConstants.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/9cf6faa7/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index e2782e9..5b82d1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -188,14 +188,14 @@ public interface ExecConstants {
String OUTPUT_FORMAT_OPTION = "store.format";
OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION, "parquet");
String PARQUET_BLOCK_SIZE = "store.parquet.block-size";
- OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new LongValidator(PARQUET_BLOCK_SIZE, 512*1024*1024);
String PARQUET_WRITER_USE_SINGLE_FS_BLOCK = "store.parquet.writer.use_single_fs_block";
OptionValidator PARQUET_WRITER_USE_SINGLE_FS_BLOCK_VALIDATOR = new BooleanValidator(
PARQUET_WRITER_USE_SINGLE_FS_BLOCK, false);
+ OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_BLOCK_SIZE, Integer.MAX_VALUE, 512 * 1024 * 1024);
String PARQUET_PAGE_SIZE = "store.parquet.page-size";
- OptionValidator PARQUET_PAGE_SIZE_VALIDATOR = new LongValidator(PARQUET_PAGE_SIZE, 1024*1024);
+ OptionValidator PARQUET_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_PAGE_SIZE, Integer.MAX_VALUE, 1024 * 1024);
String PARQUET_DICT_PAGE_SIZE = "store.parquet.dictionary.page-size";
- OptionValidator PARQUET_DICT_PAGE_SIZE_VALIDATOR = new LongValidator(PARQUET_DICT_PAGE_SIZE, 1024*1024);
+ OptionValidator PARQUET_DICT_PAGE_SIZE_VALIDATOR = new PositiveLongValidator(PARQUET_DICT_PAGE_SIZE, Integer.MAX_VALUE, 1024 * 1024);
String PARQUET_WRITER_COMPRESSION_TYPE = "store.parquet.compression";
OptionValidator PARQUET_WRITER_COMPRESSION_TYPE_VALIDATOR = new EnumeratedStringValidator(
PARQUET_WRITER_COMPRESSION_TYPE, "snappy", "snappy", "gzip", "none");
[3/6] drill git commit: DRILL-5130: Implement DrillValuesRel and
ValuesPrel as Calcite Values sub-classes
Posted by am...@apache.org.
DRILL-5130: Implement DrillValuesRel and ValuesPrel as Calcite Values sub-classes
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/33682be9
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/33682be9
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/33682be9
Branch: refs/heads/master
Commit: 33682be99e719dce9cb326e2835ebc4ae434104a
Parents: 886ccdc
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Thu Jun 15 18:03:34 2017 +0300
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sat Jun 24 09:17:29 2017 -0700
----------------------------------------------------------------------
.../exec/planner/common/DrillValuesRelBase.java | 251 ++++++++++++++++++
.../exec/planner/logical/DrillValuesRel.java | 255 +------------------
.../drill/exec/planner/physical/ValuesPrel.java | 32 +--
.../exec/planner/physical/ValuesPrule.java | 14 +-
.../java/org/apache/drill/TestUnionAll.java | 11 +
.../physical/impl/filter/TestLargeInClause.java | 2 +-
6 files changed, 290 insertions(+), 275 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/33682be9/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillValuesRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillValuesRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillValuesRelBase.java
new file mode 100644
index 0000000..4d54fce
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillValuesRelBase.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.common;
+
+import com.fasterxml.jackson.core.JsonLocation;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.util.TokenBuffer;
+import com.google.common.collect.ImmutableList;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.plan.RelTraitSet;
+import org.apache.calcite.rel.core.Values;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexLiteral;
+import org.apache.calcite.util.NlsString;
+import org.apache.drill.common.JSONOptions;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.exec.vector.complex.fn.ExtendedJsonOutput;
+import org.apache.drill.exec.vector.complex.fn.JsonOutput;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeConstants;
+import org.joda.time.Period;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.List;
+
+import static org.apache.drill.exec.planner.logical.DrillOptiq.isLiteralNull;
+
+/**
+ * Base class for logical and physical Values implemented in Drill.
+ */
+public abstract class DrillValuesRelBase extends Values implements DrillRelNode {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillValuesRelBase.class);
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ protected final JSONOptions content;
+
+ public DrillValuesRelBase(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits) {
+ this(cluster, rowType, tuples, traits, convertToJsonOptions(rowType, tuples));
+ }
+
+ /**
+ * This constructor helps to avoid unnecessary tuples parsing into json options
+ * during copying or logical to physical values conversion.
+ */
+ public DrillValuesRelBase(RelOptCluster cluster,
+ RelDataType rowType,
+ ImmutableList<ImmutableList<RexLiteral>> tuples,
+ RelTraitSet traits,
+ JSONOptions content) {
+ super(cluster, rowType, tuples, traits);
+ this.content = content;
+ }
+
+ /**
+ * @return values content represented as json
+ */
+ public JSONOptions getContent() {
+ return content;
+ }
+
+ /**
+ * Converts tuples into json representation taking into account row type.
+ * Example: [['A']] -> [{"EXPR$0":"A"}], [[1]] -> [{"EXPR$0":{"$numberLong":1}}]
+ *
+ * @param rowType row type
+ * @param tuples list of constant values in a row-expression
+ * @return json representation of tuples
+ */
+ private static JSONOptions convertToJsonOptions(RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples) {
+ try {
+ return new JSONOptions(convertToJsonNode(rowType, tuples), JsonLocation.NA);
+ } catch (IOException e) {
+ throw new DrillRuntimeException("Failure while attempting to encode Values in JSON.", e);
+ }
+ }
+
+ private static JsonNode convertToJsonNode(RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples) throws IOException {
+ TokenBuffer out = new TokenBuffer(MAPPER.getFactory().getCodec(), false);
+ JsonOutput json = new ExtendedJsonOutput(out);
+ json.writeStartArray();
+ String[] fields = rowType.getFieldNames().toArray(new String[rowType.getFieldCount()]);
+
+ for (List<RexLiteral> row : tuples) {
+ json.writeStartObject();
+ int i = 0;
+ for (RexLiteral field : row) {
+ json.writeFieldName(fields[i]);
+ writeLiteral(field, json);
+ i++;
+ }
+ json.writeEndObject();
+ }
+ json.writeEndArray();
+ json.flush();
+ return out.asParser().readValueAsTree();
+ }
+
+ private static void writeLiteral(RexLiteral literal, JsonOutput out) throws IOException {
+ switch (literal.getType().getSqlTypeName()) {
+ case BIGINT:
+ if (isLiteralNull(literal)) {
+ out.writeBigIntNull();
+ } else {
+ out.writeBigInt((((BigDecimal) literal.getValue()).setScale(0, BigDecimal.ROUND_HALF_UP)).longValue());
+ }
+ return;
+
+ case BOOLEAN:
+ if (isLiteralNull(literal)) {
+ out.writeBooleanNull();
+ } else {
+ out.writeBoolean((Boolean) literal.getValue());
+ }
+ return;
+
+ case CHAR:
+ if (isLiteralNull(literal)) {
+ out.writeVarcharNull();
+ } else {
+ // Since Calcite treats string literals as fixed char and adds trailing spaces to the strings to make them the
+ // same length, here we do an rtrim() to get the string without the trailing spaces. If we don't rtrim, the comparison
+ // with Drill's varchar column values would not return a match.
+ // TODO: However, note that if the user had explicitly added spaces in the string literals then even those would get
+ // trimmed, so this exposes another issue that needs to be resolved.
+ out.writeVarChar(((NlsString) literal.getValue()).rtrim().getValue());
+ }
+ return;
+
+ case DOUBLE:
+ if (isLiteralNull(literal)) {
+ out.writeDoubleNull();
+ } else {
+ out.writeDouble(((BigDecimal) literal.getValue()).doubleValue());
+ }
+ return;
+
+ case FLOAT:
+ if (isLiteralNull(literal)) {
+ out.writeFloatNull();
+ } else {
+ out.writeFloat(((BigDecimal) literal.getValue()).floatValue());
+ }
+ return;
+
+ case INTEGER:
+ if (isLiteralNull(literal)) {
+ out.writeIntNull();
+ } else {
+ out.writeInt((((BigDecimal) literal.getValue()).setScale(0, BigDecimal.ROUND_HALF_UP)).intValue());
+ }
+ return;
+
+ case DECIMAL:
+ if (isLiteralNull(literal)) {
+ out.writeDoubleNull();
+ } else {
+ out.writeDouble(((BigDecimal) literal.getValue()).doubleValue());
+ }
+ logger.warn("Converting exact decimal into approximate decimal. Should be fixed once full decimal support is implemented.");
+ return;
+
+ case VARCHAR:
+ if (isLiteralNull(literal)) {
+ out.writeVarcharNull();
+ } else {
+ out.writeVarChar(((NlsString) literal.getValue()).getValue());
+ }
+ return;
+
+ case SYMBOL:
+ if (isLiteralNull(literal)) {
+ out.writeVarcharNull();
+ } else {
+ out.writeVarChar(literal.getValue().toString());
+ }
+ return;
+
+ case DATE:
+ if (isLiteralNull(literal)) {
+ out.writeDateNull();
+ } else {
+ out.writeDate(new DateTime(literal.getValue()));
+ }
+ return;
+
+ case TIME:
+ if (isLiteralNull(literal)) {
+ out.writeTimeNull();
+ } else {
+ out.writeTime(new DateTime(literal.getValue()));
+ }
+ return;
+
+ case TIMESTAMP:
+ if (isLiteralNull(literal)) {
+ out.writeTimestampNull();
+ } else {
+ out.writeTimestamp(new DateTime(literal.getValue()));
+ }
+ return;
+
+ case INTERVAL_YEAR_MONTH:
+ if (isLiteralNull(literal)) {
+ out.writeIntervalNull();
+ } else {
+ int months = ((BigDecimal) (literal.getValue())).intValue();
+ out.writeInterval(new Period().plusMonths(months));
+ }
+ return;
+
+ case INTERVAL_DAY_TIME:
+ if (isLiteralNull(literal)) {
+ out.writeIntervalNull();
+ } else {
+ long millis = ((BigDecimal) (literal.getValue())).longValue();
+ int days = (int) (millis / DateTimeConstants.MILLIS_PER_DAY);
+ millis = millis - (days * DateTimeConstants.MILLIS_PER_DAY);
+ out.writeInterval(new Period().plusDays(days).plusMillis((int) millis));
+ }
+ return;
+
+ case NULL:
+ out.writeUntypedNull();
+ return;
+
+ case ANY:
+ default:
+ throw new UnsupportedOperationException(
+ String.format("Unable to convert the value of %s and type %s to a Drill constant expression.",
+ literal, literal.getType().getSqlTypeName()));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/33682be9/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
index 816506a..fb66acd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillValuesRel.java
@@ -17,282 +17,45 @@
*/
package org.apache.drill.exec.planner.logical;
-import static org.apache.drill.exec.planner.logical.DrillOptiq.isLiteralNull;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.util.GregorianCalendar;
import java.util.List;
import com.google.common.collect.ImmutableList;
-import org.apache.calcite.rel.AbstractRelNode;
-import org.apache.calcite.rel.RelWriter;
-import org.apache.calcite.rel.metadata.RelMetadataQuery;
-import org.apache.calcite.rel.type.RelDataTypeField;
-import org.apache.calcite.sql.SqlExplainLevel;
-import org.apache.calcite.sql.type.SqlTypeUtil;
-import org.apache.calcite.util.NlsString;
-import org.apache.calcite.util.Pair;
import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.logical.data.LogicalOperator;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.plan.RelOptCluster;
-import org.apache.calcite.plan.RelOptCost;
-import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexLiteral;
-import com.fasterxml.jackson.core.JsonLocation;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.util.TokenBuffer;
import org.apache.drill.common.logical.data.Values;
-import org.apache.drill.exec.vector.complex.fn.ExtendedJsonOutput;
-import org.apache.drill.exec.vector.complex.fn.JsonOutput;
-import org.joda.time.DateTime;
-import org.joda.time.Period;
+import org.apache.drill.exec.planner.common.DrillValuesRelBase;
/**
- * Values implemented in Drill.
+ * Logical Values implementation in Drill.
*/
-public class DrillValuesRel extends AbstractRelNode implements DrillRel {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillValuesRel.class);
- private static final ObjectMapper MAPPER = new ObjectMapper();
-
- private static final long MILLIS_IN_DAY = 1000*60*60*24;
-
- private final JSONOptions options;
- private final double rowCount;
-
- protected DrillValuesRel(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits) {
- super(cluster, traits);
- assert getConvention() == DRILL_LOGICAL;
- verifyRowType(tuples, rowType);
-
- this.rowType = rowType;
- this.rowCount = tuples.size();
-
- try{
- this.options = new JSONOptions(convertToJsonNode(rowType, tuples), JsonLocation.NA);
- }catch(IOException e){
- throw new DrillRuntimeException("Failure while attempting to encode ValuesRel in JSON.", e);
- }
-
- }
-
- private DrillValuesRel(RelOptCluster cluster, RelDataType rowType, RelTraitSet traits, JSONOptions options, double rowCount){
- super(cluster, traits);
- this.options = options;
- this.rowCount = rowCount;
- this.rowType = rowType;
- }
-
- private static void verifyRowType(final ImmutableList<ImmutableList<RexLiteral>> tuples, RelDataType rowType){
- for (List<RexLiteral> tuple : tuples) {
- assert (tuple.size() == rowType.getFieldCount());
-
- for (Pair<RexLiteral, RelDataTypeField> pair : Pair.zip(tuple, rowType.getFieldList())) {
- RexLiteral literal = pair.left;
- RelDataType fieldType = pair.right.getType();
-
- if ((!(RexLiteral.isNullLiteral(literal)))
- && (!(SqlTypeUtil.canAssignFrom(fieldType, literal.getType())))) {
- throw new AssertionError("to " + fieldType + " from " + literal);
- }
- }
- }
+public class DrillValuesRel extends DrillValuesRelBase implements DrillRel {
+ public DrillValuesRel(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits) {
+ super(cluster, rowType, tuples, traits);
}
- public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
- return planner.getCostFactory().makeCost(this.rowCount, 1.0d, 0.0d);
+ public DrillValuesRel(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits, JSONOptions content) {
+ super(cluster, rowType, tuples, traits, content);
}
@Override
public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
assert inputs.isEmpty();
- return new DrillValuesRel(getCluster(), rowType, traitSet, options, rowCount);
+ return new DrillValuesRel(getCluster(), rowType, tuples, traitSet, content);
}
@Override
public LogicalOperator implement(DrillImplementor implementor) {
return Values.builder()
- .content(options.asNode())
+ .content(content.asNode())
.build();
}
- public JSONOptions getTuplesAsJsonOptions() throws IOException {
- return options;
- }
-
- public double estimateRowCount(RelMetadataQuery mq) {
- return rowCount;
- }
-
- public RelWriter explainTerms(RelWriter pw) {
- return super.explainTerms(pw)
- .itemIf("type", this.rowType, pw.getDetailLevel() == SqlExplainLevel.DIGEST_ATTRIBUTES)
- .itemIf("type", this.rowType.getFieldList(), pw.nest())
- .itemIf("tuplesCount", rowCount, pw.getDetailLevel() != SqlExplainLevel.ALL_ATTRIBUTES)
- .itemIf("tuples", options.asNode(), pw.getDetailLevel() == SqlExplainLevel.ALL_ATTRIBUTES);
- }
- private static JsonNode convertToJsonNode(RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples) throws IOException{
- TokenBuffer out = new TokenBuffer(MAPPER.getFactory().getCodec(), false);
- JsonOutput json = new ExtendedJsonOutput(out);
- json.writeStartArray();
- String[] fields = rowType.getFieldNames().toArray(new String[rowType.getFieldCount()]);
-
- for(List<RexLiteral> row : tuples){
- json.writeStartObject();
- int i =0;
- for(RexLiteral field : row){
- json.writeFieldName(fields[i]);
- writeLiteral(field, json);
- i++;
- }
- json.writeEndObject();
- }
- json.writeEndArray();
- json.flush();
- return out.asParser().readValueAsTree();
- }
-
-
- private static void writeLiteral(RexLiteral literal, JsonOutput out) throws IOException{
-
- switch(literal.getType().getSqlTypeName()){
- case BIGINT:
- if (isLiteralNull(literal)) {
- out.writeBigIntNull();
- }else{
- out.writeBigInt((((BigDecimal) literal.getValue()).setScale(0, BigDecimal.ROUND_HALF_UP)).longValue());
- }
- return;
-
- case BOOLEAN:
- if (isLiteralNull(literal)) {
- out.writeBooleanNull();
- }else{
- out.writeBoolean((Boolean) literal.getValue());
- }
- return;
-
- case CHAR:
- if (isLiteralNull(literal)) {
- out.writeVarcharNull();
- }else{
- // Since Calcite treats string literals as fixed char and adds trailing spaces to the strings to make them the
- // same length, here we do an rtrim() to get the string without the trailing spaces. If we don't rtrim, the comparison
- // with Drill's varchar column values would not return a match.
- // TODO: However, note that if the user had explicitly added spaces in the string literals then even those would get
- // trimmed, so this exposes another issue that needs to be resolved.
- out.writeVarChar(((NlsString)literal.getValue()).rtrim().getValue());
- }
- return ;
-
- case DOUBLE:
- if (isLiteralNull(literal)){
- out.writeDoubleNull();
- }else{
- out.writeDouble(((BigDecimal) literal.getValue()).doubleValue());
- }
- return;
-
- case FLOAT:
- if (isLiteralNull(literal)) {
- out.writeFloatNull();
- }else{
- out.writeFloat(((BigDecimal) literal.getValue()).floatValue());
- }
- return;
-
- case INTEGER:
- if (isLiteralNull(literal)) {
- out.writeIntNull();
- }else{
- out.writeInt((((BigDecimal) literal.getValue()).setScale(0, BigDecimal.ROUND_HALF_UP)).intValue());
- }
- return;
-
- case DECIMAL:
- if (isLiteralNull(literal)) {
- out.writeDoubleNull();
- }else{
- out.writeDouble(((BigDecimal) literal.getValue()).doubleValue());
- }
- logger.warn("Converting exact decimal into approximate decimal. Should be fixed once decimal is implemented.");
- return;
-
- case VARCHAR:
- if (isLiteralNull(literal)) {
- out.writeVarcharNull();
- }else{
- out.writeVarChar( ((NlsString)literal.getValue()).getValue());
- }
- return;
-
- case SYMBOL:
- if (isLiteralNull(literal)) {
- out.writeVarcharNull();
- }else{
- out.writeVarChar(literal.getValue().toString());
- }
- return;
-
- case DATE:
- if (isLiteralNull(literal)) {
- out.writeDateNull();
- }else{
- out.writeDate(new DateTime((GregorianCalendar)literal.getValue()));
- }
- return;
-
- case TIME:
- if (isLiteralNull(literal)) {
- out.writeTimeNull();
- }else{
- out.writeTime(new DateTime((GregorianCalendar)literal.getValue()));
- }
- return;
-
- case TIMESTAMP:
- if (isLiteralNull(literal)) {
- out.writeTimestampNull();
- }else{
- out.writeTimestamp(new DateTime((GregorianCalendar)literal.getValue()));
- }
- return;
-
- case INTERVAL_YEAR_MONTH:
- if (isLiteralNull(literal)) {
- out.writeIntervalNull();
- }else{
- int months = ((BigDecimal) (literal.getValue())).intValue();
- out.writeInterval(new Period().plusMonths(months));
- }
- return;
-
- case INTERVAL_DAY_TIME:
- if (isLiteralNull(literal)) {
- out.writeIntervalNull();
- }else{
- long millis = ((BigDecimal) (literal.getValue())).longValue();
- int days = (int) (millis/MILLIS_IN_DAY);
- millis = millis - (days * MILLIS_IN_DAY);
- out.writeInterval(new Period().plusDays(days).plusMillis( (int) millis));
- }
- return;
-
- case NULL:
- out.writeUntypedNull();
- return;
-
- case ANY:
- default:
- throw new UnsupportedOperationException(String.format("Unable to convert the value of %s and type %s to a Drill constant expression.", literal, literal.getType().getSqlTypeName()));
- }
- }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/33682be9/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrel.java
index cdb4787..095519f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrel.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,38 +18,39 @@
package org.apache.drill.exec.planner.physical;
import java.io.IOException;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import com.google.common.collect.ImmutableList;
import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelTraitSet;
-import org.apache.calcite.rel.AbstractRelNode;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rex.RexLiteral;
import org.apache.drill.common.JSONOptions;
import org.apache.drill.exec.physical.base.PhysicalOperator;
import org.apache.drill.exec.physical.config.Values;
+import org.apache.drill.exec.planner.common.DrillValuesRelBase;
import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import com.google.common.collect.Iterators;
-
-public class ValuesPrel extends AbstractRelNode implements Prel {
-
- @SuppressWarnings("unused")
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValuesPrel.class);
+/**
+ * Physical Values implementation in Drill.
+ */
+public class ValuesPrel extends DrillValuesRelBase implements Prel {
- private JSONOptions content;
+ public ValuesPrel(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits) {
+ super(cluster, rowType, tuples, traits);
+ }
- public ValuesPrel(RelOptCluster cluster, RelTraitSet traitSet, RelDataType rowType, JSONOptions content) {
- super(cluster, traitSet);
- this.rowType = rowType;
- this.content = content;
+ public ValuesPrel(RelOptCluster cluster, RelDataType rowType, ImmutableList<ImmutableList<RexLiteral>> tuples, RelTraitSet traits, JSONOptions content) {
+ super(cluster, rowType, tuples, traits, content);
}
@Override
public Iterator<Prel> iterator() {
- return Iterators.emptyIterator();
+ return Collections.emptyIterator();
}
@Override
@@ -59,7 +60,7 @@ public class ValuesPrel extends AbstractRelNode implements Prel {
@Override
public Prel copy(RelTraitSet traitSet, List<RelNode> inputs) {
- return new ValuesPrel(getCluster(), traitSet, rowType, content);
+ return new ValuesPrel(getCluster(), rowType, tuples, traitSet, content);
}
@Override
@@ -87,5 +88,4 @@ public class ValuesPrel extends AbstractRelNode implements Prel {
return false;
}
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/33682be9/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
index 85d568f..d1f2e28 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ValuesPrule.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,16 +17,12 @@
*/
package org.apache.drill.exec.planner.physical;
-import java.io.IOException;
-
import org.apache.calcite.plan.RelOptRule;
import org.apache.drill.exec.planner.logical.DrillValuesRel;
import org.apache.drill.exec.planner.logical.RelOptHelper;
-import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
public class ValuesPrule extends RelOptRule {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValuesPrule.class);
public static final ValuesPrule INSTANCE = new ValuesPrule();
@@ -37,13 +33,7 @@ public class ValuesPrule extends RelOptRule {
@Override
public void onMatch(final RelOptRuleCall call) {
final DrillValuesRel rel = (DrillValuesRel) call.rel(0);
- try{
- call.transformTo(new ValuesPrel(rel.getCluster(), rel.getTraitSet().plus(Prel.DRILL_PHYSICAL), rel.getRowType(), rel.getTuplesAsJsonOptions()));
- }catch(IOException e){
- logger.warn("Failure while converting JSONOptions.", e);
- }
+ call.transformTo(new ValuesPrel(rel.getCluster(), rel.getRowType(), rel.getTuples(), rel.getTraitSet().plus(Prel.DRILL_PHYSICAL), rel.getContent()));
}
-
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/33682be9/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
index 9046df6..924486f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java
@@ -1180,4 +1180,15 @@ public class TestUnionAll extends BaseTestQuery{
}
}
+ @Test // DRILL-5130
+ public void testUnionAllWithValues() throws Exception {
+ testBuilder()
+ .sqlQuery("values('A') union all values('B')")
+ .unOrdered()
+ .baselineColumns("EXPR$0")
+ .baselineValues("A")
+ .baselineValues("B")
+ .go();
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/33682be9/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
index 22826b4..aff2a60 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/filter/TestLargeInClause.java
@@ -87,7 +87,7 @@ public class TestLargeInClause extends BaseTestQuery {
testBuilder()
.sqlQuery(query)
- .ordered()
+ .unOrdered()
.baselineColumns("r_regionkey")
.baselineValues(1)
.baselineValues(2)
[4/6] drill git commit: DRILL-5590: Bugs in CSV field matching,
null columns
Posted by am...@apache.org.
DRILL-5590: Bugs in CSV field matching, null columns
Please see the problem and solution descriptions in DRILL-5590.
Also cleaned up some dead code left over from DRILL-5498.
close #855
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/dd55b5c4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/dd55b5c4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/dd55b5c4
Branch: refs/heads/master
Commit: dd55b5c49c8e3207400b99ea616a032bc6172988
Parents: 33682be
Author: Paul Rogers <pr...@maprtech.com>
Authored: Thu Jun 15 22:46:56 2017 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sat Jun 24 09:23:22 2017 -0700
----------------------------------------------------------------------
.../compliant/CompliantTextRecordReader.java | 73 ++------------------
.../easy/text/compliant/FieldVarCharOutput.java | 73 +++++++++++---------
.../text/compliant/RepeatedVarCharOutput.java | 5 +-
.../exec/store/easy/text/compliant/TestCsv.java | 22 ++++++
4 files changed, 68 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/dd55b5c4/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java
index 4a35c3b..7009584 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/CompliantTextRecordReader.java
@@ -17,15 +17,9 @@
*/
package org.apache.drill.exec.store.easy.text.compliant;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Maps;
-import com.univocity.parsers.common.TextParsingException;
-import io.netty.buffer.DrillBuf;
-
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
-import java.util.Map;
import javax.annotation.Nullable;
@@ -36,16 +30,16 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.AbstractRecordReader;
import org.apache.drill.exec.store.dfs.DrillFileSystem;
-import org.apache.drill.exec.util.CallBack;
-import org.apache.drill.exec.vector.ValueVector;
import org.apache.hadoop.mapred.FileSplit;
import com.google.common.base.Predicate;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
-import org.apache.drill.exec.expr.TypeHelper;
+import com.univocity.parsers.common.TextParsingException;
+
+import io.netty.buffer.DrillBuf;
// New text reader, complies with the RFC 4180 standard for text/csv files
public class CompliantTextRecordReader extends AbstractRecordReader {
@@ -255,63 +249,4 @@ public class CompliantTextRecordReader extends AbstractRecordReader {
logger.warn("Exception while closing stream.", e);
}
}
-
- /**
- * TextRecordReader during its first phase read to extract header should pass its own
- * OutputMutator to avoid reshaping query output.
- * This class provides OutputMutator for header extraction.
- */
- private class HeaderOutputMutator implements OutputMutator {
- private final Map<String, ValueVector> fieldVectorMap = Maps.newHashMap();
-
- @SuppressWarnings("resource")
- @Override
- public <T extends ValueVector> T addField(MaterializedField field, Class<T> clazz) throws SchemaChangeException {
- ValueVector v = fieldVectorMap.get(field);
- if (v == null || v.getClass() != clazz) {
- // Field does not exist add it to the map
- v = TypeHelper.getNewVector(field, oContext.getAllocator());
- if (!clazz.isAssignableFrom(v.getClass())) {
- throw new SchemaChangeException(String.format(
- "Class %s was provided, expected %s.", clazz.getSimpleName(), v.getClass().getSimpleName()));
- }
- fieldVectorMap.put(field.getPath(), v);
- }
- return clazz.cast(v);
- }
-
- @Override
- public void allocate(int recordCount) {
- //do nothing for now
- }
-
- @Override
- public boolean isNewSchema() {
- return false;
- }
-
- @Override
- public DrillBuf getManagedBuffer() {
- return null;
- }
-
- @Override
- public CallBack getCallBack() {
- return null;
- }
-
- /**
- * Since this OutputMutator is passed by TextRecordReader to get the header out
- * the mutator might not get cleaned up elsewhere. TextRecordReader will call
- * this method to clear any allocations
- */
- public void close() {
- for (final ValueVector v : fieldVectorMap.values()) {
- v.clear();
- }
- fieldVectorMap.clear();
- }
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/drill/blob/dd55b5c4/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/FieldVarCharOutput.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/FieldVarCharOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/FieldVarCharOutput.java
index 494c593..b8343d1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/FieldVarCharOutput.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/FieldVarCharOutput.java
@@ -17,8 +17,16 @@
*/
package org.apache.drill.exec.store.easy.text.compliant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
import org.apache.drill.common.exceptions.UserException;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.map.CaseInsensitiveMap;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.exception.SchemaChangeException;
@@ -26,12 +34,6 @@ import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.vector.VarCharVector;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Arrays;
-import java.util.List;
-
/**
* Class is responsible for generating record batches for text file inputs. We generate
* a record batch with a set of varchar vectors. A varchar vector contains all the field
@@ -61,8 +63,9 @@ class FieldVarCharOutput extends TextOutput {
private boolean rowHasData= false;
private static final int MAX_FIELD_LENGTH = 1024 * 64;
private int recordCount = 0;
- private int batchIndex = 0;
private int maxField = 0;
+ private int[] nullCols;
+ private byte nullValue[] = new byte[0];
/**
* We initialize and add the varchar vector for each incoming field in this
@@ -77,6 +80,7 @@ class FieldVarCharOutput extends TextOutput {
int totalFields = fieldNames.length;
List<String> outputColumns = new ArrayList<>(Arrays.asList(fieldNames));
+ List<Integer> nullColumns = new ArrayList<>();
if (isStarQuery) {
maxField = totalFields - 1;
@@ -84,11 +88,14 @@ class FieldVarCharOutput extends TextOutput {
Arrays.fill(selectedFields, true);
} else {
List<Integer> columnIds = new ArrayList<Integer>();
- String pathStr;
- int index;
+ Map<String, Integer> headers = CaseInsensitiveMap.newHashMap();
+ for (int i = 0; i < fieldNames.length; i++) {
+ headers.put(fieldNames[i], i);
+ }
for (SchemaPath path : columns) {
- pathStr = path.getRootSegment().getPath();
+ int index;
+ String pathStr = path.getRootSegment().getPath();
if (pathStr.equals(COL_NAME) && path.getRootSegment().getChild() != null) {
//TODO: support both field names and columns index along with predicate pushdown
throw UserException
@@ -98,12 +105,15 @@ class FieldVarCharOutput extends TextOutput {
.addContext("column index", path.getRootSegment().getChild())
.build(logger);
} else {
- index = outputColumns.indexOf(pathStr);
- if (index < 0) {
+ Integer value = headers.get(pathStr);
+ if (value == null) {
// found col that is not a part of fieldNames, add it
// this col might be part of some another scanner
index = totalFields++;
outputColumns.add(pathStr);
+ nullColumns.add(index);
+ } else {
+ index = value;
}
}
columnIds.add(index);
@@ -128,6 +138,12 @@ class FieldVarCharOutput extends TextOutput {
this.fieldBytes = new byte[MAX_FIELD_LENGTH];
+ // Keep track of the null columns to be filled in.
+
+ nullCols = new int[nullColumns.size()];
+ for (int i = 0; i < nullCols.length; i++) {
+ nullCols[i] = nullColumns.get(i);
+ }
}
/**
@@ -135,11 +151,10 @@ class FieldVarCharOutput extends TextOutput {
*/
@Override
public void startBatch() {
- this.recordCount = 0;
- this.batchIndex = 0;
- this.currentFieldIndex= -1;
- this.collect = true;
- this.fieldOpen = false;
+ recordCount = 0;
+ currentFieldIndex= -1;
+ collect = true;
+ fieldOpen = false;
}
@Override
@@ -173,7 +188,7 @@ class FieldVarCharOutput extends TextOutput {
public boolean endField() {
fieldOpen = false;
- if(collect) {
+ if (collect) {
assert currentVector != null;
currentVector.getMutator().setSafe(recordCount, fieldBytes, 0, currentDataPointer);
}
@@ -192,25 +207,20 @@ class FieldVarCharOutput extends TextOutput {
@Override
public void finishRecord() {
- if(fieldOpen){
+ if (fieldOpen){
endField();
}
+ // Fill in null (really empty) values.
+
+ for (int i = 0; i < nullCols.length; i++) {
+ vectors[nullCols[i]].getMutator().setSafe(recordCount, nullValue, 0, 0);
+ }
recordCount++;
}
- // Sets the record count in this batch within the value vector
@Override
- public void finishBatch() {
- batchIndex++;
-
- for (int i = 0; i <= maxField; i++) {
- if (this.vectors[i] != null) {
- this.vectors[i].getMutator().setValueCount(batchIndex);
- }
- }
-
- }
+ public void finishBatch() { }
@Override
public long getRecordCount() {
@@ -221,5 +231,4 @@ class FieldVarCharOutput extends TextOutput {
public boolean rowHasData() {
return this.rowHasData;
}
-
- }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/dd55b5c4/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java
index eda2feb..156d6c2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/text/compliant/RepeatedVarCharOutput.java
@@ -345,9 +345,6 @@ class RepeatedVarCharOutput extends TextOutput {
return out;
}
- // Sets the record count in this batch within the value vector
@Override
- public void finishBatch() {
- mutator.setValueCount(batchIndex);
- }
+ public void finishBatch() { }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/dd55b5c4/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
index 7d38cf9..c18adc9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsv.java
@@ -135,6 +135,28 @@ public class TestCsv extends ClusterTest {
.verifyAndClear(actual);
}
+ // Test fix for DRILL-5590
+ @Test
+ public void testCsvHeadersCaseInsensitive() throws IOException {
+ String fileName = "case2.csv";
+ buildFile(fileName, validHeaders);
+ String sql = "SELECT A, b, C FROM `dfs.data`.`" + fileName + "`";
+ RowSet actual = client.queryBuilder().sql(sql).rowSet();
+
+ BatchSchema expectedSchema = new SchemaBuilder()
+ .add("A", MinorType.VARCHAR)
+ .add("b", MinorType.VARCHAR)
+ .add("C", MinorType.VARCHAR)
+ .build();
+ assertEquals(expectedSchema, actual.batchSchema());
+
+ RowSet expected = new RowSetBuilder(client.allocator(), expectedSchema)
+ .add("10", "foo", "bar")
+ .build();
+ new RowSetComparison(expected)
+ .verifyAndClear(actual);
+ }
+
private String makeStatement(String fileName) {
return "SELECT * FROM `dfs.data`.`" + fileName + "`";
}
[2/6] drill git commit: DRILL-5568: Include hadoop-common jars inside
drill-jdbc-all.jar 1) Introduce a new file inside exec/jdbc-all resource
package which contains the property key/value pair for prefix 2) At build
time based upon the profile choosen t
Posted by am...@apache.org.
DRILL-5568: Include hadoop-common jars inside drill-jdbc-all.jar 1) Introduce a new file inside exec/jdbc-all resource package which contains the property key/value pair for prefix 2) At build time based upon the profile choosen this property value is set inside the file 3) It is later consumed by SecurityConfiguration class to rename classpath for classes inside hadoop package.
DRILL-5568: Code review changes
close apache/drill#849
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/886ccdc1
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/886ccdc1
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/886ccdc1
Branch: refs/heads/master
Commit: 886ccdc1a850a0dc054a37797f45cba409b1d0a4
Parents: 964a947
Author: Sorabh Hamirwasia <sh...@maprtech.com>
Authored: Mon Jun 5 13:45:27 2017 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sat Jun 24 09:17:03 2017 -0700
----------------------------------------------------------------------
.../rpc/security/SecurityConfiguration.java | 68 ++++
.../rpc/security/kerberos/KerberosFactory.java | 3 +-
.../exec/rpc/security/plain/PlainFactory.java | 3 +-
exec/jdbc-all/pom.xml | 310 ++++++++++++++++++-
exec/jdbc-all/src/main/resources/profile.props | 18 ++
.../apache/drill/jdbc/impl/DrillFactory.java | 6 +-
6 files changed, 397 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/886ccdc1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/SecurityConfiguration.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/SecurityConfiguration.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/SecurityConfiguration.java
new file mode 100644
index 0000000..275fd84
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/SecurityConfiguration.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.rpc.security;
+
+import com.google.common.base.Strings;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+
+import java.io.InputStream;
+import java.util.Properties;
+
+
+public class SecurityConfiguration extends Configuration {
+ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SecurityConfiguration.class);
+
+ static {
+ Properties prop = new Properties();
+ try {
+ // Load the properties file containing the namespace prefix based on profile used
+ // to build jdbc-all package. This prefix is used to add on certain Hadoop classes class path.
+ final InputStream inputStream = SecurityConfiguration.class.getClassLoader().getResourceAsStream("profile.props");
+
+ // For null inputStream prop.load() throws NullPointerException
+ // Get the property value and set it in system property
+ prop.load(inputStream);
+ System.setProperty("drill.security.namespacePrefix", prop.getProperty("package.namespace.prefix").trim());
+
+ } catch (Exception ex) {
+ // Ignore the exception which means that property value will be null and is handled in consumer of System Property
+ }
+ }
+
+ public SecurityConfiguration() {
+ super();
+ updateGroupMapping();
+ }
+
+ /**
+ * Update the GroupMapping class name to add namespace prefix retrieved from System Property. This is needed since
+ * in drill-jdbc-all jar we are packaging hadoop dependencies under that namespace. This will help application
+ * using this jar as driver to avoid conflict with it's own hadoop dependency if any. The property is needed only
+ * when Hadoop classes are relocated to different namespace which is done inside jdbc-all package. For normal build
+ * this property is not required as Hadoop classes will be used normally.
+ */
+ private void updateGroupMapping() {
+ final String originalClassName = get(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING);
+ final String profilePrefix = System.getProperty("drill.security.namespacePrefix");
+
+ if (!Strings.isNullOrEmpty(profilePrefix)) {
+ set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, profilePrefix + originalClassName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/886ccdc1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java
index e14d411..cc4146e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/kerberos/KerberosFactory.java
@@ -22,6 +22,7 @@ import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.rpc.security.AuthenticatorFactory;
import org.apache.drill.exec.rpc.security.FastSaslClientFactory;
import org.apache.drill.exec.rpc.security.FastSaslServerFactory;
+import org.apache.drill.exec.rpc.security.SecurityConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.HadoopKerberosName;
@@ -56,7 +57,7 @@ public class KerberosFactory implements AuthenticatorFactory {
@Override
public UserGroupInformation createAndLoginUser(final Map<String, ?> properties) throws IOException {
- final Configuration conf = new Configuration();
+ final Configuration conf = new SecurityConfiguration();
conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
UserGroupInformation.AuthenticationMethod.KERBEROS.toString());
UserGroupInformation.setConfiguration(conf);
http://git-wip-us.apache.org/repos/asf/drill/blob/886ccdc1/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/plain/PlainFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/plain/PlainFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/plain/PlainFactory.java
index 4a0db95..d20faf0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/plain/PlainFactory.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/security/plain/PlainFactory.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.rpc.security.plain;
import org.apache.drill.common.config.DrillProperties;
import org.apache.drill.exec.rpc.security.AuthenticatorFactory;
import org.apache.drill.exec.rpc.security.FastSaslClientFactory;
+import org.apache.drill.exec.rpc.security.SecurityConfiguration;
import org.apache.drill.exec.rpc.user.security.UserAuthenticator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
@@ -58,7 +59,7 @@ public class PlainFactory implements AuthenticatorFactory {
@Override
public UserGroupInformation createAndLoginUser(Map<String, ?> properties) throws IOException {
- final Configuration conf = new Configuration();
+ final Configuration conf = new SecurityConfiguration();
UserGroupInformation.setConfiguration(conf);
try {
return UserGroupInformation.getCurrentUser();
http://git-wip-us.apache.org/repos/asf/drill/blob/886ccdc1/exec/jdbc-all/pom.xml
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/pom.xml b/exec/jdbc-all/pom.xml
index 17af111..238288e 100644
--- a/exec/jdbc-all/pom.xml
+++ b/exec/jdbc-all/pom.xml
@@ -81,10 +81,6 @@
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
@@ -184,6 +180,14 @@
</dependencies>
<build>
+
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
<plugins>
<plugin>
@@ -265,12 +269,12 @@
<!--dependencyReducedPomLocation>${project.build.directory}/generated/shade/dependency-reduced-pom.xml</dependencyReducedPomLocation-->
<minimizeJar>false</minimizeJar>
+ <!-- Exclude dependencies at artifact level. Format is "groupId:artifactId[[:type]:classifier]" -->
<artifactSet>
<includes>
<include>*:*</include>
</includes>
<excludes>
- <exclude>org.slf4j:jcl-over-slf4j</exclude>
<exclude>com.dyuproject.protostuff:*</exclude>
<exclude>org.apache.calcite:calcite-core</exclude>
<exclude>org.apache.calcite:calcite-linq4j</exclude>
@@ -305,7 +309,13 @@
<exclude>org.mockito:mockito-core</exclude>
<exclude>org.objenesis:objenesis</exclude>
<exclude>org.eclipse.jetty:*</exclude>
+ <exclude>javax.xml.bind:jaxb-api</exclude>
+ <exclude>javax.xml.stream:stax-api</exclude>
+ <exclude>javax.activation:activation</exclude>
+ <exclude>commons-cli:commons-cli</exclude>
<exclude>commons-io:commons-io</exclude>
+ <exclude>commons-beanutils:commons-beanutils-core:jar:*</exclude>
+ <exclude>commons-beanutils:commons-beanutils:jar:*</exclude>
</excludes>
</artifactSet>
<relocations>
@@ -367,12 +377,15 @@
<relocation><pattern>org.apache.xmlcommons.</pattern><shadedPattern>oadd.org.apache.xmlcommons.</shadedPattern></relocation>
<relocation><pattern>org.apache.xpath.</pattern><shadedPattern>oadd.org.apache.xpath.</shadedPattern></relocation>
<relocation><pattern>org.apache.zookeeper.</pattern><shadedPattern>oadd.org.apache.zookeeper.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.hadoop.</pattern><shadedPattern>oadd.org.apache.hadoop.</shadedPattern></relocation>
</relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>drill-module.conf</resource>
</transformer>
</transformers>
+
+ <!-- Remove the particular directory or class level dependency from final jar -->
<filters>
<filter>
<artifact>*:*</artifact>
@@ -424,6 +437,28 @@
<exclude>org/apache/drill/exec/server/rest/**</exclude>
<exclude>org/apache/drill/exec/rpc/data/**</exclude>
<exclude>org/apache/drill/exec/rpc/control/**</exclude>
+ <exclude>org/apache/drill/exec/work/**</exclude>
+ <exclude>org/apache/hadoop/crypto/**</exclude>
+ <exclude>org/apache/hadoop/ha/**</exclude>
+ <exclude>org/apache/hadoop/http/**</exclude>
+ <exclude>org/apache/hadoop/ipc/**</exclude>
+ <exclude>org/apache/hadoop/jmx/**</exclude>
+ <exclude>org/apache/hadoop/log/**</exclude>
+ <exclude>org/apache/hadoop/metrics/**</exclude>
+ <exclude>org/apache/hadoop/net/**</exclude>
+ <exclude>org/apache/hadoop/record/**</exclude>
+ <exclude>org/apache/hadoop/service/**</exclude>
+ <exclude>org/apache/hadoop/tracing/**</exclude>
+ <exclude>org/apache/hadoop/tools/**</exclude>
+ <exclude>org/apache/hadoop/yarn/**</exclude>
+ <exclude>org/apache/commons/pool2/**</exclude>
+ <exclude>org/apache/http/**</exclude>
+ <exclude>org/apache/directory/**</exclude>
+ <exclude>com/jcraft/**</exclude>
+ <exclude>**/mapr/**</exclude>
+ <exclude>org/yaml/**</exclude>
+ <exclude>hello/**</exclude>
+ <exclude>webapps/**</exclude>
</excludes>
</filter>
</filters>
@@ -450,7 +485,7 @@
This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users.
</message>
- <maxsize>22000000</maxsize>
+ <maxsize>29000000</maxsize>
<minsize>15000000</minsize>
<files>
<file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file>
@@ -483,6 +518,269 @@
<profiles>
<profile>
+ <id>default</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <package.namespace.prefix>oadd.</package.namespace.prefix>
+ </properties>
+ </profile>
+ <profile>
+ <id>mapr</id>
+ <properties>
+ <package.namespace.prefix></package.namespace.prefix>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>enforce-jdbc-jar-compactness</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <rules>
+ <requireFilesSize>
+ <message>
+
+ The file drill-jdbc-all-${project.version}.jar is outside the expected size range.
+
+ This is likely due to you adding new dependencies to a java-exec and not updating the excludes in this module. This is important as it minimizes the size of the dependency of Drill application users.
+
+ </message>
+ <maxsize>29000000</maxsize>
+ <minsize>15000000</minsize>
+ <files>
+ <file>${project.build.directory}/drill-jdbc-all-${project.version}.jar</file>
+ </files>
+ </requireFilesSize>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.4.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <!-- TODO DRILL-4336: try to move the dependencyReducedPom out of the default location (the module root).
+ Putting it here caused the target directory to be run as a submodule (oddly
+ only when trying to run the maven release goal) -->
+ <!--dependencyReducedPomLocation>${project.build.directory}/dependency-reduced-pom.xml</dependencyReducedPomLocation-->
+ <!-- TODO DRILL-4336: an attempt to fix the issue by moving the file elsewhere, had issues executing
+ but may be able to be modified to to fix the issue-->
+ <!--dependencyReducedPomLocation>${project.build.directory}/generated/shade/dependency-reduced-pom.xml</dependencyReducedPomLocation-->
+ <minimizeJar>false</minimizeJar>
+
+ <!-- Exclude dependencies at artifact level. Format is "groupId:artifactId[[:type]:classifier]" -->
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ <excludes>
+ <exclude>org.slf4j:jcl-over-slf4j</exclude>
+ <exclude>com.dyuproject.protostuff:*</exclude>
+ <exclude>org.apache.calcite:calcite-core</exclude>
+ <exclude>org.apache.calcite:calcite-linq4j</exclude>
+ <exclude>org.pentaho:*</exclude>
+ <exclude>org.msgpack:*</exclude>
+ <exclude>xerces:*</exclude>
+ <exclude>xalan:*</exclude>
+ <exclude>org.apache.avro:*</exclude>
+ <exclude>org.mongodb:*</exclude>
+ <exclude>com.googlecode.json-simple:*</exclude>
+ <exclude>dom4j:*</exclude>
+ <exclude>org.hibernate:*</exclude>
+ <exclude>antlr:*</exclude>
+ <exclude>org.ow2.asm:*</exclude>
+ <exclude>com.univocity:*</exclude>
+ <exclude>net.sf.jpam:*</exclude>
+ <exclude>com.twitter:*</exclude>
+ <exclude>org.apache.parquet:*</exclude>
+ <exclude>javax.inject:*</exclude>
+ <exclude>com.beust:*</exclude>
+ <exclude>jline:*</exclude>
+ <exclude>io.netty:netty:jar:3.7.0.Final</exclude>
+ <exclude>org.xerial.snappy:*</exclude>
+ <exclude>org.apache.avro:*</exclude>
+ <exclude>org.tukaani:*</exclude>
+ <exclude>org.apache.velocity:*</exclude>
+ <exclude>net.hydromatic:linq4j</exclude>
+ <exclude>org.codehaus.janino:*</exclude>
+ <exclude>org.mortbay.jetty:*</exclude>
+ <exclude>org.slf4j:jul-to-slf4j</exclude>
+ <exclude>org.slf4j:log4j-over-slf4j</exclude>
+ <exclude>org.hamcrest:hamcrest-core</exclude>
+ <exclude>org.mockito:mockito-core</exclude>
+ <exclude>org.objenesis:objenesis</exclude>
+ <exclude>org.eclipse.jetty:*</exclude>
+ <exclude>org.apache.hadoop:*</exclude>
+ <exclude>javax.xml.bind:jaxb-api</exclude>
+ <exclude>javax.xml.stream:stax-api</exclude>
+ <exclude>javax.activation:activation</exclude>
+ <exclude>commons-cli:commons-cli</exclude>
+ <exclude>commons-io:commons-io</exclude>
+ <exclude>commons-beanutils:commons-beanutils-core:jar:*</exclude>
+ <exclude>commons-beanutils:commons-beanutils:jar:*</exclude>
+ </excludes>
+ </artifactSet>
+ <relocations>
+ <!-- Relocate Drill classes to minimize classloader hell. -->
+ <relocation><pattern>org.apache.drill.exec.</pattern><shadedPattern>oadd.org.apache.drill.exec.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.drill.common.</pattern><shadedPattern>oadd.org.apache.drill.common.</shadedPattern></relocation>
+
+ <!-- Move dependencies out of path -->
+ <relocation><pattern>antlr.</pattern><shadedPattern>oadd.antlr.</shadedPattern></relocation>
+ <relocation><pattern>antlr.</pattern><shadedPattern>oadd.antlr.</shadedPattern></relocation>
+ <relocation><pattern>io.</pattern><shadedPattern>oadd.io.</shadedPattern></relocation>
+ <relocation><pattern>javacc.</pattern><shadedPattern>oadd.javacc.</shadedPattern></relocation>
+ <relocation><pattern>java_cup.</pattern><shadedPattern>oadd.java_cup.</shadedPattern></relocation>
+ <relocation><pattern>javassist.</pattern><shadedPattern>oadd.javassist.</shadedPattern></relocation>
+ <relocation><pattern>jline.</pattern><shadedPattern>oadd.jline.</shadedPattern></relocation>
+ <relocation><pattern>license.</pattern><shadedPattern>oadd.license.</shadedPattern></relocation>
+ <relocation><pattern>net.</pattern><shadedPattern>oadd.net.</shadedPattern></relocation>
+ <relocation><pattern>parquet.</pattern><shadedPattern>oadd.parquet.</shadedPattern></relocation>
+ <relocation><pattern>test.</pattern><shadedPattern>oadd.test.</shadedPattern></relocation>
+ <relocation><pattern>trax.</pattern><shadedPattern>oadd.trax.</shadedPattern></relocation>
+ <relocation><pattern>org.antlr.</pattern><shadedPattern>oadd.org.antlr.</shadedPattern></relocation>
+ <relocation><pattern>org.codehaus.</pattern><shadedPattern>oadd.org.codehaus.</shadedPattern></relocation>
+ <relocation><pattern>org.eigenbase.</pattern><shadedPattern>oadd.org.eigenbase.</shadedPattern></relocation>
+ <relocation><pattern>org.hamcrest.</pattern><shadedPattern>oadd.org.hamcrest.</shadedPattern></relocation>
+ <relocation><pattern>org.jboss.</pattern><shadedPattern>oadd.org.jboss.</shadedPattern></relocation>
+ <relocation><pattern>org.joda.</pattern><shadedPattern>oadd.org.joda.</shadedPattern></relocation>
+ <relocation><pattern>org.json.</pattern><shadedPattern>oadd.org.json.</shadedPattern></relocation>
+ <relocation><pattern>org.mockito.</pattern><shadedPattern>oadd.org.mockito.</shadedPattern></relocation>
+ <relocation><pattern>org.msgpack.</pattern><shadedPattern>oadd.org.msgpack.</shadedPattern></relocation>
+ <relocation><pattern>org.objectweb.</pattern><shadedPattern>oadd.org.objectweb.</shadedPattern></relocation>
+ <relocation><pattern>org.objensis.</pattern><shadedPattern>oadd.org.objensis.</shadedPattern></relocation>
+ <relocation><pattern>org.pentaho.</pattern><shadedPattern>oadd.org.pentaho.</shadedPattern></relocation>
+ <relocation><pattern>org.reflections.</pattern><shadedPattern>oadd.org.reflections.</shadedPattern></relocation>
+ <relocation><pattern>org.tukaani.</pattern><shadedPattern>oadd.org.tukaani.</shadedPattern></relocation>
+ <relocation><pattern>org.xerial.</pattern><shadedPattern>oadd.org.xerial.</shadedPattern></relocation>
+ <relocation><pattern>com.beust.</pattern><shadedPattern>oadd.com.beust.</shadedPattern></relocation>
+ <relocation><pattern>com.carrotsearch.</pattern><shadedPattern>oadd.com.carrotsearch.</shadedPattern></relocation>
+ <relocation><pattern>com.codahale.</pattern><shadedPattern>oadd.com.codahale.</shadedPattern></relocation>
+ <relocation><pattern>com.dyuproject.</pattern><shadedPattern>oadd.com.dyuproject.</shadedPattern></relocation>
+ <relocation><pattern>com.fasterxml.</pattern><shadedPattern>oadd.com.fasterxml.</shadedPattern></relocation>
+ <relocation><pattern>com.google.</pattern><shadedPattern>oadd.com.google.</shadedPattern></relocation>
+ <relocation><pattern>com.thoughtworks.</pattern><shadedPattern>oadd.com.thoughtworks.</shadedPattern></relocation>
+ <relocation><pattern>com.typesafe.</pattern><shadedPattern>oadd.com.typesafe.</shadedPattern></relocation>
+ <relocation><pattern>com.univocity.</pattern><shadedPattern>oadd.com.univocity.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.avro.</pattern><shadedPattern>oadd.org.apache.avro.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.bcel.</pattern><shadedPattern>oadd.org.apache.bcel.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.calcite.</pattern><shadedPattern>oadd.org.apache.calcite.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.commons.</pattern><shadedPattern>oadd.org.apache.commons.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.curator.</pattern><shadedPattern>oadd.org.apache.curator.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.html.</pattern><shadedPattern>oadd.org.apache.html.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.jute.</pattern><shadedPattern>oadd.org.apache.jute.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.log4j.</pattern><shadedPattern>oadd.org.apache.log4j.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.regexp.</pattern><shadedPattern>oadd.org.apache.regexp.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.velocity.</pattern><shadedPattern>oadd.org.apache.velocity.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.wml.</pattern><shadedPattern>oadd.org.apache.wml.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.xalan.</pattern><shadedPattern>oadd.org.apache.xalan.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.xerces.</pattern><shadedPattern>oadd.org.apache.xerces.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.xml.</pattern><shadedPattern>oadd.org.apache.xml.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.xmlcommons.</pattern><shadedPattern>oadd.org.apache.xmlcommons.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.xpath.</pattern><shadedPattern>oadd.org.apache.xpath.</shadedPattern></relocation>
+ <relocation><pattern>org.apache.zookeeper.</pattern><shadedPattern>oadd.org.apache.zookeeper.</shadedPattern></relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>drill-module.conf</resource>
+ </transformer>
+ </transformers>
+
+ <!-- Remove the particular directory or class level dependency from final jar -->
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>**/logback.xml</exclude>
+ <exclude>**/LICENSE.txt</exclude>
+ <exclude>**/*.java</exclude>
+ <exclude>META-INF/ASL2.0</exclude>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ <exclude>META-INF/drill-module-scan/**</exclude>
+ <exclude>META-INF/jboss-beans.xml</exclude>
+ <exclude>META-INF/license/**</exclude>
+ <exclude>META-INF/maven/**</exclude>
+ <exclude>META-INF/native/**</exclude>
+ <exclude>META-INF/services/com.fasterxml.*</exclude>
+ <exclude>META-INF/services/javax.ws.*</exclude>
+ <exclude>META-INF/**/*.properties</exclude>
+ <exclude>**/org.codehaus.commons.compiler.properties</exclude>
+ <exclude>**/*.SF</exclude>
+ <exclude>**/*.RSA</exclude>
+ <exclude>**/*.DSA</exclude>
+ <exclude>javax/*</exclude>
+ <exclude>javax/activation/**</exclude>
+ <exclude>javax/annotation-api/**</exclude>
+ <exclude>javax/inject/**</exclude>
+ <exclude>javax/servlet-api/**</exclude>
+ <exclude>javax/json/**</exclude>
+ <exclude>javax/ws/**</exclude>
+ <exclude>rest/**</exclude>
+ <exclude>*.tokens</exclude>
+ <exclude>codegen/**</exclude>
+ <exclude>bootstrap-storage-plugins.json</exclude>
+ <exclude>org/apache/parquet</exclude>
+ <exclude>com/google/common/math</exclude>
+ <exclude>com/google/common/net</exclude>
+ <exclude>com/google/common/primitives</exclude>
+ <exclude>com/google/common/reflect</exclude>
+ <exclude>com/google/common/util</exclude>
+ <exclude>com/google/common/cache</exclude>
+ <exclude>com/google/common/collect/Tree*</exclude>
+ <exclude>com/google/common/collect/Standard*</exclude>
+ <exclude>org/apache/drill/exec/expr/annotations/**</exclude>
+ <exclude>org/apache/drill/exec/expr/fn/**</exclude>
+ <exclude>org/apache/drill/exec/proto/beans/**</exclude>
+ <exclude>org/apache/drill/exec/compile/**</exclude>
+ <exclude>org/apache/drill/exec/planner/**</exclude>
+ <exclude>org/apache/drill/exec/physical/**</exclude>
+ <exclude>org/apache/drill/exec/store/**</exclude>
+ <exclude>org/apache/drill/exec/server/rest/**</exclude>
+ <exclude>org/apache/drill/exec/rpc/data/**</exclude>
+ <exclude>org/apache/drill/exec/rpc/control/**</exclude>
+ <exclude>org/apache/drill/exec/work/**</exclude>
+ <exclude>org/apache/hadoop/**</exclude>
+ <exclude>org/apache/commons/pool2/**</exclude>
+ <exclude>org/apache/http/**</exclude>
+ <exclude>org/apache/directory/**</exclude>
+ <exclude>com/jcraft/**</exclude>
+ <exclude>**/mapr/**</exclude>
+ <exclude>org/yaml/**</exclude>
+ <exclude>hello/**</exclude>
+ <exclude>webapps/**</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </plugin>
+ </plugins>
+
+ </build>
+ </profile>
+ <profile>
<id>apache-release</id>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/drill/blob/886ccdc1/exec/jdbc-all/src/main/resources/profile.props
----------------------------------------------------------------------
diff --git a/exec/jdbc-all/src/main/resources/profile.props b/exec/jdbc-all/src/main/resources/profile.props
new file mode 100644
index 0000000..4ed395a
--- /dev/null
+++ b/exec/jdbc-all/src/main/resources/profile.props
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+package.namespace.prefix=${package.namespace.prefix}
http://git-wip-us.apache.org/repos/asf/drill/blob/886ccdc1/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillFactory.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillFactory.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillFactory.java
index 0c3c3e8..15bc88b 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillFactory.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/impl/DrillFactory.java
@@ -18,13 +18,13 @@
package org.apache.drill.jdbc.impl;
-import java.sql.SQLException;
-import java.util.Properties;
-
import org.apache.calcite.avatica.AvaticaConnection;
import org.apache.calcite.avatica.AvaticaFactory;
import org.apache.calcite.avatica.UnregisteredDriver;
+import java.sql.SQLException;
+import java.util.Properties;
+
/**
* Partial implementation of {@link net.hydromatic.avatica.AvaticaFactory}