You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ph...@apache.org on 2018/05/22 03:34:11 UTC
[4/5] impala git commit: IMPALA-7019: Schedule EC as remote & disable
failed tests
IMPALA-7019: Schedule EC as remote & disable failed tests
This patch schedules HDFS EC files without considering locality. Failed
tests are disabled and a jenkins build should succeed with export
ERASURE_COINDG=true.
Testing: It passes core tests.
Cherry-picks: not for 2.x.
Change-Id: I138738d3e28e5daa1718c05c04cd9dd146c4ff84
Reviewed-on: http://gerrit.cloudera.org:8080/10413
Reviewed-by: Taras Bobrovytsky <tb...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/21d92aac
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/21d92aac
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/21d92aac
Branch: refs/heads/master
Commit: 21d92aacbfdbe9780b983acfacd02ced4bb0c132
Parents: 482ea39
Author: Tianyi Wang <tw...@cloudera.com>
Authored: Mon May 14 12:14:35 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue May 22 01:10:14 2018 +0000
----------------------------------------------------------------------
common/fbs/CatalogObjects.fbs | 3 +++
.../apache/impala/catalog/HdfsPartition.java | 28 +++++++++++++-------
.../org/apache/impala/catalog/HdfsTable.java | 18 ++++++-------
.../org/apache/impala/planner/HdfsScanNode.java | 3 ++-
tests/common/skip.py | 11 ++++++--
.../custom_cluster/test_admission_controller.py | 4 ++-
tests/custom_cluster/test_hdfs_fd_caching.py | 3 ++-
tests/metadata/test_explain.py | 3 ++-
tests/query_test/test_hdfs_caching.py | 3 ++-
tests/query_test/test_insert.py | 4 ++-
tests/query_test/test_insert_parquet.py | 3 ++-
tests/query_test/test_mt_dop.py | 3 +++
tests/query_test/test_nested_types.py | 5 +++-
tests/query_test/test_queries.py | 5 ++++
tests/query_test/test_query_mem_limit.py | 2 ++
tests/query_test/test_scanners.py | 4 +++
tests/util/filesystem_utils.py | 1 +
17 files changed, 75 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/common/fbs/CatalogObjects.fbs
----------------------------------------------------------------------
diff --git a/common/fbs/CatalogObjects.fbs b/common/fbs/CatalogObjects.fbs
index c08099d..d320dfa 100644
--- a/common/fbs/CatalogObjects.fbs
+++ b/common/fbs/CatalogObjects.fbs
@@ -73,4 +73,7 @@ table FbFileDesc {
// List of FbFileBlocks that make up this file.
file_blocks: [FbFileBlock] (id: 4);
+
+ // Whether this file is erasure-coded
+ is_ec: bool = false (id: 5);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
index 1b05804..3939ae2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
@@ -102,22 +102,29 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
* Creates the file descriptor of a file represented by 'fileStatus' with blocks
* stored in 'blockLocations'. 'fileSystem' is the filesystem where the
* file resides and 'hostIndex' stores the network addresses of the hosts that store
- * blocks of the parent HdfsTable. Populates 'numUnknownDiskIds' with the number of
- * unknown disk ids.
+ * blocks of the parent HdfsTable. 'isEc' indicates whether the file is erasure-coded.
+ * Populates 'numUnknownDiskIds' with the number of unknown disk ids.
*/
public static FileDescriptor create(FileStatus fileStatus,
BlockLocation[] blockLocations, FileSystem fileSystem,
- ListMap<TNetworkAddress> hostIndex, Reference<Long> numUnknownDiskIds)
- throws IOException {
+ ListMap<TNetworkAddress> hostIndex, boolean isEc,
+ Reference<Long> numUnknownDiskIds) throws IOException {
Preconditions.checkState(FileSystemUtil.supportsStorageIds(fileSystem));
FlatBufferBuilder fbb = new FlatBufferBuilder(1);
int[] fbFileBlockOffsets = new int[blockLocations.length];
int blockIdx = 0;
for (BlockLocation loc: blockLocations) {
- fbFileBlockOffsets[blockIdx++] = FileBlock.createFbFileBlock(fbb, loc, hostIndex,
- numUnknownDiskIds);
+ if (isEc) {
+ fbFileBlockOffsets[blockIdx++] = FileBlock.createFbFileBlock(fbb,
+ loc.getOffset(), loc.getLength(),
+ (short) hostIndex.getIndex(REMOTE_NETWORK_ADDRESS));
+ } else {
+ fbFileBlockOffsets[blockIdx++] =
+ FileBlock.createFbFileBlock(fbb, loc, hostIndex, numUnknownDiskIds);
+ }
}
- return new FileDescriptor(createFbFileDesc(fbb, fileStatus, fbFileBlockOffsets));
+ return new FileDescriptor(createFbFileDesc(fbb, fileStatus, fbFileBlockOffsets,
+ isEc));
}
/**
@@ -132,7 +139,8 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
FlatBufferBuilder fbb = new FlatBufferBuilder(1);
int[] fbFileBlockOffets =
synthesizeFbBlockMd(fbb, fileStatus, fileFormat, hostIndex);
- return new FileDescriptor(createFbFileDesc(fbb, fileStatus, fbFileBlockOffets));
+ return new FileDescriptor(createFbFileDesc(fbb, fileStatus, fbFileBlockOffets,
+ false));
}
/**
@@ -142,13 +150,14 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
* buffer.
*/
private static FbFileDesc createFbFileDesc(FlatBufferBuilder fbb,
- FileStatus fileStatus, int[] fbFileBlockOffets) {
+ FileStatus fileStatus, int[] fbFileBlockOffets, boolean isEc) {
int fileNameOffset = fbb.createString(fileStatus.getPath().getName());
int blockVectorOffset = FbFileDesc.createFileBlocksVector(fbb, fbFileBlockOffets);
FbFileDesc.startFbFileDesc(fbb);
FbFileDesc.addFileName(fbb, fileNameOffset);
FbFileDesc.addLength(fbb, fileStatus.getLen());
FbFileDesc.addLastModificationTime(fbb, fileStatus.getModificationTime());
+ FbFileDesc.addIsEc(fbb, isEc);
HdfsCompression comp = HdfsCompression.fromFileName(fileStatus.getPath().getName());
FbFileDesc.addCompression(fbb, comp.toFb());
FbFileDesc.addFileBlocks(fbb, blockVectorOffset);
@@ -209,6 +218,7 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
public long getModificationTime() { return fbFileDescriptor_.lastModificationTime(); }
public int getNumFileBlocks() { return fbFileDescriptor_.fileBlocksLength(); }
+ public boolean getIsEc() {return fbFileDescriptor_.isEc(); }
public FbFileBlock getFbFileBlock(int idx) {
return fbFileDescriptor_.fileBlocks(idx);
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 9910eb8..d6b8fb8 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -434,14 +434,14 @@ public class HdfsTable extends Table {
continue;
}
FileDescriptor fd;
- // Block locations are manually synthesized if the underlying fs does not support
- // the block location API.
if (synthesizeFileMd) {
+ // Block locations are manually synthesized if the underlying fs does not support
+ // the block location API.
fd = FileDescriptor.createWithSynthesizedBlockMd(fileStatus,
partitions.get(0).getFileFormat(), hostIndex_);
} else {
- fd = FileDescriptor.create(fileStatus,
- fileStatus.getBlockLocations(), fs, hostIndex_, numUnknownDiskIds);
+ fd = FileDescriptor.create(fileStatus, fileStatus.getBlockLocations(), fs,
+ hostIndex_, fileStatus.isErasureCoded(), numUnknownDiskIds);
}
newFileDescs.add(fd);
++loadStats.loadedFiles;
@@ -508,13 +508,13 @@ public class HdfsTable extends Table {
FileDescriptor fd = fileDescsByName.get(fileName);
if (isPartitionMarkedCached || hasFileChanged(fd, fileStatus)) {
if (synthesizeFileMd) {
- fd = FileDescriptor.createWithSynthesizedBlockMd(fileStatus,
- fileFormat, hostIndex_);
+ fd = FileDescriptor.createWithSynthesizedBlockMd(fileStatus, fileFormat,
+ hostIndex_);
} else {
BlockLocation[] locations =
- fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
- fd = FileDescriptor.create(fileStatus, locations, fs, hostIndex_,
- numUnknownDiskIds);
+ fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
+ fd = FileDescriptor.create(fileStatus, locations, fs, hostIndex_,
+ fileStatus.isErasureCoded(), numUnknownDiskIds);
}
++loadStats.loadedFiles;
} else {
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index dbaa965..d0c83c3 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -801,7 +801,8 @@ public class HdfsScanNode extends ScanNode {
// Translate from network address to the global (to this request) host index.
Integer globalHostIdx = analyzer.getHostIndex().getIndex(networkAddress);
location.setHost_idx(globalHostIdx);
- if (checkMissingDiskIds && FileBlock.getDiskId(block, i) == -1) {
+ if (checkMissingDiskIds && !fileDesc.getIsEc() &&
+ FileBlock.getDiskId(block, i) == -1) {
++numScanRangesNoDiskIds_;
partitionMissingDiskIds = true;
fileDescMissingDiskIds = true;
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/common/skip.py
----------------------------------------------------------------------
diff --git a/tests/common/skip.py b/tests/common/skip.py
index 9a5d95d..53fbe85 100644
--- a/tests/common/skip.py
+++ b/tests/common/skip.py
@@ -26,11 +26,12 @@ from functools import partial
from tests.common.environ import IMPALAD_BUILD
from tests.util.filesystem_utils import (
+ IS_ADLS,
+ IS_EC,
+ IS_HDFS,
IS_ISILON,
IS_LOCAL,
- IS_HDFS,
IS_S3,
- IS_ADLS,
SECONDARY_FILESYSTEM)
@@ -143,3 +144,9 @@ class SkipIfNotHdfsMinicluster:
class SkipIfBuildType:
not_dev_build = pytest.mark.skipif(not IMPALAD_BUILD.is_dev(),
reason="Tests depends on debug build startup option.")
+
+class SkipIfEC:
+ remote_read = pytest.mark.skipif(IS_EC, reason="EC files are read remotely and "
+ "features relying on local read do not work.")
+ oom = pytest.mark.skipif(IS_EC, reason="Probably broken by HDFS-13540.")
+ fix_later = pytest.mark.skipif(IS_EC, reason="It should work but doesn't.")
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/custom_cluster/test_admission_controller.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index d1d9dd8..eba0358 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -32,7 +32,8 @@ from tests.common.environ import specific_build_type_timeout, IMPALAD_BUILD
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import (
SkipIfS3,
- SkipIfADLS)
+ SkipIfADLS,
+ SkipIfEC)
from tests.common.test_dimensions import (
create_single_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -384,6 +385,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfADLS.hdfs_block_size
+ @SkipIfEC.fix_later
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args=impalad_admission_ctrl_flags(max_requests=1, max_queued=1,
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/custom_cluster/test_hdfs_fd_caching.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_hdfs_fd_caching.py b/tests/custom_cluster/test_hdfs_fd_caching.py
index ad80cef..1afe431 100644
--- a/tests/custom_cluster/test_hdfs_fd_caching.py
+++ b/tests/custom_cluster/test_hdfs_fd_caching.py
@@ -18,7 +18,7 @@
import pytest
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
-from tests.common.skip import SkipIfLocal
+from tests.common.skip import SkipIfLocal, SkipIfEC
from tests.util.filesystem_utils import (
IS_ISILON,
IS_S3,
@@ -26,6 +26,7 @@ from tests.util.filesystem_utils import (
from time import sleep
@SkipIfLocal.hdfs_fd_caching
+@SkipIfEC.remote_read
class TestHdfsFdCaching(CustomClusterTestSuite):
"""Tests that if HDFS file handle caching is enabled, file handles are actually cached
and the associated metrics return valid results. In addition, tests that the upper bound
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/metadata/test_explain.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_explain.py b/tests/metadata/test_explain.py
index 3ad411a..ba206f2 100644
--- a/tests/metadata/test_explain.py
+++ b/tests/metadata/test_explain.py
@@ -20,12 +20,13 @@
import re
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfLocal, SkipIfNotHdfsMinicluster
+from tests.common.skip import SkipIfLocal, SkipIfNotHdfsMinicluster, SkipIfEC
from tests.util.filesystem_utils import WAREHOUSE
# Tests the different explain levels [0-3] on a few queries.
# TODO: Clean up this test to use an explain level test dimension and appropriate
# result sub-sections for the expected explain plans.
+@SkipIfEC.fix_later
class TestExplain(ImpalaTestSuite):
# Value for the num_scanner_threads query option to ensure that the memory estimates of
# scan nodes are consistent even when run on machines with different numbers of cores.
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_hdfs_caching.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_hdfs_caching.py b/tests/query_test/test_hdfs_caching.py
index c013ed4..f16a4a4 100644
--- a/tests/query_test/test_hdfs_caching.py
+++ b/tests/query_test/test_hdfs_caching.py
@@ -25,7 +25,7 @@ from subprocess import check_call
from tests.common.environ import specific_build_type_timeout
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite, LOG
-from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
+from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal, SkipIfEC
from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.util.filesystem_utils import get_fs_path
from tests.util.shell_util import exec_process
@@ -35,6 +35,7 @@ from tests.util.shell_util import exec_process
@SkipIfADLS.caching
@SkipIfIsilon.caching
@SkipIfLocal.caching
+@SkipIfEC.fix_later
class TestHdfsCaching(ImpalaTestSuite):
@classmethod
def get_workload(self):
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_insert.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 88aafb2..20fee41 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -22,7 +22,7 @@ import pytest
from testdata.common import widetable
from tests.common.impala_cluster import ImpalaCluster
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfLocal, SkipIfNotHdfsMinicluster
+from tests.common.skip import SkipIfEC, SkipIfLocal, SkipIfNotHdfsMinicluster
from tests.common.test_dimensions import (
create_exec_option_dimension,
create_uncompressed_text_dimension)
@@ -112,6 +112,8 @@ class TestInsertQueries(ImpalaTestSuite):
super(TestInsertQueries, cls).setup_class()
@pytest.mark.execute_serially
+ # Erasure coding doesn't respect memory limit
+ @SkipIfEC.fix_later
def test_insert(self, vector):
if (vector.get_value('table_format').file_format == 'parquet'):
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_insert_parquet.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 1e8ce6e..4af81c9 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -29,7 +29,7 @@ from parquet.ttypes import ColumnOrder, SortingColumn, TypeDefinedOrder
from tests.common.environ import impalad_basedir
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.parametrize import UniqueDatabase
-from tests.common.skip import SkipIfIsilon, SkipIfLocal, SkipIfS3, SkipIfADLS
+from tests.common.skip import SkipIfEC, SkipIfIsilon, SkipIfLocal, SkipIfS3, SkipIfADLS
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_vector import ImpalaTestDimension
from tests.util.filesystem_utils import get_fs_path
@@ -101,6 +101,7 @@ class TestInsertParquetQueries(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').compression_codec == 'none')
+ @SkipIfEC.oom
@SkipIfLocal.multiple_impalad
@UniqueDatabase.parametrize(sync_ddl=True)
def test_insert_parquet(self, vector, unique_database):
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_mt_dop.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_mt_dop.py b/tests/query_test/test_mt_dop.py
index b6e08e5..0766f2e 100644
--- a/tests/query_test/test_mt_dop.py
+++ b/tests/query_test/test_mt_dop.py
@@ -22,6 +22,7 @@ import pytest
from copy import deepcopy
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.kudu_test_suite import KuduTestSuite
+from tests.common.skip import SkipIfEC
from tests.common.test_vector import ImpalaTestDimension
# COMPUTE STATS on Parquet tables automatically sets MT_DOP=4, so include
@@ -97,6 +98,8 @@ class TestMtDopParquet(ImpalaTestSuite):
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
self.run_test_case('QueryTest/mt-dop-parquet-nested', vector)
+ # Impala scans fewer row groups than it should with erasure coding.
+ @SkipIfEC.fix_later
def test_parquet_filtering(self, vector):
"""IMPALA-4624: Test that dictionary filtering eliminates row groups correctly."""
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_nested_types.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index e62bf4f..0603745 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -27,8 +27,10 @@ from tests.common.skip import (
SkipIfIsilon,
SkipIfS3,
SkipIfADLS,
+ SkipIfEC,
SkipIfLocal,
- SkipIfNotHdfsMinicluster)
+ SkipIfNotHdfsMinicluster
+ )
from tests.common.test_vector import ImpalaTestDimension
from tests.util.filesystem_utils import WAREHOUSE, get_fs_path
@@ -86,6 +88,7 @@ class TestNestedTypes(ImpalaTestSuite):
a 3-node HDFS minicluster."""
self.run_test_case('QueryTest/nested-types-tpch-mem-limit', vector)
+ @SkipIfEC.fix_later
def test_parquet_stats(self, vector):
"""Queries that test evaluation of Parquet row group statistics."""
self.run_test_case('QueryTest/nested-types-parquet-stats', vector)
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_queries.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_queries.py b/tests/query_test/test_queries.py
index a4fb91e..90cb392 100644
--- a/tests/query_test/test_queries.py
+++ b/tests/query_test/test_queries.py
@@ -22,6 +22,7 @@ import pytest
import re
from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.skip import SkipIfEC
from tests.common.test_dimensions import create_uncompressed_text_dimension, extend_exec_option_dimension
from tests.common.test_vector import ImpalaTestVector
@@ -166,12 +167,14 @@ class TestQueriesTextTables(ImpalaTestSuite):
vector.get_value('exec_option')['num_nodes'] = 1
self.run_test_case('QueryTest/distinct-estimate', vector)
+ @SkipIfEC.oom
def test_random(self, vector):
# These results will vary slightly depending on how the values get split up
# so only run with 1 node and on text.
vector.get_value('exec_option')['num_nodes'] = 1
self.run_test_case('QueryTest/random', vector)
+ @SkipIfEC.oom
def test_values(self, vector):
self.run_test_case('QueryTest/values', vector)
@@ -188,6 +191,7 @@ class TestQueriesParquetTables(ImpalaTestSuite):
def get_workload(cls):
return 'functional-query'
+ @SkipIfEC.oom
@pytest.mark.execute_serially
def test_very_large_strings(self, vector):
"""Regression test for IMPALA-1619. Doesn't need to be run on all file formats.
@@ -219,6 +223,7 @@ class TestHdfsQueries(ImpalaTestSuite):
def get_workload(cls):
return 'functional-query'
+ @SkipIfEC.oom
def test_hdfs_scan_node(self, vector):
self.run_test_case('QueryTest/hdfs-scan-node', vector)
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_query_mem_limit.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_query_mem_limit.py b/tests/query_test/test_query_mem_limit.py
index 17ea9f5..97d3ae7 100644
--- a/tests/query_test/test_query_mem_limit.py
+++ b/tests/query_test/test_query_mem_limit.py
@@ -24,6 +24,7 @@ from copy import copy
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.skip import SkipIfEC
from tests.common.test_dimensions import (
ImpalaTestDimension,
create_single_exec_option_dimension,
@@ -87,6 +88,7 @@ class TestQueryMemLimit(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('exec_option')['batch_size'] == 0)
+ @SkipIfEC.oom
@pytest.mark.execute_serially
def test_mem_limit(self, vector):
mem_limit = copy(vector.get_value('mem_limit'))
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 4ded221..61a4862 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -35,6 +35,7 @@ from tests.common.impala_test_suite import ImpalaTestSuite, LOG
from tests.common.skip import (
SkipIfS3,
SkipIfADLS,
+ SkipIfEC,
SkipIfIsilon,
SkipIfLocal)
from tests.common.test_dimensions import (
@@ -485,6 +486,7 @@ class TestParquet(ImpalaTestSuite):
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
+ @SkipIfEC.fix_later
def test_misaligned_parquet_row_groups(self, vector):
"""IMPALA-3989: Test that no warnings are issued when misaligned row groups are
encountered. Make sure that 'NumScannersWithNoReads' counters are set to the number of
@@ -555,6 +557,7 @@ class TestParquet(ImpalaTestSuite):
@SkipIfADLS.hdfs_block_size
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
+ @SkipIfEC.fix_later
def test_multiple_blocks_one_row_group(self, vector):
# For IMPALA-1881. The table functional_parquet.lineitem_multiblock_one_row_group has
# 3 blocks but only one row group across these blocks. We test to see that only one
@@ -954,6 +957,7 @@ class TestOrc(ImpalaTestSuite):
@SkipIfS3.hdfs_block_size
@SkipIfADLS.hdfs_block_size
+ @SkipIfEC.fix_later
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.multiple_impalad
def test_misaligned_orc_stripes(self, vector, unique_database):
http://git-wip-us.apache.org/repos/asf/impala/blob/21d92aac/tests/util/filesystem_utils.py
----------------------------------------------------------------------
diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py
index 77112be..82f6584 100644
--- a/tests/util/filesystem_utils.py
+++ b/tests/util/filesystem_utils.py
@@ -30,6 +30,7 @@ IS_ISILON = FILESYSTEM == "isilon"
IS_LOCAL = FILESYSTEM == "local"
IS_HDFS = FILESYSTEM == "hdfs"
IS_ADLS = FILESYSTEM == "adls"
+IS_EC = os.getenv("ERASURE_CODING") == "true"
# This condition satisfies both the states where one can assume a default fs
# - The environment variable is set to an empty string.
# - Tne environment variables is unset ( None )