You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2024/11/01 10:33:13 UTC
(hive) branch master updated: HIVE-28411: Iceberg: Support Bucket Map Join on Iceberg tables (Shohei Okumiya, reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new db09a5262c5 HIVE-28411: Iceberg: Support Bucket Map Join on Iceberg tables (Shohei Okumiya, reviewed by Denys Kuzmenko)
db09a5262c5 is described below
commit db09a5262c52b608f050bcce15308cfa5ce707ad
Author: okumin <gi...@okumin.com>
AuthorDate: Fri Nov 1 19:33:06 2024 +0900
HIVE-28411: Iceberg: Support Bucket Map Join on Iceberg tables (Shohei Okumiya, reviewed by Denys Kuzmenko)
Closes #5409
---
.../org/apache/iceberg/mr/InputFormatConfig.java | 1 +
.../iceberg/mr/hive/HiveIcebergInputFormat.java | 4 +-
.../apache/iceberg/mr/hive/HiveIcebergSplit.java | 31 +-
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 31 +
.../apache/iceberg/mr/hive/IcebergTableUtil.java | 6 +
.../mr/hive/plan/IcebergBucketFunction.java | 151 ++++
.../iceberg/mr/mapreduce/IcebergInputFormat.java | 69 +-
.../iceberg/mr/mapreduce/IcebergRecordReader.java | 6 +-
.../apache/iceberg/mr/mapreduce/IcebergSplit.java | 49 +-
.../queries/positive/iceberg_bucket_map_join_1.q | 127 ++++
.../queries/positive/iceberg_bucket_map_join_2.q | 95 +++
.../queries/positive/iceberg_bucket_map_join_3.q | 47 ++
.../queries/positive/iceberg_bucket_map_join_4.q | 60 ++
.../queries/positive/iceberg_bucket_map_join_5.q | 54 ++
.../queries/positive/iceberg_bucket_map_join_6.q | 27 +
.../queries/positive/iceberg_bucket_map_join_7.q | 128 ++++
.../queries/positive/iceberg_bucket_map_join_8.q | 50 ++
.../positive/llap/iceberg_bucket_map_join_1.q.out | 614 ++++++++++++++++
.../positive/llap/iceberg_bucket_map_join_2.q.out | 548 ++++++++++++++
.../positive/llap/iceberg_bucket_map_join_3.q.out | 238 ++++++
.../positive/llap/iceberg_bucket_map_join_4.q.out | 350 +++++++++
.../positive/llap/iceberg_bucket_map_join_5.q.out | 283 ++++++++
.../positive/llap/iceberg_bucket_map_join_6.q.out | 177 +++++
.../positive/llap/iceberg_bucket_map_join_7.q.out | 800 +++++++++++++++++++++
.../positive/llap/iceberg_bucket_map_join_8.q.out | 218 ++++++
.../positive/vectorized_iceberg_merge_mixed.q.out | 19 +-
.../test/resources/testconfiguration.properties | 16 +
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 27 +-
.../hive/ql/exec/tez/CustomPartitionVertex.java | 71 +-
.../VectorReduceSinkObjectHashOperator.java | 20 +-
.../apache/hadoop/hive/ql/io/HiveInputFormat.java | 31 +
.../hadoop/hive/ql/io/PartitionAwareSplit.java | 34 +
.../hive/ql/metadata/HiveStorageHandler.java | 24 +
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 92 ++-
.../annotation/OpTraitsRulesProcFactory.java | 161 ++++-
.../hadoop/hive/ql/plan/CustomBucketFunction.java | 70 ++
.../org/apache/hadoop/hive/ql/plan/OpTraits.java | 32 +-
.../ql/plan/PartitionAwareOptimizationCtx.java | 46 ++
.../apache/hadoop/hive/ql/plan/ReduceSinkDesc.java | 35 +
.../apache/hadoop/hive/ql/plan/TableScanDesc.java | 29 +
.../hadoop/hive/ql/udf/generic/GenericUDF.java | 22 +
41 files changed, 4699 insertions(+), 194 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
index e15e556f12a..e7778b40869 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
@@ -39,6 +39,7 @@ public class InputFormatConfig {
public static final String SKIP_RESIDUAL_FILTERING = "skip.residual.filtering";
public static final String AS_OF_TIMESTAMP = "iceberg.mr.as.of.time";
public static final String FILTER_EXPRESSION = "iceberg.mr.filter.expression";
+ public static final String GROUPING_PARTITION_COLUMNS = "iceberg.mr.grouping.partition.columns";
public static final String IN_MEMORY_DATA_MODEL = "iceberg.mr.in.memory.data.model";
public static final String READ_SCHEMA = "iceberg.mr.read.schema";
public static final String SNAPSHOT_ID = "iceberg.mr.snapshot.id";
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
index 98fd4b30163..a8e0589a900 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergInputFormat.java
@@ -154,6 +154,7 @@ public class HiveIcebergInputFormat extends MapredIcebergInputFormat<Record>
}
job.set(InputFormatConfig.SELECTED_COLUMNS, job.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, ""));
+ job.set(InputFormatConfig.GROUPING_PARTITION_COLUMNS, job.get(TableScanDesc.GROUPING_PARTITION_COLUMNS, ""));
job.setBoolean(InputFormatConfig.FETCH_VIRTUAL_COLUMNS,
job.getBoolean(ColumnProjectionUtils.FETCH_VIRTUAL_COLUMNS_CONF_STR, false));
job.set(InputFormatConfig.AS_OF_TIMESTAMP, job.get(TableScanDesc.AS_OF_TIMESTAMP, "-1"));
@@ -162,8 +163,9 @@ public class HiveIcebergInputFormat extends MapredIcebergInputFormat<Record>
job.set(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF, job.get(TableScanDesc.SNAPSHOT_REF, ""));
String location = job.get(InputFormatConfig.TABLE_LOCATION);
+ int numBuckets = job.getInt(TableScanDesc.GROUPING_NUM_BUCKETS, -1);
return Arrays.stream(super.getSplits(job, numSplits))
- .map(split -> new HiveIcebergSplit((IcebergSplit) split, location))
+ .map(split -> new HiveIcebergSplit((IcebergSplit) split, location, numBuckets))
.toArray(InputSplit[]::new);
}
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java
index a9827ffb976..e0e9aa78a22 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergSplit.java
@@ -24,17 +24,24 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Collection;
+import java.util.OptionalInt;
+import java.util.stream.IntStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.tez.HashableInputSplit;
+import org.apache.hadoop.hive.ql.io.PartitionAwareSplit;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.mr.hive.plan.IcebergBucketFunction;
import org.apache.iceberg.mr.mapreduce.IcebergSplit;
import org.apache.iceberg.mr.mapreduce.IcebergSplitContainer;
import org.apache.iceberg.relocated.com.google.common.primitives.Longs;
import org.apache.iceberg.util.SerializationUtil;
// Hive requires file formats to return splits that are instances of `FileSplit`.
-public class HiveIcebergSplit extends FileSplit implements IcebergSplitContainer, HashableInputSplit {
+public class HiveIcebergSplit extends FileSplit
+ implements IcebergSplitContainer, HashableInputSplit, PartitionAwareSplit {
private IcebergSplit innerSplit;
@@ -43,14 +50,16 @@ public class HiveIcebergSplit extends FileSplit implements IcebergSplitContainer
// That way, `HiveInputFormat` and `CombineHiveInputFormat` can read files with different file formats in the same
// MapReduce job and merge compatible splits together.
private String tableLocation;
+ private int numBuckets;
// public no-argument constructor for deserialization
public HiveIcebergSplit() {
}
- HiveIcebergSplit(IcebergSplit split, String tableLocation) {
+ HiveIcebergSplit(IcebergSplit split, String tableLocation, int numBuckets) {
this.innerSplit = split;
this.tableLocation = tableLocation;
+ this.numBuckets = numBuckets;
}
@Override
@@ -75,7 +84,7 @@ public class HiveIcebergSplit extends FileSplit implements IcebergSplitContainer
@Override
public byte[] getBytesForHash() {
- Collection<FileScanTask> fileScanTasks = innerSplit.task().files();
+ Collection<FileScanTask> fileScanTasks = innerSplit.taskGroup().tasks();
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
for (FileScanTask task : fileScanTasks) {
@@ -88,6 +97,20 @@ public class HiveIcebergSplit extends FileSplit implements IcebergSplitContainer
}
}
+ @Override
+ public OptionalInt getBucketId() {
+ final StructLike key = innerSplit.taskGroup().groupingKey();
+ if (key.size() == 0) {
+ return OptionalInt.empty();
+ }
+ final int[] bucketIds = IntStream
+ .range(0, key.size())
+ .map(i -> key.get(i, Integer.class))
+ .toArray();
+ final int hashCode = IcebergBucketFunction.getHashCode(bucketIds);
+ return OptionalInt.of(ObjectInspectorUtils.getBucketNumber(hashCode, numBuckets));
+ }
+
@Override
public long getStart() {
return 0;
@@ -98,6 +121,7 @@ public class HiveIcebergSplit extends FileSplit implements IcebergSplitContainer
byte[] bytes = SerializationUtil.serializeToBytes(tableLocation);
out.writeInt(bytes.length);
out.write(bytes);
+ out.writeInt(numBuckets);
innerSplit.write(out);
}
@@ -107,6 +131,7 @@ public class HiveIcebergSplit extends FileSplit implements IcebergSplitContainer
byte[] bytes = new byte[in.readInt()];
in.readFully(bytes);
tableLocation = SerializationUtil.deserializeFromBytes(bytes);
+ numBuckets = in.readInt();
innerSplit = new IcebergSplit();
innerSplit.readFields(in);
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index fc7a9df77d2..9058f62963d 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -114,6 +114,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.plan.MergeTaskProperties;
+import org.apache.hadoop.hive.ql.plan.PartitionAwareOptimizationCtx;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
import org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils;
@@ -184,6 +185,7 @@ import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.mr.Catalogs;
import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.mr.hive.actions.HiveIcebergDeleteOrphanFiles;
+import org.apache.iceberg.mr.hive.plan.IcebergBucketFunction;
import org.apache.iceberg.puffin.Blob;
import org.apache.iceberg.puffin.BlobMetadata;
import org.apache.iceberg.puffin.Puffin;
@@ -770,6 +772,35 @@ public class HiveIcebergStorageHandler implements HiveStoragePredicateHandler, H
).collect(Collectors.toList()));
}
+ @Override
+ public boolean supportsPartitionAwareOptimization(org.apache.hadoop.hive.ql.metadata.Table table) {
+ if (hasUndergonePartitionEvolution(table)) {
+ // Don't support complex cases yet
+ return false;
+ }
+ final List<TransformSpec> specs = getPartitionTransformSpec(table);
+ // Currently, we support the only bucket transform
+ return specs.stream().anyMatch(IcebergTableUtil::isBucket);
+ }
+
+ @Override
+ public PartitionAwareOptimizationCtx createPartitionAwareOptimizationContext(
+ org.apache.hadoop.hive.ql.metadata.Table table) {
+ // Currently, we support the only bucket transform
+ final List<String> bucketColumnNames = Lists.newArrayList();
+ final List<Integer> numBuckets = Lists.newArrayList();
+ getPartitionTransformSpec(table).stream().filter(IcebergTableUtil::isBucket).forEach(spec -> {
+ bucketColumnNames.add(spec.getColumnName());
+ numBuckets.add(spec.getTransformParam().get());
+ });
+
+ if (bucketColumnNames.isEmpty()) {
+ return null;
+ }
+ final IcebergBucketFunction bucketFunction = new IcebergBucketFunction(bucketColumnNames, numBuckets);
+ return new PartitionAwareOptimizationCtx(bucketFunction);
+ }
+
@Override
public String getFileFormatPropertyKey() {
return TableProperties.DEFAULT_FILE_FORMAT;
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
index d6ddf637a90..42bafe0b9fc 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveUtils;
import org.apache.hadoop.hive.ql.parse.AlterTableExecuteSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.TransformSpec;
+import org.apache.hadoop.hive.ql.parse.TransformSpec.TransformType;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionStateUtil;
@@ -278,6 +279,11 @@ public class IcebergTableUtil {
return table.spec().fields().stream().anyMatch(f -> f.transform().toString().startsWith("bucket["));
}
+ public static boolean isBucket(TransformSpec spec) {
+ // Iceberg's bucket transform requires a bucket number to be specified
+ return spec.getTransformType() == TransformType.BUCKET && spec.getTransformParam().isPresent();
+ }
+
/**
* Roll an iceberg table's data back to a specific snapshot identified either by id or before a given timestamp.
* @param table the iceberg table
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/plan/IcebergBucketFunction.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/plan/IcebergBucketFunction.java
new file mode 100644
index 00000000000..3161acd719d
--- /dev/null
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/plan/IcebergBucketFunction.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iceberg.mr.hive.plan;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.CustomBucketFunction;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.SettableDeferredJavaObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.iceberg.mr.hive.udf.GenericUDFIcebergBucket;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+
+public class IcebergBucketFunction implements CustomBucketFunction {
+ private static final long serialVersionUID = 1L;
+
+ private final List<String> sourceColumnNames;
+ private final List<Integer> numBuckets;
+
+ private transient List<GenericUDFIcebergBucket> bucketUdfs;
+ private transient List<IntObjectInspector> bucketIdInspectors;
+ private transient SettableDeferredJavaObject[] deferredObjects;
+
+ public IcebergBucketFunction(List<String> sourceColumnNames, List<Integer> numBuckets) {
+ Objects.requireNonNull(sourceColumnNames);
+ Objects.requireNonNull(numBuckets);
+ Preconditions.checkArgument(sourceColumnNames.size() == numBuckets.size());
+ Preconditions.checkArgument(!sourceColumnNames.isEmpty());
+ this.sourceColumnNames = sourceColumnNames;
+ this.numBuckets = numBuckets;
+ }
+
+ @Override
+ public Optional<CustomBucketFunction> select(boolean[] retainedColumns) {
+ Preconditions.checkArgument(retainedColumns.length == numBuckets.size());
+ Preconditions.checkState(bucketUdfs == null);
+ Preconditions.checkState(bucketIdInspectors == null);
+ Preconditions.checkState(deferredObjects == null);
+
+ final List<String> newSourceColumnNames = Lists.newArrayList();
+ final List<Integer> newNumBuckets = Lists.newArrayList();
+ for (int i = 0; i < retainedColumns.length; i++) {
+ if (retainedColumns[i]) {
+ newSourceColumnNames.add(sourceColumnNames.get(i));
+ newNumBuckets.add(numBuckets.get(i));
+ }
+ }
+
+ if (newSourceColumnNames.isEmpty()) {
+ return Optional.empty();
+ }
+
+ final IcebergBucketFunction newBucketFunction = new IcebergBucketFunction(newSourceColumnNames, newNumBuckets);
+ return Optional.of(newBucketFunction);
+ }
+
+ @Override
+ public void initialize(ObjectInspector[] objectInspectors) {
+ Preconditions.checkArgument(objectInspectors.length == numBuckets.size());
+ bucketIdInspectors = Lists.newArrayList();
+ bucketUdfs = Lists.newArrayList();
+ for (int i = 0; i < objectInspectors.length; i++) {
+ final GenericUDFIcebergBucket udf = new GenericUDFIcebergBucket();
+ final ObjectInspector numBucketInspector = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(
+ TypeInfoFactory.intTypeInfo,
+ new IntWritable(numBuckets.get(i)));
+ final ObjectInspector[] args = { objectInspectors[i], numBucketInspector };
+ try {
+ final ObjectInspector inspector = udf.initialize(args);
+ Preconditions.checkState(inspector.getCategory() == Category.PRIMITIVE);
+ final PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) inspector;
+ Preconditions.checkState(primitiveInspector.getPrimitiveCategory() == PrimitiveCategory.INT);
+ bucketIdInspectors.add((IntObjectInspector) primitiveInspector);
+ } catch (UDFArgumentException e) {
+ throw new IllegalArgumentException("The given object inspector is illegal", e);
+ }
+ bucketUdfs.add(udf);
+ }
+ deferredObjects = new SettableDeferredJavaObject[1];
+ deferredObjects[0] = new SettableDeferredJavaObject();
+
+ Preconditions.checkState(bucketIdInspectors.size() == numBuckets.size());
+ Preconditions.checkState(bucketUdfs.size() == numBuckets.size());
+ }
+
+ @Override
+ public List<String> getSourceColumnNames() {
+ return sourceColumnNames;
+ }
+
+ @Override
+ public int getNumBuckets() {
+ return numBuckets.stream().reduce(1, Math::multiplyExact);
+ }
+
+ @Override
+ public int getBucketHashCode(Object[] bucketFields) {
+ final int[] bucketIds = new int[numBuckets.size()];
+ for (int i = 0; i < bucketUdfs.size(); i++) {
+ deferredObjects[0].set(bucketFields[i]);
+ try {
+ final Object bucketId = bucketUdfs.get(i).evaluate(deferredObjects);
+ bucketIds[i] = bucketIdInspectors.get(i).get(bucketId);
+ } catch (HiveException e) {
+ throw new IllegalArgumentException("Failed to evaluate the given objects", e);
+ }
+ }
+ return getHashCode(bucketIds);
+ }
+
+ public static int getHashCode(int[] bucketIds) {
+ int hashCode = 0;
+ for (int bucketId : bucketIds) {
+ hashCode = 31 * hashCode + bucketId;
+ }
+ return hashCode;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("IcebergBucketFunction{sourceColumnNames=%s,numBuckets=%s}",
+ sourceColumnNames, numBuckets);
+ }
+}
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
index 00e63740e28..7a024db550d 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
@@ -24,6 +24,7 @@ import java.io.UncheckedIOException;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutorService;
+import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -44,7 +45,9 @@ import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.DataTableScan;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.IncrementalAppendScan;
+import org.apache.iceberg.Partitioning;
import org.apache.iceberg.Scan;
+import org.apache.iceberg.ScanTaskGroup;
import org.apache.iceberg.Schema;
import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.SystemConfigs;
@@ -58,7 +61,9 @@ import org.apache.iceberg.mr.Catalogs;
import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.mr.hive.HiveIcebergStorageHandler;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types.StructType;
import org.apache.iceberg.util.SerializationUtil;
+import org.apache.iceberg.util.TableScanUtil;
import org.apache.iceberg.util.ThreadPools;
/**
@@ -205,22 +210,18 @@ public class IcebergInputFormat<T> extends InputFormat<Void, T> {
HiveConf.ConfVars.HIVE_ICEBERG_ALLOW_DATAFILES_IN_TABLE_LOCATION_ONLY.defaultBoolVal);
Path tableLocation = new Path(conf.get(InputFormatConfig.TABLE_LOCATION));
-
- try (CloseableIterable<CombinedScanTask> tasksIterable = scan.planTasks()) {
- tasksIterable.forEach(task -> {
- if (applyResidual && (model == InputFormatConfig.InMemoryDataModel.HIVE ||
- model == InputFormatConfig.InMemoryDataModel.PIG)) {
- // TODO: We do not support residual evaluation for HIVE and PIG in memory data model yet
- checkResiduals(task);
- }
- if (allowDataFilesWithinTableLocationOnly) {
- validateFileLocations(task, tableLocation);
- }
- splits.add(new IcebergSplit(conf, task));
- });
- } catch (IOException e) {
- throw new UncheckedIOException(String.format("Failed to close table scan: %s", scan), e);
- }
+ String[] groupingPartitionColumns = conf.getStrings(InputFormatConfig.GROUPING_PARTITION_COLUMNS);
+ generateInputSplits(scan, table, groupingPartitionColumns, taskGroup -> {
+ if (applyResidual && (model == InputFormatConfig.InMemoryDataModel.HIVE ||
+ model == InputFormatConfig.InMemoryDataModel.PIG)) {
+ // TODO: We do not support residual evaluation for HIVE and PIG in memory data model yet
+ checkResiduals(taskGroup);
+ }
+ if (allowDataFilesWithinTableLocationOnly) {
+ validateFileLocations(taskGroup, tableLocation);
+ }
+ splits.add(new IcebergSplit(conf, taskGroup));
+ });
// If enabled, do not serialize FileIO hadoop config to decrease split size
// However, do not skip serialization for metatable queries, because some metadata tasks cache the IO object and we
@@ -232,16 +233,43 @@ public class IcebergInputFormat<T> extends InputFormat<Void, T> {
return splits;
}
- private static void validateFileLocations(CombinedScanTask split, Path tableLocation) {
- for (FileScanTask fileScanTask : split.files()) {
+ private static void validateFileLocations(ScanTaskGroup<FileScanTask> split, Path tableLocation) {
+ for (FileScanTask fileScanTask : split.tasks()) {
if (!FileUtils.isPathWithinSubtree(new Path(fileScanTask.file().path().toString()), tableLocation)) {
throw new AuthorizationException("The table contains paths which are outside the table location");
}
}
}
- private static void checkResiduals(CombinedScanTask task) {
- task.files().forEach(fileScanTask -> {
+ private static void generateInputSplits(Scan<?, FileScanTask, CombinedScanTask> scan, Table table,
+ String[] groupingPartitionColumns, Consumer<ScanTaskGroup<FileScanTask>> consumer) {
+ if (groupingPartitionColumns == null) {
+ try (CloseableIterable<CombinedScanTask> tasksIterable = scan.planTasks()) {
+ tasksIterable.forEach(consumer);
+ } catch (IOException e) {
+ throw new UncheckedIOException(String.format("Failed to close table scan: %s", scan), e);
+ }
+ } else {
+ final StructType groupingKeyType = Partitioning.groupingKeyType(
+ table.schema().select(groupingPartitionColumns), table.specs().values());
+ try (CloseableIterable<FileScanTask> taskIterable = scan.planFiles()) {
+ final List<FileScanTask> tasks = Lists.newArrayList(taskIterable);
+ final List<ScanTaskGroup<FileScanTask>> partitionScanTaskGroups =
+ TableScanUtil.planTaskGroups(
+ tasks,
+ scan.targetSplitSize(),
+ scan.splitLookback(),
+ scan.splitOpenFileCost(),
+ groupingKeyType);
+ partitionScanTaskGroups.forEach(consumer);
+ } catch (IOException e) {
+ throw new UncheckedIOException(String.format("Failed to close table scan: %s", scan), e);
+ }
+ }
+ }
+
+ private static void checkResiduals(ScanTaskGroup<FileScanTask> task) {
+ task.tasks().forEach(fileScanTask -> {
Expression residual = fileScanTask.residual();
if (residual != null && !residual.equals(Expressions.alwaysTrue())) {
throw new UnsupportedOperationException(
@@ -256,5 +284,4 @@ public class IcebergInputFormat<T> extends InputFormat<Void, T> {
public RecordReader<Void, T> createRecordReader(InputSplit split, TaskAttemptContext context) {
return split instanceof IcebergMergeSplit ? new IcebergMergeRecordReader<>() : new IcebergRecordReader<>();
}
-
}
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
index 916082fe469..7208a5f4076 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergRecordReader.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DataTask;
import org.apache.iceberg.FileFormat;
@@ -39,6 +38,7 @@ import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.ScanTaskGroup;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
import org.apache.iceberg.avro.Avro;
@@ -99,8 +99,8 @@ public final class IcebergRecordReader<T> extends AbstractIcebergRecordReader<T>
public void initialize(InputSplit split, TaskAttemptContext newContext) {
// For now IcebergInputFormat does its own split planning and does not accept FileSplit instances
super.initialize(split, newContext);
- CombinedScanTask task = ((IcebergSplit) split).task();
- this.tasks = task.files();
+ ScanTaskGroup<FileScanTask> task = ((IcebergSplit) split).taskGroup();
+ this.tasks = task.tasks();
this.currentIterator = nextTask();
}
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java
index 395ea35fe47..e8a0f7f3c96 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergSplit.java
@@ -22,21 +22,29 @@ package org.apache.iceberg.mr.mapreduce;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.iceberg.CombinedScanTask;
import org.apache.iceberg.FileScanTask;
-import org.apache.iceberg.hadoop.Util;
+import org.apache.iceberg.ScanTaskGroup;
import org.apache.iceberg.mr.InputFormatConfig;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.util.SerializationUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
// Since this class extends `mapreduce.InputSplit and implements `mapred.InputSplit`, it can be returned by both MR v1
// and v2 file formats.
public class IcebergSplit extends InputSplit implements IcebergSplitContainer {
+ private static final Logger LOG = LoggerFactory.getLogger(IcebergSplit.class);
public static final String[] ANYWHERE = new String[]{"*"};
- private CombinedScanTask task;
+ private ScanTaskGroup<FileScanTask> taskGroup;
private transient String[] locations;
private transient Configuration conf;
@@ -45,13 +53,13 @@ public class IcebergSplit extends InputSplit implements IcebergSplitContainer {
public IcebergSplit() {
}
- IcebergSplit(Configuration conf, CombinedScanTask task) {
- this.task = task;
+ IcebergSplit(Configuration conf, ScanTaskGroup<FileScanTask> taskGroup) {
+ this.taskGroup = taskGroup;
this.conf = conf;
}
- public CombinedScanTask task() {
- return task;
+ public ScanTaskGroup<FileScanTask> taskGroup() {
+ return taskGroup;
}
@Override
@@ -61,7 +69,7 @@ public class IcebergSplit extends InputSplit implements IcebergSplitContainer {
@Override
public long getLength() {
- return task.files().stream().mapToLong(FileScanTask::length).sum();
+ return taskGroup.tasks().stream().mapToLong(FileScanTask::length).sum();
}
@Override
@@ -70,7 +78,7 @@ public class IcebergSplit extends InputSplit implements IcebergSplitContainer {
// getLocations() won't be accurate when called on worker nodes and will always return "*"
if (locations == null && conf != null) {
boolean localityPreferred = conf.getBoolean(InputFormatConfig.LOCALITY, false);
- locations = localityPreferred ? Util.blockLocations(task, conf) : ANYWHERE;
+ locations = localityPreferred ? blockLocations(taskGroup, conf) : ANYWHERE;
} else {
locations = ANYWHERE;
}
@@ -78,9 +86,28 @@ public class IcebergSplit extends InputSplit implements IcebergSplitContainer {
return locations;
}
+ // We should move to Util.blockLocations once the following PR is merged and shipped
+ // https://github.com/apache/iceberg/pull/11053
+ private static String[] blockLocations(ScanTaskGroup<FileScanTask> task, Configuration conf) {
+ final Set<String> locationSets = Sets.newHashSet();
+ task.tasks().forEach(fileScanTask -> {
+ final Path path = new Path(fileScanTask.file().path().toString());
+ try {
+ final FileSystem fs = path.getFileSystem(conf);
+ for (BlockLocation location : fs.getFileBlockLocations(path, fileScanTask.start(), fileScanTask.length())) {
+ locationSets.addAll(Arrays.asList(location.getHosts()));
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get block locations for path {}", path, e);
+ }
+ });
+
+ return locationSets.toArray(new String[0]);
+ }
+
@Override
public void write(DataOutput out) throws IOException {
- byte[] data = SerializationUtil.serializeToBytes(this.task);
+ byte[] data = SerializationUtil.serializeToBytes(this.taskGroup);
out.writeInt(data.length);
out.write(data);
}
@@ -89,6 +116,6 @@ public class IcebergSplit extends InputSplit implements IcebergSplitContainer {
public void readFields(DataInput in) throws IOException {
byte[] data = new byte[in.readInt()];
in.readFully(data);
- this.task = SerializationUtil.deserializeFromBytes(data);
+ this.taskGroup = SerializationUtil.deserializeFromBytes(data);
}
}
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_1.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_1.q
new file mode 100644
index 00000000000..cf9dafe18e9
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_1.q
@@ -0,0 +1,127 @@
+-- Copied from bucket_map_join_tez3.q
+set hive.explain.user=true;
+
+create table target_table(date_col date, string_col string, decimal_col decimal(38,0)) partitioned by spec (bucket(7, decimal_col)) stored by iceberg;
+insert into table target_table values
+('2017-05-17', 'pipeline', '50000000000000000000441610525'),
+('2018-12-20', 'pipeline', '50000000000000000001048981030'),
+('2020-06-30', 'pipeline', '50000000000000000002332575516'),
+('2021-08-16', 'pipeline', '50000000000000000003897973989'),
+('2017-06-06', 'pipeline', '50000000000000000000449148729'),
+('2017-09-08', 'pipeline', '50000000000000000000525378314'),
+('2022-08-30', 'pipeline', '50000000000000000005905545593'),
+('2022-08-16', 'pipeline', '50000000000000000005905545593'),
+('2018-05-03', 'pipeline', '50000000000000000000750826355'),
+('2020-01-10', 'pipeline', '50000000000000000001816579677'),
+('2021-11-01', 'pipeline', '50000000000000000004269423714'),
+('2017-11-07', 'pipeline', '50000000000000000000585901787'),
+('2019-10-15', 'pipeline', '50000000000000000001598843430'),
+('2020-04-01', 'pipeline', '50000000000000000002035795461'),
+('2020-02-24', 'pipeline', '50000000000000000001932600185'),
+('2020-04-27', 'pipeline', '50000000000000000002108160849'),
+('2016-07-05', 'pipeline', '50000000000000000000054405114'),
+('2020-06-02', 'pipeline', '50000000000000000002234387967'),
+('2020-08-21', 'pipeline', '50000000000000000002529168758'),
+('2021-02-17', 'pipeline', '50000000000000000003158511687');
+
+create table source_table(date_col date, string_col string, decimal_col decimal(38,0)) partitioned by spec (bucket(7, decimal_col)) stored by iceberg;
+insert into table source_table values
+('2022-08-30', 'pipeline', '50000000000000000005905545593'),
+('2022-08-16', 'pipeline', '50000000000000000005905545593'),
+('2022-09-01', 'pipeline', '50000000000000000006008686831'),
+('2022-08-30', 'pipeline', '50000000000000000005992620837'),
+('2022-09-01', 'pipeline', '50000000000000000005992620837'),
+('2022-09-01', 'pipeline', '50000000000000000005992621067'),
+('2022-08-30', 'pipeline', '50000000000000000005992621067');
+
+
+-- Test 2 queries in 4 configs.
+
+-- Each query has 1 join that can be converted to bucket join.
+-- One of the query receives the small table from Map vertex while the other recives it from Reducer vertex.
+
+-- 4 configs enfoce MapJoin to be converted to one of the following joins:
+-- 1. BucketMapJoin, 2. MapJoin, 3. VectorBucketMapJoin, 4. VectorMapJoin
+
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+-- 1. BucketMapJoin
+set hive.convert.join.bucket.mapjoin.tez=true;
+set hive.vectorized.execution.enabled=false;
+
+explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+
+explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+
+-- 2. MapJoin
+set hive.convert.join.bucket.mapjoin.tez=false;
+set hive.vectorized.execution.enabled=false;
+
+explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+
+explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+
+-- 3. VectorBucketMapJoin
+set hive.convert.join.bucket.mapjoin.tez=true;
+set hive.vectorized.execution.enabled=true;
+
+explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+
+explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+
+-- 4. VectorMapJoin
+set hive.convert.join.bucket.mapjoin.tez=false;
+set hive.vectorized.execution.enabled=true;
+
+explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+
+explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_2.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_2.q
new file mode 100644
index 00000000000..d347999d88e
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_2.q
@@ -0,0 +1,95 @@
+-- Copied from bucketmapjoin5.q
+set hive.explain.user=true;
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+CREATE TABLE srcbucket_mapjoin_n0_tmp(key int, value string) STORED AS TEXTFILE;
+load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_n0_tmp;
+load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_n0_tmp;
+CREATE TABLE srcbucket_mapjoin_n0(key int, value string) PARTITIONED BY SPEC (bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_n0 SELECT * FROM srcbucket_mapjoin_n0_tmp;
+
+CREATE TABLE srcbucket_mapjoin_part_n0_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE;
+load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09');
+load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09');
+load data local inpath '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09');
+load data local inpath '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09');
+CREATE TABLE srcbucket_mapjoin_part_n0 (key int, value string, ds string) partitioned by spec (ds, bucket(4, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_n0 SELECT * FROM srcbucket_mapjoin_part_n0_tmp;
+
+CREATE TABLE srcbucket_mapjoin_part_2_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE;
+load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-09');
+load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-09');
+CREATE TABLE srcbucket_mapjoin_part_2 (key int, value string, ds string) partitioned by spec (ds, bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_2 SELECT * FROM srcbucket_mapjoin_part_2_tmp;
+
+create table bucketmapjoin_hash_result_1 (key bigint , value1 bigint, value2 bigint);
+create table bucketmapjoin_hash_result_2 (key bigint , value1 bigint, value2 bigint);
+set hive.convert.join.bucket.mapjoin.tez=true;
+create table bucketmapjoin_tmp_result (key string , value1 string, value2 string);
+
+explain
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key;
+
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key;
+
+select count(1) from bucketmapjoin_tmp_result;
+insert overwrite table bucketmapjoin_hash_result_1
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result;
+
+set hive.convert.join.bucket.mapjoin.tez=false;
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key;
+
+select count(1) from bucketmapjoin_tmp_result;
+insert overwrite table bucketmapjoin_hash_result_2
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result;
+
+select a.key-b.key, a.value1-b.value1, a.value2-b.value2
+from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b
+on a.key = b.key;
+
+
+set hive.convert.join.bucket.mapjoin.tez=true;
+explain
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key;
+
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key;
+
+select count(1) from bucketmapjoin_tmp_result;
+insert overwrite table bucketmapjoin_hash_result_1
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result;
+
+set hive.convert.join.bucket.mapjoin.tez=false;
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key;
+
+select count(1) from bucketmapjoin_tmp_result;
+insert overwrite table bucketmapjoin_hash_result_2
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result;
+
+select a.key-b.key, a.value1-b.value1, a.value2-b.value2
+from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b
+on a.key = b.key;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_3.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_3.q
new file mode 100644
index 00000000000..8b6a743ce08
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_3.q
@@ -0,0 +1,47 @@
+-- Copied from bucketmapjoin8.q
+set hive.explain.user=true;
+
+--! qt:dataset:part
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+CREATE TABLE srcbucket_mapjoin_part_1_n1_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n1_tmp PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n1_tmp PARTITION (part='1');
+CREATE TABLE srcbucket_mapjoin_part_1_n1 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_1_n1 SELECT * FROM srcbucket_mapjoin_part_1_n1_tmp;
+
+CREATE TABLE srcbucket_mapjoin_part_2_n4_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n4_tmp PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n4_tmp PARTITION (part='1');
+CREATE TABLE srcbucket_mapjoin_part_2_n4 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_2_n4 SELECT * FROM srcbucket_mapjoin_part_2_n4_tmp;
+
+ALTER TABLE srcbucket_mapjoin_part_2_n4 SET PARTITION SPEC (part, bucket(3, key));
+
+set hive.convert.join.bucket.mapjoin.tez=true;
+-- The partition bucketing metadata match but the tables have different numbers of buckets, bucket map join should still be used
+
+EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
+
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
+
+ALTER TABLE srcbucket_mapjoin_part_2_n4 SET PARTITION SPEC (part, bucket(2, value));
+
+-- The partition bucketing metadata match but the tables are bucketed on different columns, bucket map join should still be used
+
+EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
+
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_4.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_4.q
new file mode 100644
index 00000000000..6bb6b3386a2
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_4.q
@@ -0,0 +1,60 @@
+-- Copied from bucketmapjoin11.q
+set hive.explain.user=true;
+
+--! qt:dataset:part
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+CREATE TABLE srcbucket_mapjoin_part_1_n2_tmp1 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp1 PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp1 PARTITION (part='1');
+CREATE TABLE srcbucket_mapjoin_part_1_n2 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_1_n2 SELECT * FROM srcbucket_mapjoin_part_1_n2_tmp1;
+
+ALTER TABLE srcbucket_mapjoin_part_1_n2 SET PARTITION SPEC (part, bucket(4, key));
+CREATE TABLE srcbucket_mapjoin_part_1_n2_tmp2 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2');
+INSERT INTO srcbucket_mapjoin_part_1_n2 SELECT * FROM srcbucket_mapjoin_part_1_n2_tmp2;
+
+CREATE TABLE srcbucket_mapjoin_part_2_n6_tmp1 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1');
+CREATE TABLE srcbucket_mapjoin_part_2_n6 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(4, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_2_n6 SELECT * FROM srcbucket_mapjoin_part_2_n6_tmp1;
+
+ALTER TABLE srcbucket_mapjoin_part_2_n6 SET PARTITION SPEC (part, bucket(2, key));
+CREATE TABLE srcbucket_mapjoin_part_2_n6_tmp2 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp2 PARTITION (part='2');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp2 PARTITION (part='2');
+INSERT INTO srcbucket_mapjoin_part_2_n6 SELECT * FROM srcbucket_mapjoin_part_2_n6_tmp2;
+
+set hive.convert.join.bucket.mapjoin.tez=true;
+
+-- The partition spec has been changed. This pattern is not supported yet
+
+EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part IS NOT NULL AND b.part IS NOT NULL;
+
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part IS NOT NULL AND b.part IS NOT NULL;
+
+EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part = b.part AND a.part IS NOT NULL AND b.part IS NOT NULL;
+
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part = b.part AND a.part IS NOT NULL AND b.part IS NOT NULL;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_5.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_5.q
new file mode 100644
index 00000000000..e325343bb6a
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_5.q
@@ -0,0 +1,54 @@
+-- Copied from bucketmapjoin12.q
+set hive.explain.user=true;
+
+--! qt:dataset:part
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+CREATE TABLE srcbucket_mapjoin_part_1_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_tmp PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_tmp PARTITION (part='1');
+CREATE TABLE srcbucket_mapjoin_part_1 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_1 SELECT * FROM srcbucket_mapjoin_part_1_tmp;
+
+CREATE TABLE srcbucket_mapjoin_part_2_n0_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n0_tmp PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n0_tmp PARTITION (part='1');
+CREATE TABLE srcbucket_mapjoin_part_2_n0 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_2_n0 SELECT * FROM srcbucket_mapjoin_part_2_n0_tmp;
+
+ALTER TABLE srcbucket_mapjoin_part_2_n0 SET PARTITION SPEC (part);
+
+CREATE TABLE srcbucket_mapjoin_part_3_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE;
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_3_tmp PARTITION (part='1');
+LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_3_tmp PARTITION (part='1');
+CREATE TABLE srcbucket_mapjoin_part_3 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_3 SELECT * FROM srcbucket_mapjoin_part_3_tmp;
+
+ALTER TABLE srcbucket_mapjoin_part_3 SET PARTITION SPEC (part, bucket(2, key));
+set hive.convert.join.bucket.mapjoin.tez=true;
+
+-- The partition bucketing metadata match but one table is not bucketed, bucket map join should still be used
+
+EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
+
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
+
+-- The table bucketing metadata match and one partition is not bucketed, bucket map join should be used
+
+EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
+
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b
+ON a.key = b.key AND a.part = '1' and b.part = '1';
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_6.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_6.q
new file mode 100644
index 00000000000..c4b7dd9b613
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_6.q
@@ -0,0 +1,27 @@
+-- Copied from bucketmapjoin_negative2.q
+set hive.explain.user=true;
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+
+CREATE TABLE srcbucket_mapjoin_n5_tmp(key int, value string) STORED AS TEXTFILE;
+load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_n5_tmp;
+load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_n5_tmp;
+CREATE TABLE srcbucket_mapjoin_n5(key int, value string) PARTITIONED BY SPEC(bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_n5 SELECT * FROM srcbucket_mapjoin_n5_tmp;
+
+CREATE TABLE srcbucket_mapjoin_part_2_n7_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE;
+load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-08');
+load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-09');
+load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-09');
+CREATE TABLE srcbucket_mapjoin_part_2_n7 (key int, value string, ds string) partitioned by spec (ds, bucket(2, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_mapjoin_part_2_n7 SELECT * FROM srcbucket_mapjoin_part_2_n7_tmp;
+
+set hive.convert.join.bucket.mapjoin.tez=true;
+create table bucketmapjoin_tmp_result_n3 (key string , value1 string, value2 string);
+
+explain
+insert overwrite table bucketmapjoin_tmp_result_n3
+select /*+mapjoin(b)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n5 a join srcbucket_mapjoin_part_2_n7 b
+on a.key=b.key;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_7.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_7.q
new file mode 100644
index 00000000000..d6be20c3003
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_7.q
@@ -0,0 +1,128 @@
+--! qt:dataset:src
+set hive.explain.user=true;
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+set hive.convert.join.bucket.mapjoin.tez=true;
+
+CREATE TABLE srcbucket_big(key1 string, key2 string, value string)
+PARTITIONED BY SPEC(bucket(4, key1), bucket(2, key2)) STORED BY ICEBERG;
+INSERT INTO srcbucket_big
+SELECT key AS key1, value AS key2, value FROM src;
+
+-- Using both key1 and key2
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20;
+
+-- Using both key1 and key2, with a predicate
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20;
+
+-- Using only key1
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+ORDER BY a.key1
+LIMIT 20;
+
+-- Using only key1, with a predicate with key1
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20;
+
+-- Using only key1, with a predicate with key2
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key2 NOT IN ('val_0', 'val_100')
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key2 NOT IN ('val_0', 'val_100')
+ORDER BY a.key1
+LIMIT 20;
+
+-- Using only key2
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20;
+
+-- Using both key1, key2, and an additional column
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value AND a.value = b.value
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value AND a.value = b.value
+ORDER BY a.key1
+LIMIT 20;
+
+-- Using only a non bucketing column
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.value = b.value
+ORDER BY a.key1
+LIMIT 20;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.value = b.value
+ORDER BY a.key1
+LIMIT 20;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_8.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_8.q
new file mode 100644
index 00000000000..91b30c7b231
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_bucket_map_join_8.q
@@ -0,0 +1,50 @@
+--! qt:dataset:src
+set hive.explain.user=true;
+set hive.auto.convert.join=true;
+set hive.optimize.dynamic.partition.hashjoin=false;
+set hive.convert.join.bucket.mapjoin.tez=true;
+
+CREATE TABLE srcbucket_big(key int, value string, id int)
+PARTITIONED BY SPEC(bucket(4, key)) STORED BY ICEBERG;
+INSERT INTO srcbucket_big VALUES
+(101, 'val_101', 1),
+(null, 'val_102', 2),
+(103, 'val_103', 3),
+(104, null, 4),
+(105, 'val_105', 5),
+(null, null, 6);
+
+CREATE TABLE src_small(key int, value string);
+INSERT INTO src_small VALUES
+(101, 'val_101'),
+(null, 'val_102'),
+(103, 'val_103'),
+(104, null),
+(105, 'val_105'),
+(null, null);
+
+SELECT * FROM srcbucket_big ORDER BY id;
+
+-- Using the bucket column
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.key = b.key
+ORDER BY a.id;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.key = b.key
+ORDER BY a.id;
+
+-- Using a non-bucket column
+EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.value = b.value
+ORDER BY a.id;
+
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.value = b.value
+ORDER BY a.id;
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out
new file mode 100644
index 00000000000..81d63017866
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_1.q.out
@@ -0,0 +1,614 @@
+PREHOOK: query: create table target_table(date_col date, string_col string, decimal_col decimal(38,0)) partitioned by spec (bucket(7, decimal_col)) stored by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@target_table
+POSTHOOK: query: create table target_table(date_col date, string_col string, decimal_col decimal(38,0)) partitioned by spec (bucket(7, decimal_col)) stored by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@target_table
+PREHOOK: query: insert into table target_table values
+('2017-05-17', 'pipeline', '50000000000000000000441610525'),
+('2018-12-20', 'pipeline', '50000000000000000001048981030'),
+('2020-06-30', 'pipeline', '50000000000000000002332575516'),
+('2021-08-16', 'pipeline', '50000000000000000003897973989'),
+('2017-06-06', 'pipeline', '50000000000000000000449148729'),
+('2017-09-08', 'pipeline', '50000000000000000000525378314'),
+('2022-08-30', 'pipeline', '50000000000000000005905545593'),
+('2022-08-16', 'pipeline', '50000000000000000005905545593'),
+('2018-05-03', 'pipeline', '50000000000000000000750826355'),
+('2020-01-10', 'pipeline', '50000000000000000001816579677'),
+('2021-11-01', 'pipeline', '50000000000000000004269423714'),
+('2017-11-07', 'pipeline', '50000000000000000000585901787'),
+('2019-10-15', 'pipeline', '50000000000000000001598843430'),
+('2020-04-01', 'pipeline', '50000000000000000002035795461'),
+('2020-02-24', 'pipeline', '50000000000000000001932600185'),
+('2020-04-27', 'pipeline', '50000000000000000002108160849'),
+('2016-07-05', 'pipeline', '50000000000000000000054405114'),
+('2020-06-02', 'pipeline', '50000000000000000002234387967'),
+('2020-08-21', 'pipeline', '50000000000000000002529168758'),
+('2021-02-17', 'pipeline', '50000000000000000003158511687')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@target_table
+POSTHOOK: query: insert into table target_table values
+('2017-05-17', 'pipeline', '50000000000000000000441610525'),
+('2018-12-20', 'pipeline', '50000000000000000001048981030'),
+('2020-06-30', 'pipeline', '50000000000000000002332575516'),
+('2021-08-16', 'pipeline', '50000000000000000003897973989'),
+('2017-06-06', 'pipeline', '50000000000000000000449148729'),
+('2017-09-08', 'pipeline', '50000000000000000000525378314'),
+('2022-08-30', 'pipeline', '50000000000000000005905545593'),
+('2022-08-16', 'pipeline', '50000000000000000005905545593'),
+('2018-05-03', 'pipeline', '50000000000000000000750826355'),
+('2020-01-10', 'pipeline', '50000000000000000001816579677'),
+('2021-11-01', 'pipeline', '50000000000000000004269423714'),
+('2017-11-07', 'pipeline', '50000000000000000000585901787'),
+('2019-10-15', 'pipeline', '50000000000000000001598843430'),
+('2020-04-01', 'pipeline', '50000000000000000002035795461'),
+('2020-02-24', 'pipeline', '50000000000000000001932600185'),
+('2020-04-27', 'pipeline', '50000000000000000002108160849'),
+('2016-07-05', 'pipeline', '50000000000000000000054405114'),
+('2020-06-02', 'pipeline', '50000000000000000002234387967'),
+('2020-08-21', 'pipeline', '50000000000000000002529168758'),
+('2021-02-17', 'pipeline', '50000000000000000003158511687')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@target_table
+PREHOOK: query: create table source_table(date_col date, string_col string, decimal_col decimal(38,0)) partitioned by spec (bucket(7, decimal_col)) stored by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@source_table
+POSTHOOK: query: create table source_table(date_col date, string_col string, decimal_col decimal(38,0)) partitioned by spec (bucket(7, decimal_col)) stored by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@source_table
+PREHOOK: query: insert into table source_table values
+('2022-08-30', 'pipeline', '50000000000000000005905545593'),
+('2022-08-16', 'pipeline', '50000000000000000005905545593'),
+('2022-09-01', 'pipeline', '50000000000000000006008686831'),
+('2022-08-30', 'pipeline', '50000000000000000005992620837'),
+('2022-09-01', 'pipeline', '50000000000000000005992620837'),
+('2022-09-01', 'pipeline', '50000000000000000005992621067'),
+('2022-08-30', 'pipeline', '50000000000000000005992621067')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@source_table
+POSTHOOK: query: insert into table source_table values
+('2022-08-30', 'pipeline', '50000000000000000005905545593'),
+('2022-08-16', 'pipeline', '50000000000000000005905545593'),
+('2022-09-01', 'pipeline', '50000000000000000006008686831'),
+('2022-08-30', 'pipeline', '50000000000000000005992620837'),
+('2022-09-01', 'pipeline', '50000000000000000005992620837'),
+('2022-09-01', 'pipeline', '50000000000000000005992621067'),
+('2022-08-30', 'pipeline', '50000000000000000005992621067')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@source_table
+PREHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 2 (CUSTOM_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 llap
+ File Output Operator [FS_10]
+ Select Operator [SEL_9] (rows=30 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_45] (rows=30 width=336)
+ BucketMapJoin:true,Conds:SEL_2._col0, _col1=RS_7._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Map 2 [CUSTOM_EDGE] llap
+ MULTICAST [RS_7]
+ PartitionCols:_col1
+ Select Operator [SEL_5] (rows=3 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_14] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_2] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_13] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
+PREHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Reducer 3 (CUSTOM_EDGE)
+Reducer 3 <- Map 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 llap
+ File Output Operator [FS_14]
+ Select Operator [SEL_13] (rows=10 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_49] (rows=10 width=336)
+ BucketMapJoin:true,Conds:SEL_2._col0, _col1=RS_11._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Reducer 3 [CUSTOM_EDGE] llap
+ MULTICAST [RS_11]
+ PartitionCols:_col1
+ Group By Operator [GBY_8] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Map 2 [SIMPLE_EDGE] llap
+ SHUFFLE [RS_7]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_6] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:date_col, decimal_col
+ Filter Operator [FIL_18] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_2] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_17] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
+PREHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 2 (BROADCAST_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 llap
+ File Output Operator [FS_10]
+ Select Operator [SEL_9] (rows=30 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_45] (rows=30 width=336)
+ Conds:SEL_2._col0, _col1=RS_7._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Map 2 [BROADCAST_EDGE] llap
+ BROADCAST [RS_7]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_5] (rows=3 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_14] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_2] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_13] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
+PREHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Reducer 3 (BROADCAST_EDGE)
+Reducer 3 <- Map 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 llap
+ File Output Operator [FS_14]
+ Select Operator [SEL_13] (rows=10 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_49] (rows=10 width=336)
+ Conds:SEL_2._col0, _col1=RS_11._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Reducer 3 [BROADCAST_EDGE] llap
+ BROADCAST [RS_11]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_8] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Map 2 [SIMPLE_EDGE] llap
+ SHUFFLE [RS_7]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_6] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:date_col, decimal_col
+ Filter Operator [FIL_18] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_2] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_17] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
+PREHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 2 (CUSTOM_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 vectorized, llap
+ File Output Operator [FS_54]
+ Select Operator [SEL_53] (rows=30 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_52] (rows=30 width=336)
+ BucketMapJoin:true,Conds:SEL_51._col0, _col1=RS_49._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Map 2 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_49]
+ PartitionCols:_col1
+ Select Operator [SEL_48] (rows=3 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_47] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_51] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_50] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
+PREHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Reducer 3 (CUSTOM_EDGE)
+Reducer 3 <- Map 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 vectorized, llap
+ File Output Operator [FS_60]
+ Select Operator [SEL_59] (rows=10 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_58] (rows=10 width=336)
+ BucketMapJoin:true,Conds:SEL_57._col0, _col1=RS_55._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Reducer 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_55]
+ PartitionCols:_col1
+ Group By Operator [GBY_54] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Map 2 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_53]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_52] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:date_col, decimal_col
+ Filter Operator [FIL_51] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_57] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_56] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:7,Grouping Partition Columns:["decimal_col"],Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
+PREHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 2 (BROADCAST_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 vectorized, llap
+ File Output Operator [FS_54]
+ Select Operator [SEL_53] (rows=30 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_52] (rows=30 width=336)
+ Conds:SEL_51._col0, _col1=RS_49._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Map 2 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_49]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_48] (rows=3 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_47] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_51] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_50] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
+PREHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Reducer 3 (BROADCAST_EDGE)
+Reducer 3 <- Map 2 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Map 1 vectorized, llap
+ File Output Operator [FS_60]
+ Select Operator [SEL_59] (rows=10 width=520)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Map Join Operator [MAPJOIN_58] (rows=10 width=336)
+ Conds:SEL_57._col0, _col1=RS_55._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3"]
+ <-Reducer 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_55]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_54] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:KEY._col0, KEY._col1
+ <-Map 2 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_53]
+ PartitionCols:_col0, _col1
+ Group By Operator [GBY_52] (rows=1 width=168)
+ Output:["_col0","_col1"],keys:date_col, decimal_col
+ Filter Operator [FIL_51] (rows=3 width=168)
+ predicate:(if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_3] (rows=7 width=168)
+ default@source_table,source_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","decimal_col"]
+ <-Select Operator [SEL_57] (rows=10 width=168)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_56] (rows=10 width=260)
+ predicate:((string_col = 'pipeline') and if(decimal_col is not null, (CAST( decimal_col AS STRING) = '50000000000000000005905545593'), false) and date_col is not null and decimal_col is not null)
+ TableScan [TS_0] (rows=20 width=260)
+ default@target_table,target_table,Tbl:COMPLETE,Col:COMPLETE,Output:["date_col","string_col","decimal_col"]
+
+PREHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+PREHOOK: type: QUERY
+PREHOOK: Input: default@source_table
+PREHOOK: Input: default@target_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from target_table inner join
+(select distinct date_col, 'pipeline' string_col, decimal_col from source_table where coalesce(decimal_col,'') = '50000000000000000005905545593') s
+on s.date_col = target_table.date_col AND s.string_col = target_table.string_col AND s.decimal_col = target_table.decimal_col
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@source_table
+POSTHOOK: Input: default@target_table
+#### A masked pattern was here ####
+2022-08-30 pipeline 50000000000000000005905545593 2022-08-30 pipeline 50000000000000000005905545593
+2022-08-16 pipeline 50000000000000000005905545593 2022-08-16 pipeline 50000000000000000005905545593
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out
new file mode 100644
index 00000000000..413fd81aa95
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_2.q.out
@@ -0,0 +1,548 @@
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_n0_tmp(key int, value string) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_n0_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n0_tmp(key int, value string) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_n0_tmp
+PREHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_n0_tmp
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_n0_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_n0_tmp
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_n0_tmp
+PREHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_n0_tmp
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_n0_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_n0_tmp
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_n0_tmp
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_n0(key int, value string) PARTITIONED BY SPEC (bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_n0
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n0(key int, value string) PARTITIONED BY SPEC (bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_n0
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_n0 SELECT * FROM srcbucket_mapjoin_n0_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n0_tmp
+PREHOOK: Output: default@srcbucket_mapjoin_n0
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_n0 SELECT * FROM srcbucket_mapjoin_n0_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n0_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_n0
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_n0_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_n0_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp
+PREHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+PREHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+PREHOOK: query: load data local inpath '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+PREHOOK: query: load data local inpath '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_n0_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_n0 (key int, value string, ds string) partitioned by spec (ds, bucket(4, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_n0 (key int, value string, ds string) partitioned by spec (ds, bucket(4, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_n0 SELECT * FROM srcbucket_mapjoin_part_n0_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_n0_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+PREHOOK: Input: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+PREHOOK: Output: default@srcbucket_mapjoin_part_n0
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_n0 SELECT * FROM srcbucket_mapjoin_part_n0_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_n0_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-08
+POSTHOOK: Input: default@srcbucket_mapjoin_part_n0_tmp@ds=2008-04-09
+POSTHOOK: Output: default@srcbucket_mapjoin_part_n0
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_tmp
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-09
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-09
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-09
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2 (key int, value string, ds string) partitioned by spec (ds, bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2 (key int, value string, ds string) partitioned by spec (ds, bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_2 SELECT * FROM srcbucket_mapjoin_part_2_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-08
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-09
+PREHOOK: Output: default@srcbucket_mapjoin_part_2
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_2 SELECT * FROM srcbucket_mapjoin_part_2_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-08
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_tmp@ds=2008-04-09
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2
+PREHOOK: query: create table bucketmapjoin_hash_result_1 (key bigint , value1 bigint, value2 bigint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucketmapjoin_hash_result_1
+POSTHOOK: query: create table bucketmapjoin_hash_result_1 (key bigint , value1 bigint, value2 bigint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucketmapjoin_hash_result_1
+PREHOOK: query: create table bucketmapjoin_hash_result_2 (key bigint , value1 bigint, value2 bigint)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucketmapjoin_hash_result_2
+POSTHOOK: query: create table bucketmapjoin_hash_result_2 (key bigint , value1 bigint, value2 bigint)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucketmapjoin_hash_result_2
+PREHOOK: query: create table bucketmapjoin_tmp_result (key string , value1 string, value2 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: query: create table bucketmapjoin_tmp_result (key string , value1 string, value2 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucketmapjoin_tmp_result
+PREHOOK: query: explain
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n0
+PREHOOK: Input: default@srcbucket_mapjoin_part_n0
+PREHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: query: explain
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n0
+POSTHOOK: Input: default@srcbucket_mapjoin_part_n0
+POSTHOOK: Output: default@bucketmapjoin_tmp_result
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 2 <- Map 1 (CUSTOM_EDGE)
+Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE)
+
+Stage-3
+ Stats Work{}
+ Stage-0
+ Move Operator
+ table:{"name:":"default.bucketmapjoin_tmp_result"}
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Reducer 3 vectorized, llap
+ File Output Operator [FS_48]
+ Select Operator [SEL_47] (rows=1 width=798)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
+ Group By Operator [GBY_46] (rows=1 width=500)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"]
+ <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ File Output Operator [FS_42]
+ table:{"name:":"default.bucketmapjoin_tmp_result"}
+ Select Operator [SEL_41] (rows=785 width=366)
+ Output:["_col0","_col1","_col2"]
+ Map Join Operator [MAPJOIN_40] (rows=785 width=186)
+ BucketMapJoin:true,Conds:RS_37._col0=SEL_39._col0(Inner),Output:["_col0","_col1","_col3"]
+ <-Map 1 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_37]
+ PartitionCols:_col0
+ Select Operator [SEL_36] (rows=238 width=95)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_35] (rows=238 width=95)
+ predicate:key is not null
+ TableScan [TS_0] (rows=238 width=95)
+ default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_39] (rows=1000 width=95)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_38] (rows=1000 width=95)
+ predicate:key is not null
+ TableScan [TS_3] (rows=1000 width=95)
+ default@srcbucket_mapjoin_part_n0,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value"]
+ PARTITION_ONLY_SHUFFLE [RS_45]
+ Group By Operator [GBY_44] (rows=1 width=704)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"]
+ Select Operator [SEL_43] (rows=785 width=366)
+ Output:["key","value1","value2"]
+ Please refer to the previous Select Operator [SEL_41]
+
+PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n0
+PREHOOK: Input: default@srcbucket_mapjoin_part_n0
+PREHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n0
+POSTHOOK: Input: default@srcbucket_mapjoin_part_n0
+POSTHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.key EXPRESSION [(srcbucket_mapjoin_n0)a.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin_n0)a.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part_n0)b.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select count(1) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+928
+PREHOOK: query: insert overwrite table bucketmapjoin_hash_result_1
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+PREHOOK: Output: default@bucketmapjoin_hash_result_1
+POSTHOOK: query: insert overwrite table bucketmapjoin_hash_result_1
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+POSTHOOK: Output: default@bucketmapjoin_hash_result_1
+POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ]
+PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n0
+PREHOOK: Input: default@srcbucket_mapjoin_part_n0
+PREHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_n0 b
+on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n0
+POSTHOOK: Input: default@srcbucket_mapjoin_part_n0
+POSTHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.key EXPRESSION [(srcbucket_mapjoin_n0)a.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin_n0)a.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part_n0)b.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select count(1) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+928
+PREHOOK: query: insert overwrite table bucketmapjoin_hash_result_2
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+PREHOOK: Output: default@bucketmapjoin_hash_result_2
+POSTHOOK: query: insert overwrite table bucketmapjoin_hash_result_2
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+POSTHOOK: Output: default@bucketmapjoin_hash_result_2
+POSTHOOK: Lineage: bucketmapjoin_hash_result_2.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_2.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_2.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ]
+PREHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2
+from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b
+on a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_hash_result_1
+PREHOOK: Input: default@bucketmapjoin_hash_result_2
+#### A masked pattern was here ####
+POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2
+from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b
+on a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_hash_result_1
+POSTHOOK: Input: default@bucketmapjoin_hash_result_2
+#### A masked pattern was here ####
+0 0 0
+PREHOOK: query: explain
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n0
+PREHOOK: Input: default@srcbucket_mapjoin_part_2
+PREHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: query: explain
+insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n0
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2
+POSTHOOK: Output: default@bucketmapjoin_tmp_result
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 2 <- Map 1 (CUSTOM_EDGE)
+Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE)
+
+Stage-3
+ Stats Work{}
+ Stage-0
+ Move Operator
+ table:{"name:":"default.bucketmapjoin_tmp_result"}
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Reducer 3 vectorized, llap
+ File Output Operator [FS_48]
+ Select Operator [SEL_47] (rows=1 width=798)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
+ Group By Operator [GBY_46] (rows=1 width=500)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"]
+ <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ File Output Operator [FS_42]
+ table:{"name:":"default.bucketmapjoin_tmp_result"}
+ Select Operator [SEL_41] (rows=809 width=366)
+ Output:["_col0","_col1","_col2"]
+ Map Join Operator [MAPJOIN_40] (rows=809 width=186)
+ BucketMapJoin:true,Conds:RS_37._col0=SEL_39._col0(Inner),Output:["_col0","_col1","_col3"]
+ <-Map 1 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_37]
+ PartitionCols:_col0
+ Select Operator [SEL_36] (rows=238 width=95)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_35] (rows=238 width=95)
+ predicate:key is not null
+ TableScan [TS_0] (rows=238 width=95)
+ default@srcbucket_mapjoin_n0,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_39] (rows=524 width=95)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_38] (rows=524 width=95)
+ predicate:key is not null
+ TableScan [TS_3] (rows=524 width=95)
+ default@srcbucket_mapjoin_part_2,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"]
+ PARTITION_ONLY_SHUFFLE [RS_45]
+ Group By Operator [GBY_44] (rows=1 width=704)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"]
+ Select Operator [SEL_43] (rows=809 width=366)
+ Output:["key","value1","value2"]
+ Please refer to the previous Select Operator [SEL_41]
+
+PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n0
+PREHOOK: Input: default@srcbucket_mapjoin_part_2
+PREHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n0
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2
+POSTHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.key EXPRESSION [(srcbucket_mapjoin_n0)a.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin_n0)a.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part_2)b.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select count(1) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+0
+PREHOOK: query: insert overwrite table bucketmapjoin_hash_result_1
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+PREHOOK: Output: default@bucketmapjoin_hash_result_1
+POSTHOOK: query: insert overwrite table bucketmapjoin_hash_result_1
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+POSTHOOK: Output: default@bucketmapjoin_hash_result_1
+POSTHOOK: Lineage: bucketmapjoin_hash_result_1.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_1.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ]
+PREHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n0
+PREHOOK: Input: default@srcbucket_mapjoin_part_2
+PREHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: query: insert overwrite table bucketmapjoin_tmp_result
+select /*+mapjoin(a)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n0 a join srcbucket_mapjoin_part_2 b
+on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n0
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2
+POSTHOOK: Output: default@bucketmapjoin_tmp_result
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.key EXPRESSION [(srcbucket_mapjoin_n0)a.FieldSchema(name:key, type:int, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value1 SIMPLE [(srcbucket_mapjoin_n0)a.FieldSchema(name:value, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_tmp_result.value2 SIMPLE [(srcbucket_mapjoin_part_2)b.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: select count(1) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+#### A masked pattern was here ####
+0
+PREHOOK: query: insert overwrite table bucketmapjoin_hash_result_2
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_tmp_result
+PREHOOK: Output: default@bucketmapjoin_hash_result_2
+POSTHOOK: query: insert overwrite table bucketmapjoin_hash_result_2
+select sum(hash(key)), sum(hash(value1)), sum(hash(value2)) from bucketmapjoin_tmp_result
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_tmp_result
+POSTHOOK: Output: default@bucketmapjoin_hash_result_2
+POSTHOOK: Lineage: bucketmapjoin_hash_result_2.key EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:key, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_2.value1 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value1, type:string, comment:null), ]
+POSTHOOK: Lineage: bucketmapjoin_hash_result_2.value2 EXPRESSION [(bucketmapjoin_tmp_result)bucketmapjoin_tmp_result.FieldSchema(name:value2, type:string, comment:null), ]
+PREHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2
+from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b
+on a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@bucketmapjoin_hash_result_1
+PREHOOK: Input: default@bucketmapjoin_hash_result_2
+#### A masked pattern was here ####
+POSTHOOK: query: select a.key-b.key, a.value1-b.value1, a.value2-b.value2
+from bucketmapjoin_hash_result_1 a left outer join bucketmapjoin_hash_result_2 b
+on a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@bucketmapjoin_hash_result_1
+POSTHOOK: Input: default@bucketmapjoin_hash_result_2
+#### A masked pattern was here ####
+NULL NULL NULL
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out
new file mode 100644
index 00000000000..bd82280c9c0
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_3.q.out
@@ -0,0 +1,238 @@
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n1_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n1_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n1_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n1_tmp
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n1_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n1_tmp
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n1_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n1_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n1_tmp@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n1_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n1_tmp@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n1_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n1_tmp@part=1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n1 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n1
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n1 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n1
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_1_n1 SELECT * FROM srcbucket_mapjoin_part_1_n1_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n1_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n1_tmp@part=1
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n1
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_1_n1 SELECT * FROM srcbucket_mapjoin_part_1_n1_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n1_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n1_tmp@part=1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n4_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n4_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n4_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4_tmp
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n4_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n4_tmp
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n4_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4_tmp@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n4_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n4_tmp@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n4_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4_tmp@part=1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n4 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n4
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n4 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n4 SELECT * FROM srcbucket_mapjoin_part_2_n4_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4_tmp@part=1
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n4
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n4 SELECT * FROM srcbucket_mapjoin_part_2_n4_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4_tmp@part=1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4
+PREHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n4 SET PARTITION SPEC (part, bucket(3, key))
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+POSTHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n4 SET PARTITION SPEC (part, bucket(3, key))
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_40]
+ Group By Operator [GBY_39] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_38]
+ Group By Operator [GBY_37] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_36] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_31] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_3] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+ <-Select Operator [SEL_35] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_34] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_0] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
+
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+464
+PREHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n4 SET PARTITION SPEC (part, bucket(2, value))
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+POSTHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n4 SET PARTITION SPEC (part, bucket(2, value))
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n4
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_40]
+ Group By Operator [GBY_39] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_38]
+ Group By Operator [GBY_37] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_36] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_31] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_3] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_2_n4,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+ <-Select Operator [SEL_35] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_34] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_0] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_1_n1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
+
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n1 a JOIN srcbucket_mapjoin_part_2_n4 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n4
+#### A masked pattern was here ####
+464
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out
new file mode 100644
index 00000000000..4ddd445ecbf
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_4.q.out
@@ -0,0 +1,350 @@
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n2_tmp1 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp1
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n2_tmp1 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp1 PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp1 PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp1@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp1 PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp1@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp1 PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp1@part=1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n2 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n2 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_1_n2 SELECT * FROM srcbucket_mapjoin_part_1_n2_tmp1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp1
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp1@part=1
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_1_n2 SELECT * FROM srcbucket_mapjoin_part_1_n2_tmp1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp1@part=1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: query: ALTER TABLE srcbucket_mapjoin_part_1_n2 SET PARTITION SPEC (part, bucket(4, key))
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: query: ALTER TABLE srcbucket_mapjoin_part_1_n2 SET PARTITION SPEC (part, bucket(4, key))
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n2_tmp2 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_n2_tmp2 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_1_n2_tmp2 PARTITION (part='2')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_1_n2 SELECT * FROM srcbucket_mapjoin_part_1_n2_tmp2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp2
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_1_n2 SELECT * FROM srcbucket_mapjoin_part_1_n2_tmp2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp2
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2_tmp2@part=2
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n6_tmp1 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n6_tmp1 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000002_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000003_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp1 PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n6 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(4, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n6 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(4, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n6 SELECT * FROM srcbucket_mapjoin_part_2_n6_tmp1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp1
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n6 SELECT * FROM srcbucket_mapjoin_part_2_n6_tmp1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp1@part=1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6
+PREHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n6 SET PARTITION SPEC (part, bucket(2, key))
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+POSTHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n6 SET PARTITION SPEC (part, bucket(2, key))
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n6_tmp2 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp2
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n6_tmp2 (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp2 PARTITION (part='2')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp2 PARTITION (part='2')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp2
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp2@part=2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp2 PARTITION (part='2')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp2@part=2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n6_tmp2 PARTITION (part='2')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6_tmp2@part=2
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n6 SELECT * FROM srcbucket_mapjoin_part_2_n6_tmp2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp2
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp2@part=2
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n6
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n6 SELECT * FROM srcbucket_mapjoin_part_2_n6_tmp2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp2
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6_tmp2@part=2
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n6
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part IS NOT NULL AND b.part IS NOT NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part IS NOT NULL AND b.part IS NOT NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (BROADCAST_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_40]
+ Group By Operator [GBY_39] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_38]
+ Group By Operator [GBY_37] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_36] (rows=1797 width=8)
+ Conds:SEL_35._col0=RS_33._col0(Inner)
+ <-Map 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=738 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_31] (rows=738 width=89)
+ predicate:(part is not null and key is not null)
+ TableScan [TS_3] (rows=738 width=89)
+ default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+ <-Select Operator [SEL_35] (rows=738 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_34] (rows=738 width=89)
+ predicate:(part is not null and key is not null)
+ TableScan [TS_0] (rows=738 width=89)
+ default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part IS NOT NULL AND b.part IS NOT NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part IS NOT NULL AND b.part IS NOT NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+2420
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part = b.part AND a.part IS NOT NULL AND b.part IS NOT NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part = b.part AND a.part IS NOT NULL AND b.part IS NOT NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (BROADCAST_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_49]
+ Group By Operator [GBY_48] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_47]
+ Group By Operator [GBY_46] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_45] (rows=1797 width=8)
+ Conds:SEL_44._col0, _col1=RS_39._col0, _col1(Inner)
+ <-Map 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_39]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_38] (rows=738 width=89)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_37] (rows=738 width=89)
+ predicate:(part is not null and key is not null)
+ TableScan [TS_3] (rows=738 width=89)
+ default@srcbucket_mapjoin_part_2_n6,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+ Dynamic Partitioning Event Operator [EVENT_42] (rows=2 width=85)
+ Group By Operator [GBY_41] (rows=2 width=85)
+ Output:["_col0"],keys:_col0
+ Select Operator [SEL_40] (rows=738 width=85)
+ Output:["_col0"]
+ Please refer to the previous Select Operator [SEL_38]
+ <-Select Operator [SEL_44] (rows=738 width=89)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_43] (rows=738 width=89)
+ predicate:(part is not null and key is not null)
+ TableScan [TS_0] (rows=738 width=89)
+ default@srcbucket_mapjoin_part_1_n2,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part = b.part AND a.part IS NOT NULL AND b.part IS NOT NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1_n2 a JOIN srcbucket_mapjoin_part_2_n6 b
+ON a.key = b.key AND a.part = b.part AND a.part IS NOT NULL AND b.part IS NOT NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_n2
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n6
+#### A masked pattern was here ####
+928
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out
new file mode 100644
index 00000000000..4b92f571e02
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_5.q.out
@@ -0,0 +1,283 @@
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_tmp
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_tmp
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_1_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_tmp@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_1_tmp@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_1_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1_tmp@part=1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_1
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_1 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_1 SELECT * FROM srcbucket_mapjoin_part_1_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_1_tmp@part=1
+PREHOOK: Output: default@srcbucket_mapjoin_part_1
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_1 SELECT * FROM srcbucket_mapjoin_part_1_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1_tmp@part=1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n0_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n0_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n0_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n0_tmp
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n0_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n0_tmp
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n0_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n0_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n0_tmp@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n0_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n0_tmp@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n0_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n0_tmp@part=1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n0 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n0
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n0 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part, bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n0
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n0 SELECT * FROM srcbucket_mapjoin_part_2_n0_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n0_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n0_tmp@part=1
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n0
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n0 SELECT * FROM srcbucket_mapjoin_part_2_n0_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n0_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n0_tmp@part=1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n0
+PREHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n0 SET PARTITION SPEC (part)
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n0
+POSTHOOK: query: ALTER TABLE srcbucket_mapjoin_part_2_n0 SET PARTITION SPEC (part)
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n0
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n0
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_3_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_3_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_3_tmp (key INT, value STRING) PARTITIONED BY (part STRING)
+STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_3_tmp
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_3_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_3_tmp
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_part_3_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_3_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_3_tmp@part=1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_3_tmp PARTITION (part='1')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_3_tmp@part=1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_part_3_tmp PARTITION (part='1')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_3_tmp@part=1
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_3 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_3
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_3 (key INT, value STRING, part STRING) PARTITIONED BY SPEC (part) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_3
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_3 SELECT * FROM srcbucket_mapjoin_part_3_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_3_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_3_tmp@part=1
+PREHOOK: Output: default@srcbucket_mapjoin_part_3
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_3 SELECT * FROM srcbucket_mapjoin_part_3_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_3_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_3_tmp@part=1
+POSTHOOK: Output: default@srcbucket_mapjoin_part_3
+PREHOOK: query: ALTER TABLE srcbucket_mapjoin_part_3 SET PARTITION SPEC (part, bucket(2, key))
+PREHOOK: type: ALTERTABLE_SETPARTSPEC
+PREHOOK: Input: default@srcbucket_mapjoin_part_3
+POSTHOOK: query: ALTER TABLE srcbucket_mapjoin_part_3 SET PARTITION SPEC (part, bucket(2, key))
+POSTHOOK: type: ALTERTABLE_SETPARTSPEC
+POSTHOOK: Input: default@srcbucket_mapjoin_part_3
+POSTHOOK: Output: default@srcbucket_mapjoin_part_3
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n0
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n0
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_40]
+ Group By Operator [GBY_39] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_38]
+ Group By Operator [GBY_37] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_36] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_31] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_3] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_2_n0,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+ <-Select Operator [SEL_35] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_34] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_0] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
+
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n0
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_2_n0 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n0
+#### A masked pattern was here ####
+464
+PREHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1
+PREHOOK: Input: default@srcbucket_mapjoin_part_3
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_3
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_40]
+ Group By Operator [GBY_39] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count(VALUE._col0)"]
+ <-Map 1 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ PARTITION_ONLY_SHUFFLE [RS_38]
+ Group By Operator [GBY_37] (rows=1 width=8)
+ Output:["_col0"],aggregations:["count()"]
+ Map Join Operator [MAPJOIN_36] (rows=372 width=8)
+ BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner)
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_31] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_3] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_3,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","part"]
+ <-Select Operator [SEL_35] (rows=238 width=4)
+ Output:["_col0"]
+ Filter Operator [FIL_34] (rows=238 width=89)
+ predicate:((part = '1') and key is not null)
+ TableScan [TS_0] (rows=238 width=89)
+ default@srcbucket_mapjoin_part_1,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","part"]
+
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_1
+PREHOOK: Input: default@srcbucket_mapjoin_part_3
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ count(*)
+FROM srcbucket_mapjoin_part_1 a JOIN srcbucket_mapjoin_part_3 b
+ON a.key = b.key AND a.part = '1' and b.part = '1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_1
+POSTHOOK: Input: default@srcbucket_mapjoin_part_3
+#### A masked pattern was here ####
+464
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out
new file mode 100644
index 00000000000..11477c98ecc
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_6.q.out
@@ -0,0 +1,177 @@
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_n5_tmp(key int, value string) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_n5_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n5_tmp(key int, value string) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_n5_tmp
+PREHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_n5_tmp
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_n5_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000000_0' INTO TABLE srcbucket_mapjoin_n5_tmp
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_n5_tmp
+PREHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_n5_tmp
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_n5_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj/000001_0' INTO TABLE srcbucket_mapjoin_n5_tmp
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_n5_tmp
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_n5(key int, value string) PARTITIONED BY SPEC(bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_n5
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_n5(key int, value string) PARTITIONED BY SPEC(bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_n5
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_n5 SELECT * FROM srcbucket_mapjoin_n5_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n5_tmp
+PREHOOK: Output: default@srcbucket_mapjoin_n5
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_n5 SELECT * FROM srcbucket_mapjoin_n5_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n5_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_n5
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n7_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n7_tmp (key int, value string) partitioned by (ds string) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-08')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-08
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-08')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-08
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000000_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-09
+PREHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-09')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-09
+POSTHOOK: query: load data local inpath '../../data/files/bmj2/000001_0' INTO TABLE srcbucket_mapjoin_part_2_n7_tmp partition(ds='2008-04-09')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-09
+PREHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n7 (key int, value string, ds string) partitioned by spec (ds, bucket(2, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n7
+POSTHOOK: query: CREATE TABLE srcbucket_mapjoin_part_2_n7 (key int, value string, ds string) partitioned by spec (ds, bucket(2, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7
+PREHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n7 SELECT * FROM srcbucket_mapjoin_part_2_n7_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n7_tmp
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-08
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-09
+PREHOOK: Output: default@srcbucket_mapjoin_part_2_n7
+POSTHOOK: query: INSERT INTO srcbucket_mapjoin_part_2_n7 SELECT * FROM srcbucket_mapjoin_part_2_n7_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n7_tmp
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-08
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n7_tmp@ds=2008-04-09
+POSTHOOK: Output: default@srcbucket_mapjoin_part_2_n7
+PREHOOK: query: create table bucketmapjoin_tmp_result_n3 (key string , value1 string, value2 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@bucketmapjoin_tmp_result_n3
+POSTHOOK: query: create table bucketmapjoin_tmp_result_n3 (key string , value1 string, value2 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@bucketmapjoin_tmp_result_n3
+PREHOOK: query: explain
+insert overwrite table bucketmapjoin_tmp_result_n3
+select /*+mapjoin(b)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n5 a join srcbucket_mapjoin_part_2_n7 b
+on a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_mapjoin_n5
+PREHOOK: Input: default@srcbucket_mapjoin_part_2_n7
+PREHOOK: Output: default@bucketmapjoin_tmp_result_n3
+POSTHOOK: query: explain
+insert overwrite table bucketmapjoin_tmp_result_n3
+select /*+mapjoin(b)*/ a.key, a.value, b.value
+from srcbucket_mapjoin_n5 a join srcbucket_mapjoin_part_2_n7 b
+on a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_mapjoin_n5
+POSTHOOK: Input: default@srcbucket_mapjoin_part_2_n7
+POSTHOOK: Output: default@bucketmapjoin_tmp_result_n3
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 2 <- Map 1 (CUSTOM_EDGE)
+Reducer 3 <- Map 2 (CUSTOM_SIMPLE_EDGE)
+
+Stage-3
+ Stats Work{}
+ Stage-0
+ Move Operator
+ table:{"name:":"default.bucketmapjoin_tmp_result_n3"}
+ Stage-2
+ Dependency Collection{}
+ Stage-1
+ Reducer 3 vectorized, llap
+ File Output Operator [FS_48]
+ Select Operator [SEL_47] (rows=1 width=798)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17"]
+ Group By Operator [GBY_46] (rows=1 width=500)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(VALUE._col0)","avg(VALUE._col1)","count(VALUE._col2)","count(VALUE._col3)","compute_bit_vector_hll(VALUE._col4)","max(VALUE._col5)","avg(VALUE._col6)","count(VALUE._col7)","compute_bit_vector_hll(VALUE._col8)","max(VALUE._col9)","avg(VALUE._col10)","count(VALUE._col11)","compute_bit_vector_hll(VALUE._col12)"]
+ <-Map 2 [CUSTOM_SIMPLE_EDGE] vectorized, llap
+ File Output Operator [FS_42]
+ table:{"name:":"default.bucketmapjoin_tmp_result_n3"}
+ Select Operator [SEL_41] (rows=809 width=366)
+ Output:["_col0","_col1","_col2"]
+ Map Join Operator [MAPJOIN_40] (rows=809 width=186)
+ BucketMapJoin:true,Conds:RS_37._col0=SEL_39._col0(Inner),Output:["_col0","_col1","_col3"]
+ <-Map 1 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_37]
+ PartitionCols:_col0
+ Select Operator [SEL_36] (rows=238 width=95)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_35] (rows=238 width=95)
+ predicate:key is not null
+ TableScan [TS_0] (rows=238 width=95)
+ default@srcbucket_mapjoin_n5,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_39] (rows=524 width=95)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_38] (rows=524 width=95)
+ predicate:key is not null
+ TableScan [TS_3] (rows=524 width=95)
+ default@srcbucket_mapjoin_part_2_n7,b,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:2,Grouping Partition Columns:["key"],Output:["key","value"]
+ PARTITION_ONLY_SHUFFLE [RS_45]
+ Group By Operator [GBY_44] (rows=1 width=704)
+ Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12"],aggregations:["max(length(key))","avg(COALESCE(length(key),0))","count(1)","count(key)","compute_bit_vector_hll(key)","max(length(value1))","avg(COALESCE(length(value1),0))","count(value1)","compute_bit_vector_hll(value1)","max(length(value2))","avg(COALESCE(length(value2),0))","count(value2)","compute_bit_vector_hll(value2)"]
+ Select Operator [SEL_43] (rows=809 width=366)
+ Output:["key","value1","value2"]
+ Please refer to the previous Select Operator [SEL_41]
+
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out
new file mode 100644
index 00000000000..6701fbaf410
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_7.q.out
@@ -0,0 +1,800 @@
+PREHOOK: query: CREATE TABLE srcbucket_big(key1 string, key2 string, value string)
+PARTITIONED BY SPEC(bucket(4, key1), bucket(2, key2)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_big
+POSTHOOK: query: CREATE TABLE srcbucket_big(key1 string, key2 string, value string)
+PARTITIONED BY SPEC(bucket(4, key1), bucket(2, key2)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_big
+PREHOOK: query: INSERT INTO srcbucket_big
+SELECT key AS key1, value AS key2, value FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@srcbucket_big
+POSTHOOK: query: INSERT INTO srcbucket_big
+SELECT key AS key1, value AS key2, value FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@srcbucket_big
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_61]
+ Limit [LIM_60] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_59] (rows=791 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_58]
+ Top N Key Operator [TNK_57] (rows=791 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_56] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_55._col0, _col1=RS_53._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_53]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_52] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_51] (rows=500 width=178)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_55] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_54] (rows=500 width=269)
+ predicate:(key1 is not null and key2 is not null)
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+10 val_10 val_10 10 val_10
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_61]
+ Limit [LIM_60] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_59] (rows=791 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_58]
+ Top N Key Operator [TNK_57] (rows=791 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_56] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_55._col0, _col1=RS_53._col0, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_53]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_52] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_51] (rows=500 width=178)
+ predicate:((key <> '0') and (key <> '100') and value is not null)
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_55] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_54] (rows=500 width=269)
+ predicate:((key1 <> '0') and (key1 <> '100') and key2 is not null)
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+10 val_10 val_10 10 val_10
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+105 val_105 val_105 105 val_105
+11 val_11 val_11 11 val_11
+111 val_111 val_111 111 val_111
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+114 val_114 val_114 114 val_114
+116 val_116 val_116 116 val_116
+118 val_118 val_118 118 val_118
+118 val_118 val_118 118 val_118
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_41]
+ Limit [LIM_40] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_39] (rows=791 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_38]
+ Top N Key Operator [TNK_37] (rows=791 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_36] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_31] (rows=500 width=178)
+ predicate:key is not null
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_35] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_34] (rows=500 width=269)
+ predicate:key1 is not null
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+10 val_10 val_10 10 val_10
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_41]
+ Limit [LIM_40] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_39] (rows=791 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_38]
+ Top N Key Operator [TNK_37] (rows=791 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_36] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_31] (rows=500 width=178)
+ predicate:((key <> '0') and (key <> '100'))
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_35] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_34] (rows=500 width=269)
+ predicate:((key1 <> '0') and (key1 <> '100'))
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key1 NOT IN ('0', '100')
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+10 val_10 val_10 10 val_10
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+105 val_105 val_105 105 val_105
+11 val_11 val_11 11 val_11
+111 val_111 val_111 111 val_111
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+114 val_114 val_114 114 val_114
+116 val_116 val_116 116 val_116
+118 val_118 val_118 118 val_118
+118 val_118 val_118 118 val_118
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key2 NOT IN ('val_0', 'val_100')
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key2 NOT IN ('val_0', 'val_100')
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_41]
+ Limit [LIM_40] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_39] (rows=791 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_38]
+ Top N Key Operator [TNK_37] (rows=791 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_36] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_35._col0=RS_33._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_33]
+ PartitionCols:_col0
+ Select Operator [SEL_32] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_31] (rows=500 width=178)
+ predicate:key is not null
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_35] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_34] (rows=500 width=269)
+ predicate:((key2 <> 'val_0') and (key2 <> 'val_100') and key1 is not null)
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key1"],Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key2 NOT IN ('val_0', 'val_100')
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key
+WHERE a.key2 NOT IN ('val_0', 'val_100')
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+10 val_10 val_10 10 val_10
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+105 val_105 val_105 105 val_105
+11 val_11 val_11 11 val_11
+111 val_111 val_111 111 val_111
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+113 val_113 val_113 113 val_113
+114 val_114 val_114 114 val_114
+116 val_116 val_116 116 val_116
+118 val_118 val_118 118 val_118
+118 val_118 val_118 118 val_118
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (BROADCAST_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_41]
+ Limit [LIM_40] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_39] (rows=814 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_38]
+ Top N Key Operator [TNK_37] (rows=814 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_36] (rows=814 width=447)
+ Conds:SEL_35._col1=RS_33._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_33]
+ PartitionCols:_col1
+ Select Operator [SEL_32] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_31] (rows=500 width=178)
+ predicate:value is not null
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_35] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_34] (rows=500 width=269)
+ predicate:key2 is not null
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key2 = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+10 val_10 val_10 10 val_10
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value AND a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value AND a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_66]
+ Limit [LIM_65] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_64] (rows=791 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_63]
+ Top N Key Operator [TNK_62] (rows=791 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_61] (rows=791 width=447)
+ BucketMapJoin:true,Conds:SEL_60._col0, _col1, _col2=RS_58._col0, _col1, _col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_58]
+ PartitionCols:_col0, _col1
+ Select Operator [SEL_57] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_56] (rows=500 width=178)
+ predicate:(key is not null and value is not null)
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_60] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_59] (rows=500 width=269)
+ predicate:(key1 is not null and key2 is not null and value is not null)
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:8,Grouping Partition Columns:["key1","key2"],Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value AND a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.key1 = b.key AND a.key2 = b.value AND a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+10 val_10 val_10 10 val_10
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (BROADCAST_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:20
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_41]
+ Limit [LIM_40] (rows=20 width=447)
+ Number of rows:20
+ Select Operator [SEL_39] (rows=814 width=447)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_38]
+ Top N Key Operator [TNK_37] (rows=814 width=447)
+ keys:_col0,top n:20
+ Map Join Operator [MAPJOIN_36] (rows=814 width=447)
+ Conds:SEL_35._col2=RS_33._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_33]
+ PartitionCols:_col1
+ Select Operator [SEL_32] (rows=500 width=178)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_31] (rows=500 width=178)
+ predicate:value is not null
+ TableScan [TS_3] (rows=500 width=178)
+ default@src,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_35] (rows=500 width=269)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_34] (rows=500 width=269)
+ predicate:value is not null
+ TableScan [TS_0] (rows=500 width=269)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key1","key2","value"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src b ON a.value = b.value
+ORDER BY a.key1
+LIMIT 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+0 val_0 val_0 0 val_0
+10 val_10 val_10 10 val_10
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+100 val_100 val_100 100 val_100
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+103 val_103 val_103 103 val_103
+104 val_104 val_104 104 val_104
+104 val_104 val_104 104 val_104
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out
new file mode 100644
index 00000000000..b3e5082c302
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_bucket_map_join_8.q.out
@@ -0,0 +1,218 @@
+PREHOOK: query: CREATE TABLE srcbucket_big(key int, value string, id int)
+PARTITIONED BY SPEC(bucket(4, key)) STORED BY ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcbucket_big
+POSTHOOK: query: CREATE TABLE srcbucket_big(key int, value string, id int)
+PARTITIONED BY SPEC(bucket(4, key)) STORED BY ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcbucket_big
+PREHOOK: query: INSERT INTO srcbucket_big VALUES
+(101, 'val_101', 1),
+(null, 'val_102', 2),
+(103, 'val_103', 3),
+(104, null, 4),
+(105, 'val_105', 5),
+(null, null, 6)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@srcbucket_big
+POSTHOOK: query: INSERT INTO srcbucket_big VALUES
+(101, 'val_101', 1),
+(null, 'val_102', 2),
+(103, 'val_103', 3),
+(104, null, 4),
+(105, 'val_105', 5),
+(null, null, 6)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@srcbucket_big
+PREHOOK: query: CREATE TABLE src_small(key int, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_small
+POSTHOOK: query: CREATE TABLE src_small(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_small
+PREHOOK: query: INSERT INTO src_small VALUES
+(101, 'val_101'),
+(null, 'val_102'),
+(103, 'val_103'),
+(104, null),
+(105, 'val_105'),
+(null, null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@src_small
+POSTHOOK: query: INSERT INTO src_small VALUES
+(101, 'val_101'),
+(null, 'val_102'),
+(103, 'val_103'),
+(104, null),
+(105, 'val_105'),
+(null, null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@src_small
+POSTHOOK: Lineage: src_small.key SCRIPT []
+POSTHOOK: Lineage: src_small.value SCRIPT []
+PREHOOK: query: SELECT * FROM srcbucket_big ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM srcbucket_big ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+101 val_101 1
+NULL val_102 2
+103 val_103 3
+104 NULL 4
+105 val_105 5
+NULL NULL 6
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.key = b.key
+ORDER BY a.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_small
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.key = b.key
+ORDER BY a.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_small
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (CUSTOM_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_37]
+ Select Operator [SEL_36] (rows=4 width=190)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_35]
+ Map Join Operator [MAPJOIN_34] (rows=4 width=190)
+ BucketMapJoin:true,Conds:SEL_33._col0=RS_31._col0(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [CUSTOM_EDGE] vectorized, llap
+ MULTICAST [RS_31]
+ PartitionCols:_col0
+ Select Operator [SEL_30] (rows=4 width=93)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_29] (rows=4 width=93)
+ predicate:key is not null
+ TableScan [TS_3] (rows=6 width=77)
+ default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_33] (rows=4 width=97)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_32] (rows=4 width=97)
+ predicate:key is not null
+ TableScan [TS_0] (rows=6 width=81)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:4,Grouping Partition Columns:["key"],Output:["key","value","id"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.key = b.key
+ORDER BY a.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_small
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.key = b.key
+ORDER BY a.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_small
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+101 val_101 1 101 val_101
+103 val_103 3 103 val_103
+104 NULL 4 104 NULL
+105 val_105 5 105 val_105
+PREHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.value = b.value
+ORDER BY a.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_small
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.value = b.value
+ORDER BY a.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_small
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+Plan optimized by CBO.
+
+Vertex dependency in root stage
+Map 1 <- Map 3 (BROADCAST_EDGE)
+Reducer 2 <- Map 1 (SIMPLE_EDGE)
+
+Stage-0
+ Fetch Operator
+ limit:-1
+ Stage-1
+ Reducer 2 vectorized, llap
+ File Output Operator [FS_37]
+ Select Operator [SEL_36] (rows=4 width=190)
+ Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 1 [SIMPLE_EDGE] vectorized, llap
+ SHUFFLE [RS_35]
+ Map Join Operator [MAPJOIN_34] (rows=4 width=190)
+ Conds:SEL_33._col1=RS_31._col1(Inner),Output:["_col0","_col1","_col2","_col3","_col4"]
+ <-Map 3 [BROADCAST_EDGE] vectorized, llap
+ BROADCAST [RS_31]
+ PartitionCols:_col1
+ Select Operator [SEL_30] (rows=4 width=93)
+ Output:["_col0","_col1"]
+ Filter Operator [FIL_29] (rows=4 width=93)
+ predicate:value is not null
+ TableScan [TS_3] (rows=6 width=77)
+ default@src_small,b,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value"]
+ <-Select Operator [SEL_33] (rows=4 width=97)
+ Output:["_col0","_col1","_col2"]
+ Filter Operator [FIL_32] (rows=4 width=97)
+ predicate:value is not null
+ TableScan [TS_0] (rows=6 width=81)
+ default@srcbucket_big,a,Tbl:COMPLETE,Col:COMPLETE,Output:["key","value","id"]
+
+PREHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.value = b.value
+ORDER BY a.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_small
+PREHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT *
+FROM srcbucket_big a
+JOIN src_small b ON a.value = b.value
+ORDER BY a.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_small
+POSTHOOK: Input: default@srcbucket_big
+#### A masked pattern was here ####
+101 val_101 1 101 val_101
+NULL val_102 2 NULL val_102
+103 val_103 3 103 val_103
+105 val_105 5 105 val_105
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out
index 02c4de2e254..3716f48cac7 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_merge_mixed.q.out
@@ -325,7 +325,7 @@ STAGE PLANS:
Tez
#### A masked pattern was here ####
Edges:
- Map 1 <- Map 5 (BROADCAST_EDGE)
+ Map 1 <- Map 5 (CUSTOM_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Map 1 (SIMPLE_EDGE)
Reducer 4 <- Map 1 (SIMPLE_EDGE)
@@ -556,12 +556,13 @@ STAGE PLANS:
key expressions: _col8 (type: int), _col7 (type: int)
null sort order: zz
sort order: ++
- Map-reduce partition columns: _col8 (type: int), _col7 (type: int)
+ Map-reduce partition columns: _col7 (type: int)
Reduce Sink Vectorization:
- className: VectorReduceSinkMultiKeyOperator
+ className: VectorReduceSinkObjectHashOperator
keyColumns: 3:int, 2:int
native: true
nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 2:int
valueColumns: 23:int, 24:bigint, 25:string, 26:bigint, 27:string, 28:int, 1:int, 4:int, 5:int, 6:int, 7:int, 8:int, 9:int, 10:int, 11:decimal(7,2), 12:decimal(7,2), 13:decimal(7,2), 14:decimal(7,2), 15:decimal(7,2), 16:decimal(7,2), 17:decimal(7,2), 18:decimal(7,2), 19:decimal(7,2), 20:decimal(7,2), 21:decimal(7,2), 22:decimal(7,2)
Statistics: Num rows: 2 Data size: 2176 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: int), _col1 (type: bigint), _col2 (type: string), _col3 (type: bigint), _col4 (type: string), _col5 (type: int), _col6 (type: int), _col9 (type: int), _col10 (type: int), _col11 (type: int), _col12 (type: int), _col13 (type: int), _col14 (type: int), _col15 (type: int), _col16 (type: decimal(7,2)), _col17 (type: decimal(7,2)), _col18 (type: decimal(7,2)), _col19 (type: decimal(7,2)), _col20 (type: decimal(7,2)), _col21 (type: decima [...]
@@ -884,7 +885,7 @@ POSTHOOK: Output: default@merge_tmp_table
POSTHOOK: Output: default@store_sales
POSTHOOK: Output: default@store_sales
Vertex dependency in root stage
-Map 1 <- Map 5 (BROADCAST_EDGE)
+Map 1 <- Map 5 (CUSTOM_EDGE)
Reducer 2 <- Map 1 (SIMPLE_EDGE)
Reducer 3 <- Map 1 (SIMPLE_EDGE)
Reducer 4 <- Map 1 (SIMPLE_EDGE)
@@ -912,10 +913,10 @@ Stage-6
Select Operator [SEL_49] (rows=7 width=1029)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_col47","_col48","_col49"]
Map Join Operator [MAPJOIN_48] (rows=7 width=1029)
- Conds:SEL_47._col2, _col1=RS_46._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_col45","_col46","_co [...]
- <-Map 5 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_46]
- PartitionCols:_col8, _col7
+ BucketMapJoin:true,Conds:SEL_47._col2, _col1=RS_46._col8, _col7(Left Outer),Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col24","_col25","_col26","_col27","_col28","_col29","_col30","_col31","_col32","_col33","_col34","_col35","_col36","_col37","_col38","_col39","_col40","_col41","_col42","_col43","_col44","_c [...]
+ <-Map 5 [CUSTOM_EDGE] vectorized
+ MULTICAST [RS_46]
+ PartitionCols:_col7
Select Operator [SEL_45] (rows=2 width=1088)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23","_col24","_col25","_col26","_col27"]
Filter Operator [FIL_44] (rows=2 width=700)
@@ -925,7 +926,7 @@ Stage-6
<-Select Operator [SEL_47] (rows=5 width=380)
Output:["_col0","_col1","_col2","_col3","_col4","_col5","_col6","_col7","_col8","_col9","_col10","_col11","_col12","_col13","_col14","_col15","_col16","_col17","_col18","_col19","_col20","_col21","_col22","_col23"]
TableScan [TS_0] (rows=5 width=372)
- default@ssv,s,Tbl:COMPLETE,Col:COMPLETE,Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_paid_inc_tax","ss_net_profit"]
+ default@ssv,s,Tbl:COMPLETE,Col:COMPLETE,Grouping Num Buckets:3,Grouping Partition Columns:["ss_item_sk2"],Output:["ss_sold_time_sk","ss_item_sk2","ss_customer_sk2","ss_cdemo_sk","ss_hdemo_sk","ss_addr_sk","ss_store_sk","ss_promo_sk","ss_ticket_number","ss_quantity","ss_wholesale_cost","ss_list_price","ss_sales_price","ss_ext_discount_amt","ss_ext_sales_price","ss_ext_wholesale_cost","ss_ext_list_price","ss_ext_tax","ss_coupon_amt","ss_net_paid","ss_net_pai [...]
Reducer 3 vectorized
File Output Operator [FS_66]
table:{"name:":"default.store_sales"}
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 53665ee3e4d..e706ebab6c8 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -409,6 +409,14 @@ erasurecoding.only.query.files=\
erasure_simple.q
iceberg.llap.query.files=\
+ iceberg_bucket_map_join_1.q,\
+ iceberg_bucket_map_join_2.q,\
+ iceberg_bucket_map_join_3.q,\
+ iceberg_bucket_map_join_4.q,\
+ iceberg_bucket_map_join_5.q,\
+ iceberg_bucket_map_join_6.q,\
+ iceberg_bucket_map_join_7.q,\
+ iceberg_bucket_map_join_8.q,\
iceberg_clustered.q,\
iceberg_merge_delete_files.q,\
iceberg_merge_files.q,\
@@ -436,6 +444,14 @@ iceberg.llap.query.compactor.files=\
iceberg_major_compaction_unpartitioned_w_filter.q
iceberg.llap.only.query.files=\
+ iceberg_bucket_map_join_1.q,\
+ iceberg_bucket_map_join_2.q,\
+ iceberg_bucket_map_join_3.q,\
+ iceberg_bucket_map_join_4.q,\
+ iceberg_bucket_map_join_5.q,\
+ iceberg_bucket_map_join_6.q,\
+ iceberg_bucket_map_join_7.q,\
+ iceberg_bucket_map_join_8.q,\
iceberg_clustered.q,\
iceberg_merge_delete_files.q,\
iceberg_merge_files.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
index 935ae77e63e..b19e846d165 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
@@ -26,12 +26,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
-import java.util.function.BiFunction;
+import java.util.function.ToIntFunction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CompilationOpContext;
-import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.BucketCodec;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -63,8 +62,6 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
private static final long serialVersionUID = 1L;
- private transient ObjectInspector[] partitionObjectInspectors;
- private transient ObjectInspector[] bucketObjectInspectors;
private transient int buckColIdxInKey;
/**
* {@link org.apache.hadoop.hive.ql.optimizer.SortedDynPartitionOptimizer}
@@ -113,7 +110,8 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
protected transient List<List<Integer>> distinctColIndices;
protected transient Random random;
- protected transient BiFunction<Object[], ObjectInspector[], Integer> hashFunc;
+ protected transient ToIntFunction<Object[]> partitionHashFunc;
+ protected transient ToIntFunction<Object[]> bucketHashFunc;
/**
* This two dimensional array holds key data and a corresponding Union object
@@ -230,14 +228,6 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
useUniformHash = conf.getReducerTraits().contains(UNIFORM);
firstRow = true;
- // acidOp flag has to be checked to use JAVA hash which works like
- // identity function for integers, necessary to read RecordIdentifier
- // incase of ACID updates/deletes.
- boolean acidOp = conf.getWriteType() == AcidUtils.Operation.UPDATE ||
- conf.getWriteType() == AcidUtils.Operation.DELETE;
- hashFunc = getConf().getBucketingVersion() == 2 && !acidOp ?
- ObjectInspectorUtils::getBucketHashCode :
- ObjectInspectorUtils::getBucketHashCodeOld;
} catch (Exception e) {
String msg = "Error initializing ReduceSinkOperator: " + e.getMessage();
LOG.error(msg, e);
@@ -304,9 +294,11 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
conf.getOutputKeyColumnNames(), numDistributionKeys, rowInspector);
valueObjectInspector = initEvaluatorsAndReturnStruct(valueEval,
conf.getOutputValueColumnNames(), rowInspector);
- partitionObjectInspectors = initEvaluators(partitionEval, rowInspector);
+ final ObjectInspector[] partitionObjectInspectors = initEvaluators(partitionEval, rowInspector);
+ partitionHashFunc = conf.getPartitionFunction(partitionObjectInspectors);
if (bucketEval != null) {
- bucketObjectInspectors = initEvaluators(bucketEval, rowInspector);
+ final ObjectInspector[] bucketObjectInspectors = initEvaluators(bucketEval, rowInspector);
+ bucketHashFunc = conf.getPartitionFunction(bucketObjectInspectors);
}
int numKeys = numDistinctExprs > 0 ? numDistinctExprs : 1;
int keyLen = numDistinctExprs > 0 ? numDistributionKeys + 1 : numDistributionKeys;
@@ -391,8 +383,7 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
for (int i = 0; i < bucketEval.length; i++) {
bucketFieldValues[i] = bucketEval[i].evaluate(row);
}
- return ObjectInspectorUtils.getBucketNumber(
- hashFunc.apply(bucketFieldValues, bucketObjectInspectors), numBuckets);
+ return ObjectInspectorUtils.getBucketNumber(bucketHashFunc.applyAsInt(bucketFieldValues), numBuckets);
}
private void populateCachedDistributionKeys(Object row) throws HiveException {
@@ -447,7 +438,7 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
for(int i = 0; i < partitionEval.length; i++) {
bucketFieldValues[i] = partitionEval[i].evaluate(row);
}
- keyHashCode = hashFunc.apply(bucketFieldValues, partitionObjectInspectors);
+ keyHashCode = partitionHashFunc.applyAsInt(bucketFieldValues);
}
int hashCode = buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum;
if (LOG.isTraceEnabled()) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
index 1bf1ebfdf61..569ca66db82 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.tez;
+import com.google.common.primitives.UnsignedBytes;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.ByteBuffer;
@@ -25,18 +26,17 @@ import java.util.*;
import java.util.Map.Entry;
import com.google.common.collect.LinkedListMultimap;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat.HiveInputSplit;
import org.apache.hadoop.mapred.split.SplitLocationProvider;
import org.apache.tez.runtime.api.events.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.TezWork.VertexType;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.serializer.SerializationFactory;
-import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.split.TezGroupedSplit;
@@ -75,23 +75,11 @@ import com.google.protobuf.ByteString;
*/
public class CustomPartitionVertex extends VertexManagerPlugin {
- public class PathComparatorForSplit implements Comparator<InputSplit> {
+ private static class HiveInputSplitComparator implements Comparator<HiveInputSplit> {
@Override
- public int compare(InputSplit inp1, InputSplit inp2) {
- FileSplit fs1 = (FileSplit) inp1;
- FileSplit fs2 = (FileSplit) inp2;
-
- int retval = fs1.getPath().compareTo(fs2.getPath());
- if (retval != 0) {
- return retval;
- }
-
- if (fs1.getStart() != fs2.getStart()) {
- return (int) (fs1.getStart() - fs2.getStart());
- }
-
- return 0;
+ public int compare(HiveInputSplit inp1, HiveInputSplit inp2) {
+ return UnsignedBytes.lexicographicalComparator().compare(inp1.getBytesForEquality(), inp2.getBytesForEquality());
}
}
@@ -199,7 +187,7 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
}
boolean dataInformationEventSeen = false;
- Map<String, Set<FileSplit>> pathFileSplitsMap = new TreeMap<String, Set<FileSplit>>();
+ Map<Integer, Set<HiveInputSplit>> bucketFileSplitsMap = new TreeMap<>();
for (Event event : events) {
if (event instanceof InputConfigureVertexTasksEvent) {
@@ -221,28 +209,23 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
} else if (event instanceof InputDataInformationEvent) {
dataInformationEventSeen = true;
InputDataInformationEvent diEvent = (InputDataInformationEvent) event;
- FileSplit fileSplit;
+ HiveInputSplit inputSplit;
try {
- fileSplit = getFileSplitFromEvent(diEvent);
+ inputSplit = getInputSplitFromEvent(diEvent);
} catch (IOException e) {
throw new RuntimeException("Failed to get file split for event: " + diEvent, e);
}
- Set<FileSplit> fsList =
- pathFileSplitsMap.get(Utilities.getBucketFileNameFromPathSubString(fileSplit.getPath()
- .getName()));
- if (fsList == null) {
- fsList = new TreeSet<FileSplit>(new PathComparatorForSplit());
- pathFileSplitsMap.put(
- Utilities.getBucketFileNameFromPathSubString(fileSplit.getPath().getName()), fsList);
- }
- fsList.add(fileSplit);
+ final int bucketId = inputSplit.getBucketId().orElse(-1);
+ Set<HiveInputSplit> inputSplits = bucketFileSplitsMap.computeIfAbsent(bucketId,
+ k -> new TreeSet<>(new HiveInputSplitComparator()));
+ inputSplits.add(inputSplit);
}
}
- LOG.debug("Path file splits map for input name: {} is {}", inputName, pathFileSplitsMap);
+ LOG.debug("Bucket splits map for input name: {} is {}", inputName, bucketFileSplitsMap);
Multimap<Integer, InputSplit> bucketToInitialSplitMap =
- getBucketSplitMapForPath(inputName, pathFileSplitsMap);
+ getBucketSplitMapForBucket(inputName, bucketFileSplitsMap);
Preconditions.checkState(
bucketToInitialSplitMap.keySet().stream().allMatch(i -> 0 <= i && i < numBuckets));
@@ -512,8 +495,8 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
return UserPayload.create(ByteBuffer.wrap(serialized));
}
- private FileSplit getFileSplitFromEvent(InputDataInformationEvent event) throws IOException {
- InputSplit inputSplit = null;
+ private HiveInputSplit getInputSplitFromEvent(InputDataInformationEvent event) throws IOException {
+ final InputSplit inputSplit;
if (event.getDeserializedUserPayload() != null) {
inputSplit = (InputSplit) event.getDeserializedUserPayload();
} else {
@@ -522,19 +505,19 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
inputSplit = MRInputHelpers.createOldFormatSplitFromUserPayload(splitProto, serializationFactory);
}
- if (!(inputSplit instanceof FileSplit)) {
+ if (!(inputSplit instanceof HiveInputSplit)) {
throw new UnsupportedOperationException(
- "Cannot handle splits other than FileSplit for the moment. Current input split type: "
+ "Cannot handle splits other than HiveInputSplit for the moment. Current input split type: "
+ inputSplit.getClass().getSimpleName());
}
- return (FileSplit) inputSplit;
+ return (HiveInputSplit) inputSplit;
}
/*
* This method generates the map of bucket to file splits.
*/
- private Multimap<Integer, InputSplit> getBucketSplitMapForPath(String inputName,
- Map<String, Set<FileSplit>> pathFileSplitsMap) {
+ private Multimap<Integer, InputSplit> getBucketSplitMapForBucket(String inputName,
+ Map<Integer, Set<HiveInputSplit>> bucketSplitsMap) {
boolean isSMBJoin = numInputsAffectingRootInputSpecUpdate != 1;
boolean isMainWork = mainWorkName.isEmpty() || inputName.compareTo(mainWorkName) == 0;
@@ -546,14 +529,8 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
Multimap<Integer, InputSplit> bucketToSplitMap = ArrayListMultimap.create();
boolean fallback = false;
- for (Map.Entry<String, Set<FileSplit>> entry : pathFileSplitsMap.entrySet()) {
- // Extract the buckedID from pathFilesMap, this is more accurate method,
- // however. it may not work in certain cases where buckets are named
- // after files used while loading data. In such case, fallback to old
- // potential inaccurate method.
- // The accepted file names are such as 000000_0, 000001_0_copy_1.
- String bucketIdStr = Utilities.getBucketFileNameFromPathSubString(entry.getKey());
- int bucketId = Utilities.getBucketIdFromFile(bucketIdStr);
+ for (Map.Entry<Integer, Set<HiveInputSplit>> entry : bucketSplitsMap.entrySet()) {
+ int bucketId = entry.getKey();
if (bucketId < 0) {
fallback = true;
LOG.info("Fallback to using older sort based logic to assign buckets to splits.");
@@ -571,7 +548,7 @@ public class CustomPartitionVertex extends VertexManagerPlugin {
// This is the old logic which assumes that the filenames are sorted in
// alphanumeric order and mapped to appropriate bucket number.
int curSplitIndex = 0;
- for (Map.Entry<String, Set<FileSplit>> entry : pathFileSplitsMap.entrySet()) {
+ for (Map.Entry<Integer, Set<HiveInputSplit>> entry : bucketSplitsMap.entrySet()) {
int bucketId = curSplitIndex % inputBucketSize;
bucketToSplitMap.putAll(bucketId, entry.getValue());
curSplitIndex++;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java
index bf86b482b84..ae51d03f4e2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.hive.ql.exec.vector.reducesink;
import java.util.Random;
-import java.util.function.BiFunction;
+import java.util.function.ToIntFunction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.exec.vector.VectorExtractRow;
@@ -78,16 +78,15 @@ public class VectorReduceSinkObjectHashOperator extends VectorReduceSinkCommonOp
protected transient VectorSerializeRow<BinarySortableSerializeWrite> keyVectorSerializeRow;
private transient int numBuckets;
- private transient ObjectInspector[] bucketObjectInspectors;
private transient VectorExtractRow bucketVectorExtractRow;
private transient Object[] bucketFieldValues;
- private transient ObjectInspector[] partitionObjectInspectors;
private transient VectorExtractRow partitionVectorExtractRow;
private transient Object[] partitionFieldValues;
private transient Random nonPartitionRandom;
- private transient BiFunction<Object[], ObjectInspector[], Integer> hashFunc;
+ private transient ToIntFunction<Object[]> partitionHashFunc;
+ private transient ToIntFunction<Object[]> bucketHashFunc;
private transient BucketNumExpression bucketExpr = null;
/** Kryo ctor. */
@@ -168,7 +167,7 @@ public class VectorReduceSinkObjectHashOperator extends VectorReduceSinkCommonOp
} else {
numBuckets = conf.getNumBuckets();
- bucketObjectInspectors = getObjectInspectorArray(reduceSinkBucketTypeInfos);
+ bucketHashFunc = conf.getPartitionFunction(getObjectInspectorArray(reduceSinkBucketTypeInfos));
bucketVectorExtractRow = new VectorExtractRow();
bucketVectorExtractRow.init(reduceSinkBucketTypeInfos, reduceSinkBucketColumnMap);
bucketFieldValues = new Object[reduceSinkBucketTypeInfos.length];
@@ -177,17 +176,12 @@ public class VectorReduceSinkObjectHashOperator extends VectorReduceSinkCommonOp
if (isEmptyPartitions) {
nonPartitionRandom = new Random(12345);
} else {
- partitionObjectInspectors = getObjectInspectorArray(reduceSinkPartitionTypeInfos);
+ partitionHashFunc = conf.getPartitionFunction(getObjectInspectorArray(reduceSinkPartitionTypeInfos));
partitionVectorExtractRow = new VectorExtractRow();
partitionVectorExtractRow.init(reduceSinkPartitionTypeInfos, reduceSinkPartitionColumnMap);
partitionFieldValues = new Object[reduceSinkPartitionTypeInfos.length];
}
- // Set hashFunc
- hashFunc = getConf().getBucketingVersion() == 2 && !vectorDesc.getIsAcidChange() ?
- ObjectInspectorUtils::getBucketHashCode :
- ObjectInspectorUtils::getBucketHashCodeOld;
-
// Set function to evaluate _bucket_number if needed.
if (reduceSinkKeyExpressions != null) {
for (VectorExpression ve : reduceSinkKeyExpressions) {
@@ -273,14 +267,14 @@ public class VectorReduceSinkObjectHashOperator extends VectorReduceSinkCommonOp
} else {
// Compute hashCode from partitions
partitionVectorExtractRow.extractRow(batch, batchIndex, partitionFieldValues);
- hashCode = hashFunc.apply(partitionFieldValues, partitionObjectInspectors);
+ hashCode = partitionHashFunc.applyAsInt(partitionFieldValues);
}
// Compute hashCode from buckets
if (!isEmptyBuckets) {
bucketVectorExtractRow.extractRow(batch, batchIndex, bucketFieldValues);
final int bucketNum = ObjectInspectorUtils.getBucketNumber(
- hashFunc.apply(bucketFieldValues, bucketObjectInspectors), numBuckets);
+ bucketHashFunc.applyAsInt(bucketFieldValues), numBuckets);
if (bucketExpr != null) {
evaluateBucketExpr(batch, batchIndex, bucketNum);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
index ab90142162c..6e43e9a7994 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.io;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import java.util.OptionalInt;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
@@ -171,6 +172,15 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
return inputFormatClassName;
}
+ public OptionalInt getBucketId() {
+ if (inputSplit instanceof PartitionAwareSplit) {
+ return ((PartitionAwareSplit) inputSplit).getBucketId();
+ }
+
+ final int bucketId = Utilities.parseSplitBucket(inputSplit);
+ return bucketId == -1 ? OptionalInt.empty() : OptionalInt.of(bucketId);
+ }
+
@Override
public Path getPath() {
if (inputSplit instanceof FileSplit) {
@@ -260,6 +270,18 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
return allBytes;
}
}
+
+ public byte[] getBytesForEquality() {
+ if (inputSplit instanceof HashableInputSplit) {
+ return ((HashableInputSplit) inputSplit).getBytesForHash();
+ } else {
+ byte[] pathBytes = getPath().toString().getBytes();
+ byte[] allBytes = new byte[pathBytes.length + 8];
+ System.arraycopy(pathBytes, 0, allBytes, 0, pathBytes.length);
+ SerDeUtils.writeLong(allBytes, pathBytes.length, getStart());
+ return allBytes;
+ }
+ }
}
@Override
@@ -811,6 +833,7 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
pushDownProjection = true;
// push down filters and as of information
pushFiltersAndAsOf(newjob, tableScan, this.mrwork);
+ addGroupingDetails(newjob, tableScan);
}
} else {
if (LOG.isDebugEnabled()) {
@@ -1068,4 +1091,12 @@ public class HiveInputFormat<K extends WritableComparable, V extends Writable>
}
}
}
+
+ private void addGroupingDetails(JobConf conf, TableScanOperator tableScan) {
+ final List<String> groupingPartitionColumns = tableScan.getConf().getGroupingPartitionColumns();
+ if (groupingPartitionColumns != null) {
+ conf.setStrings(TableScanDesc.GROUPING_PARTITION_COLUMNS, groupingPartitionColumns.toArray(new String[0]));
+ conf.setInt(TableScanDesc.GROUPING_NUM_BUCKETS, tableScan.getConf().getGroupingNumBuckets());
+ }
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/PartitionAwareSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/PartitionAwareSplit.java
new file mode 100644
index 00000000000..b3f3eb84739
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/PartitionAwareSplit.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.io;
+
+import java.util.OptionalInt;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable;
+
+/**
+ * An InputSplit which supports Partition-Aware Table Scan.
+ */
+@Unstable
+public interface PartitionAwareSplit {
+ /**
+ * Returns the bucket number of this split. OptionalInt.empty if this is not a bucketed split.
+ */
+ @Unstable
+ OptionalInt getBucketId();
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
index f186e264687..14c96c9772c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.hive.ql.parse.StorageFormat.StorageHandlerTypes;
import org.apache.hadoop.hive.ql.parse.TransformSpec;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.UpdateSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.plan.PartitionAwareOptimizationCtx;
import org.apache.hadoop.hive.ql.plan.DynamicPartitionCtx;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
@@ -548,6 +549,29 @@ public interface HiveStorageHandler extends Configurable {
return null;
}
+ /**
+ * Checks whether the table supports Partition-Aware Optimization.
+ * @param table the table
+ * @return true if the table has partition definitions which potentially optimize query plans
+ * @throws SemanticException in case of any error.
+ */
+ default boolean supportsPartitionAwareOptimization(org.apache.hadoop.hive.ql.metadata.Table table)
+ throws SemanticException {
+ return false;
+ }
+
+ /**
+ * Creates a PartitionAwareOptimizationCtx that retains partition definitions applicable to query plan optimization.
+ * Hive can apply special optimization such as Bucket Map Join based on the storage layouts provided by this context.
+ * @param table the table
+ * @return the created PartitionAwareOptimizationCtx, null if this table doesn't support Partition-Aware Optimization
+ * @throws SemanticException in case of any error.
+ */
+ default PartitionAwareOptimizationCtx createPartitionAwareOptimizationContext(
+ org.apache.hadoop.hive.ql.metadata.Table table) throws SemanticException {
+ return null;
+ }
+
/**
* Get file format property key, if the file format is configured through a table property.
* @return table property key, can be null
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index 3e63c3d297e..37348dcef06 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.Stack;
@@ -56,6 +57,7 @@ import org.apache.hadoop.hive.ql.parse.GenTezUtils;
import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.CustomBucketFunction;
import org.apache.hadoop.hive.ql.plan.ColStatistics;
import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
import org.apache.hadoop.hive.ql.plan.DummyStoreDesc;
@@ -211,8 +213,8 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
}
// map join operator by default has no bucket cols and num of reduce sinks
// reduced by 1
- mapJoinOp.setOpTraits(new OpTraits(null, -1, null,
- joinOp.getOpTraits().getNumReduceSinks()));
+ mapJoinOp.setOpTraits(new OpTraits(null, null, -1,
+ null, joinOp.getOpTraits().getNumReduceSinks()));
preserveOperatorInfos(mapJoinOp, joinOp, context);
// propagate this change till the next RS
for (Operator<? extends OperatorDesc> childOp : mapJoinOp.getChildOperators()) {
@@ -545,8 +547,9 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
joinOp.getSchema());
context.parseContext.getContext().getPlanMapper().link(joinOp, mergeJoinOp);
int numReduceSinks = joinOp.getOpTraits().getNumReduceSinks();
- OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets,
- joinOp.getOpTraits().getSortCols(), numReduceSinks);
+ OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(),
+ joinOp.getOpTraits().getCustomBucketFunctions(), numBuckets, joinOp.getOpTraits().getSortCols(),
+ numReduceSinks);
mergeJoinOp.setOpTraits(opTraits);
mergeJoinOp.getConf().setBucketingVersion(joinOp.getConf().getBucketingVersion());
preserveOperatorInfos(mergeJoinOp, joinOp, context);
@@ -625,8 +628,9 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
if (currentOp instanceof ReduceSinkOperator) {
return;
}
- currentOp.setOpTraits(new OpTraits(opTraits.getBucketColNames(),
- opTraits.getNumBuckets(), opTraits.getSortCols(), opTraits.getNumReduceSinks()));
+ currentOp.setOpTraits(new OpTraits(opTraits.getBucketColNames(), opTraits.getCustomBucketFunctions(),
+ opTraits.getNumBuckets(), opTraits.getSortCols(),
+ opTraits.getNumReduceSinks()));
for (Operator<? extends OperatorDesc> childOp : currentOp.getChildOperators()) {
if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof GroupByOperator)) {
break;
@@ -657,6 +661,7 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
boolean updatePartitionCols = false;
List<Integer> positions = new ArrayList<>();
+ CustomBucketFunction bucketFunction = opTraits.getCustomBucketFunctions().get(0);
if (listBucketCols.get(0).size() != bigTablePartitionCols.size()) {
updatePartitionCols = true;
// Prepare updated partition columns for small table(s).
@@ -664,15 +669,29 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
int bigTableExprPos = 0;
Map<String, ExprNodeDesc> colExprMap = bigTableRS.getColumnExprMap();
+ final boolean[] retainedColumns = new boolean[listBucketCols.get(0).size()];
for (ExprNodeDesc bigTableExpr : bigTablePartitionCols) {
// It is guaranteed there is only 1 list within listBucketCols.
- for (String colName : listBucketCols.get(0)) {
+ for (int i = 0; i < listBucketCols.get(0).size(); i++) {
+ final String colName = listBucketCols.get(0).get(i);
if (colExprMap.get(colName).isSame(bigTableExpr)) {
positions.add(bigTableExprPos);
+ retainedColumns[i] = true;
}
}
bigTableExprPos = bigTableExprPos + 1;
}
+
+ Preconditions.checkState(opTraits.getCustomBucketFunctions().size() == 1);
+ if (opTraits.getCustomBucketFunctions().get(0) != null) {
+ final Optional<CustomBucketFunction> selected =
+ opTraits.getCustomBucketFunctions().get(0).select(retainedColumns);
+ if (!selected.isPresent()) {
+ LOG.info("{} can't keep itself only with {}", opTraits.getCustomBucketFunctions().get(0), retainedColumns);
+ return false;
+ }
+ bucketFunction = selected.get();
+ }
}
MapJoinOperator mapJoinOp = convertJoinMapJoin(joinOp, context, mapJoinConversion, true);
@@ -684,7 +703,7 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
joinDesc.setBucketMapJoin(true);
// we can set the traits for this join operator
- opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(),
+ opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), joinOp.getOpTraits().getCustomBucketFunctions(),
tezBucketJoinProcCtx.getNumBuckets(), null, joinOp.getOpTraits().getNumReduceSinks());
mapJoinOp.setOpTraits(opTraits);
preserveOperatorInfos(mapJoinOp, joinOp, context);
@@ -699,11 +718,7 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
if (updatePartitionCols) {
// use the positions to only pick the partitionCols which are required
// on the small table side.
- for (Operator<?> op : mapJoinOp.getParentOperators()) {
- if (!(op instanceof ReduceSinkOperator)) {
- continue;
- }
-
+ mapJoinOp.getParentOperators().stream().filter(ReduceSinkOperator.class::isInstance).forEach(op -> {
ReduceSinkOperator rsOp = (ReduceSinkOperator) op;
List<ExprNodeDesc> newPartitionCols = new ArrayList<>();
List<ExprNodeDesc> partitionCols = rsOp.getConf().getPartitionCols();
@@ -711,7 +726,21 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
newPartitionCols.add(partitionCols.get(position));
}
rsOp.getConf().setPartitionCols(newPartitionCols);
+ });
+ }
+
+ if (bucketFunction != null) {
+ final Operator<?> bigTableOp = mapJoinOp.getParentOperators().get(bigTablePosition);
+ for (TableScanOperator tso : OperatorUtils.findOperatorsUpstream(bigTableOp, TableScanOperator.class)) {
+ tso.getConf().setGroupingPartitionColumns(bucketFunction.getSourceColumnNames());
+ tso.getConf().setGroupingNumBuckets(bucketFunction.getNumBuckets());
}
+
+ final CustomBucketFunction finalBucketFunction = bucketFunction;
+ mapJoinOp.getParentOperators().stream().filter(ReduceSinkOperator.class::isInstance).forEach(op -> {
+ ReduceSinkOperator rsOp = (ReduceSinkOperator) op;
+ rsOp.getConf().setCustomPartitionFunction(finalBucketFunction);
+ });
}
// Update the memory monitor info for LLAP.
@@ -800,6 +829,11 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
return false;
}
ReduceSinkOperator rsOp = (ReduceSinkOperator) parentOp;
+ if (rsOp.getOpTraits().hasCustomBucketFunction()) {
+ LOG.info("We don't support SMB with custom bucket functions yet");
+ return false;
+ }
+
List<ExprNodeDesc> keyCols = rsOp.getConf().getKeyCols();
// For SMB, the key column(s) in RS should be same as bucket column(s) and sort column(s)`
@@ -905,8 +939,7 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
* can create a bucket map join eliminating the reduce sink.
*/
private boolean checkConvertJoinBucketMapJoin(JoinOperator joinOp,
- int bigTablePosition, TezBucketJoinProcCtx tezBucketJoinProcCtx)
- throws SemanticException {
+ int bigTablePosition, TezBucketJoinProcCtx tezBucketJoinProcCtx) {
// bail on mux-operator because mux operator masks the emit keys of the
// constituent reduce sinks
if (!(joinOp.getParentOperators().get(0) instanceof ReduceSinkOperator)) {
@@ -919,9 +952,10 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
List<List<String>> parentColNames = rs.getOpTraits().getBucketColNames();
Operator<? extends OperatorDesc> parentOfParent = rs.getParentOperators().get(0);
List<List<String>> grandParentColNames = parentOfParent.getOpTraits().getBucketColNames();
- int numBuckets = parentOfParent.getOpTraits().getNumBuckets();
- // all keys matched.
- if (!checkColEquality(grandParentColNames, parentColNames, rs.getColumnExprMap(), true)) {
+ Preconditions.checkState(rs.getOpTraits().getCustomBucketFunctions() == null
+ || rs.getOpTraits().getCustomBucketFunctions().size() == 1);
+ final boolean hasCustomBucketFunction = rs.getOpTraits().hasCustomBucketFunction();
+ if (!checkColEquality(grandParentColNames, parentColNames, rs.getColumnExprMap(), !hasCustomBucketFunction)) {
LOG.info("No info available to check for bucket map join. Cannot convert");
return false;
}
@@ -943,6 +977,9 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
* this is the case when the big table is a sub-query and is probably already bucketed by the
* join column in say a group by operation
*/
+ final CustomBucketFunction parentBucketFunction = rs.getOpTraits().getCustomBucketFunctions().get(0);
+ int numBuckets = parentBucketFunction != null ? parentBucketFunction.getNumBuckets()
+ : parentOfParent.getOpTraits().getNumBuckets();
if (numBuckets < 0) {
numBuckets = rs.getConf().getNumReducers();
}
@@ -1521,15 +1558,21 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
int estimatedBuckets = -1;
for (Operator<? extends OperatorDesc>parentOp : joinOp.getParentOperators()) {
- if (parentOp.getOpTraits().getNumBuckets() > 0) {
- numBuckets = (numBuckets < parentOp.getOpTraits().getNumBuckets()) ?
- parentOp.getOpTraits().getNumBuckets() : numBuckets;
+ if (!(parentOp instanceof ReduceSinkOperator)) {
+ continue;
+ }
+
+ final OpTraits parentOpTraits = parentOp.getOpTraits();
+ numBuckets = Math.max(numBuckets, parentOpTraits.getNumBuckets());
+
+ if (parentOpTraits.hasCustomBucketFunction()) {
+ Preconditions.checkState(parentOpTraits.getCustomBucketFunctions().size() == 1);
+ numBuckets = Math.max(numBuckets, parentOpTraits.getCustomBucketFunctions().get(0).getNumBuckets());
}
- if (!useOpTraits && parentOp instanceof ReduceSinkOperator) {
+ if (!useOpTraits) {
ReduceSinkOperator rs = (ReduceSinkOperator) parentOp;
- estimatedBuckets = (estimatedBuckets < rs.getConf().getNumReducers()) ?
- rs.getConf().getNumReducers() : estimatedBuckets;
+ estimatedBuckets = Math.max(estimatedBuckets, rs.getConf().getNumReducers());
}
}
@@ -1583,6 +1626,7 @@ public class ConvertJoinMapJoin implements SemanticNodeProcessor {
// sortCols: This is an unsorted join - no sort cols
OpTraits opTraits = new OpTraits(
joinOp.getOpTraits().getBucketColNames(),
+ joinOp.getOpTraits().getCustomBucketFunctions(),
numReducers,
null,
joinOp.getOpTraits().getNumReduceSinks());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
index 5692eb762c7..7d91d7ea216 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
@@ -18,11 +18,15 @@
package org.apache.hadoop.hive.ql.optimizer.metainfo.annotation;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import java.util.*;
+import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.Map.Entry;
+import java.util.stream.Collectors;
+import org.apache.commons.collections4.CollectionUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
@@ -94,12 +98,9 @@ public class OpTraitsRulesProcFactory {
ReduceSinkOperator rs = (ReduceSinkOperator)nd;
- List<List<String>> listBucketCols = new ArrayList<List<String>>();
- int numBuckets = -1;
int numReduceSinks = 1;
OpTraits parentOpTraits = rs.getParentOperators().get(0).getOpTraits();
if (parentOpTraits != null) {
- numBuckets = parentOpTraits.getNumBuckets();
numReduceSinks += parentOpTraits.getNumReduceSinks();
}
@@ -158,8 +159,53 @@ public class OpTraitsRulesProcFactory {
}
}
- listBucketCols.add(bucketCols);
- OpTraits opTraits = new OpTraits(listBucketCols, numBuckets,
+ final List<List<String>> listBucketCols = new ArrayList<>();
+ final List<CustomBucketFunction> bucketFunctions = new ArrayList<>();
+ int numBuckets = -1;
+ if (parentOpTraits == null || !parentOpTraits.hasCustomBucketFunction()) {
+ // No CustomBucketFunctions
+ listBucketCols.add(bucketCols);
+ bucketFunctions.add(null);
+ if (parentOpTraits != null) {
+ numBuckets = parentOpTraits.getNumBuckets();
+ }
+ } else if (parentOpTraits.getCustomBucketFunctions().size() > 1) {
+ // We don't know how to merge multiple custom bucket functions. Reset bucket attributes
+ Preconditions.checkState(parentOpTraits.getBucketColNames().size() > 1);
+ listBucketCols.add(Collections.emptyList());
+ bucketFunctions.add(null);
+ } else {
+ Preconditions.checkState(parentOpTraits.getBucketColNames().size() == 1);
+ final Map<String, String> inputToOutput = rs.getColumnExprMap().entrySet().stream()
+ .filter(entry -> entry.getValue() instanceof ExprNodeColumnDesc)
+ .filter(entry -> rs.getConf().getKeyCols().stream().anyMatch(keyDesc -> keyDesc.isSame(entry.getValue())))
+ .collect(Collectors.toMap(
+ entry -> ((ExprNodeColumnDesc) entry.getValue()).getColumn(),
+ Entry::getKey,
+ (a, b) -> a)
+ );
+ final List<String> parentBucketColNames = parentOpTraits.getBucketColNames().get(0);
+ final boolean[] retainedColumns = new boolean[parentBucketColNames.size()];
+ final List<String> rsBucketColNames = new ArrayList<>();
+ for (int i = 0; i < parentBucketColNames.size(); i++) {
+ final String rsColumnName = inputToOutput.get(parentBucketColNames.get(i));
+ if (rsColumnName != null) {
+ retainedColumns[i] = true;
+ rsBucketColNames.add(rsColumnName);
+ }
+ }
+ final Optional<CustomBucketFunction> rsBucketFunction =
+ parentOpTraits.getCustomBucketFunctions().get(0).select(retainedColumns);
+ if (rsBucketFunction.isPresent()) {
+ listBucketCols.add(rsBucketColNames);
+ bucketFunctions.add(rsBucketFunction.get());
+ } else {
+ listBucketCols.add(Collections.emptyList());
+ bucketFunctions.add(null);
+ }
+ }
+
+ OpTraits opTraits = new OpTraits(listBucketCols, bucketFunctions, numBuckets,
listBucketCols, numReduceSinks);
rs.setOpTraits(opTraits);
return null;
@@ -224,22 +270,31 @@ public class OpTraitsRulesProcFactory {
} catch (HiveException e) {
prunedPartList = null;
}
- boolean isBucketed = checkBucketedTable(table,
- opTraitsCtx.getParseContext(), prunedPartList);
- List<List<String>> bucketColsList = new ArrayList<List<String>>();
- List<List<String>> sortedColsList = new ArrayList<List<String>>();
+
+ final List<List<String>> bucketColsList = new ArrayList<>();
+ final List<List<String>> sortedColsList = new ArrayList<>();
+ final List<CustomBucketFunction> bucketFunctions = new ArrayList<>();
int numBuckets = -1;
- if (isBucketed) {
+ if (table.getStorageHandler() != null
+ && table.getStorageHandler().supportsPartitionAwareOptimization(table)
+ && HiveConf.getVar(opTraitsCtx.getConf(), HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) {
+ final PartitionAwareOptimizationCtx ctx =
+ table.getStorageHandler().createPartitionAwareOptimizationContext(table);
+ bucketColsList.add(ctx.getBucketFunction().getSourceColumnNames());
+ bucketFunctions.add(ctx.getBucketFunction());
+ } else if (checkBucketedTable(table, opTraitsCtx.getParseContext(), prunedPartList)) {
bucketColsList.add(table.getBucketCols());
numBuckets = table.getNumBuckets();
- List<String> sortCols = new ArrayList<String>();
+ List<String> sortCols = new ArrayList<>();
for (Order colSortOrder : table.getSortCols()) {
sortCols.add(colSortOrder.getCol());
}
sortedColsList.add(sortCols);
+ bucketFunctions.add(null);
}
+
// num reduce sinks hardcoded to 0 because TS has no parents
- OpTraits opTraits = new OpTraits(bucketColsList, numBuckets,
+ OpTraits opTraits = new OpTraits(bucketColsList, bucketFunctions, numBuckets,
sortedColsList, 0);
ts.setOpTraits(opTraits);
return null;
@@ -271,7 +326,8 @@ public class OpTraitsRulesProcFactory {
numReduceSinks = parentOpTraits.getNumReduceSinks();
}
listBucketCols.add(gbyKeys);
- OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols,
+ List<CustomBucketFunction> bucketFunctions = Collections.singletonList(null);
+ OpTraits opTraits = new OpTraits(listBucketCols, bucketFunctions, -1, listBucketCols,
numReduceSinks);
gbyOp.setOpTraits(opTraits);
return null;
@@ -314,7 +370,8 @@ public class OpTraitsRulesProcFactory {
}
listBucketCols.add(partitionKeys);
- OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols,
+ final List<CustomBucketFunction> bucketFunctions = Collections.singletonList(null);
+ OpTraits opTraits = new OpTraits(listBucketCols, bucketFunctions, -1, listBucketCols,
numReduceSinks);
ptfOp.setOpTraits(opTraits);
return null;
@@ -367,8 +424,8 @@ public class OpTraitsRulesProcFactory {
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
SelectOperator selOp = (SelectOperator) nd;
- List<List<String>> parentBucketColNames =
- selOp.getParentOperators().get(0).getOpTraits().getBucketColNames();
+ OpTraits parentOpTraits = selOp.getParentOperators().get(0).getOpTraits();
+ List<List<String>> parentBucketColNames = parentOpTraits.getBucketColNames();
List<List<String>> listBucketCols = null;
List<List<String>> listSortCols = null;
@@ -376,25 +433,36 @@ public class OpTraitsRulesProcFactory {
if (parentBucketColNames != null) {
listBucketCols = getConvertedColNames(parentBucketColNames, selOp, false);
}
- List<List<String>> parentSortColNames =
- selOp.getParentOperators().get(0).getOpTraits().getSortCols();
+ List<List<String>> parentSortColNames = parentOpTraits.getSortCols();
if (parentSortColNames != null) {
listSortCols = getConvertedColNames(parentSortColNames, selOp, true);
}
}
- int numBuckets = -1;
- int numReduceSinks = 0;
- OpTraits parentOpTraits = selOp.getParentOperators().get(0).getOpTraits();
- if (parentOpTraits != null) {
- // if bucket columns are empty, then numbuckets must be set to -1.
- if (listBucketCols != null &&
- !(listBucketCols.isEmpty() || listBucketCols.get(0).isEmpty())) {
- numBuckets = parentOpTraits.getNumBuckets();
+ List<CustomBucketFunction> bucketFunctions = null;
+ if (listBucketCols != null) {
+ Preconditions.checkState(parentBucketColNames.size() == listBucketCols.size());
+ bucketFunctions = new ArrayList<>();
+ for (int i = 0; i < listBucketCols.size(); i++) {
+ if (listBucketCols.get(i).isEmpty()) {
+ bucketFunctions.add(null);
+ } else {
+ Preconditions.checkState(listBucketCols.get(i).size() == parentBucketColNames.get(i).size());
+ bucketFunctions.add(parentOpTraits.getCustomBucketFunctions().get(i));
+ }
}
- numReduceSinks = parentOpTraits.getNumReduceSinks();
}
- OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols,
+
+ int numBuckets = -1;
+ if (CollectionUtils.isNotEmpty(listBucketCols)
+ && CollectionUtils.isNotEmpty(listBucketCols.get(0))
+ && bucketFunctions.get(0) == null) {
+ // if bucket columns are empty, then num buckets must be set to -1.
+ // if a custom bucket function is available, the num buckets must be set to -1.
+ numBuckets = parentOpTraits.getNumBuckets();
+ }
+ final int numReduceSinks = parentOpTraits.getNumReduceSinks();
+ OpTraits opTraits = new OpTraits(listBucketCols, bucketFunctions, numBuckets, listSortCols,
numReduceSinks);
selOp.setOpTraits(opTraits);
return null;
@@ -408,6 +476,7 @@ public class OpTraitsRulesProcFactory {
Object... nodeOutputs) throws SemanticException {
JoinOperator joinOp = (JoinOperator) nd;
List<List<String>> bucketColsList = new ArrayList<List<String>>();
+ List<CustomBucketFunction> bucketFunctions = new ArrayList<>();
List<List<String>> sortColsList = new ArrayList<List<String>>();
byte pos = 0;
int numReduceSinks = 0; // will be set to the larger of the parents
@@ -422,8 +491,27 @@ public class OpTraitsRulesProcFactory {
rsRule.process(rsOp, stack, procCtx, nodeOutputs);
}
OpTraits parentOpTraits = rsOp.getOpTraits();
- bucketColsList.add(getOutputColNames(joinOp, parentOpTraits.getBucketColNames(), pos));
- sortColsList.add(getOutputColNames(joinOp, parentOpTraits.getSortCols(), pos));
+ final Entry<List<String>, boolean[]> outputColNames =
+ getOutputColNames(joinOp, parentOpTraits.getBucketColNames(), pos);
+ if (outputColNames == null || outputColNames.getKey().isEmpty() || !parentOpTraits.hasCustomBucketFunction()) {
+ bucketColsList.add(outputColNames == null ? null : outputColNames.getKey());
+ bucketFunctions.add(null);
+ } else {
+ Preconditions.checkState(parentOpTraits.getBucketColNames().size() == 1);
+ final Optional<CustomBucketFunction> joinBucketFunction =
+ parentOpTraits.getCustomBucketFunctions().get(0).select(outputColNames.getValue());
+ if (joinBucketFunction.isPresent()) {
+ bucketColsList.add(outputColNames.getKey());
+ bucketFunctions.add(joinBucketFunction.get());
+ } else {
+ bucketColsList.add(Collections.emptyList());
+ bucketFunctions.add(null);
+ }
+ }
+
+ final Entry<List<String>, boolean[]> outputSortColNames =
+ getOutputColNames(joinOp, parentOpTraits.getSortCols(), pos);
+ sortColsList.add(outputSortColNames == null ? null : outputSortColNames.getKey());
if (parentOpTraits.getNumReduceSinks() > numReduceSinks) {
numReduceSinks = parentOpTraits.getNumReduceSinks();
}
@@ -433,11 +521,11 @@ public class OpTraitsRulesProcFactory {
// The bucketingVersion is not relevant here as it is never used.
// For SMB, we look at the parent tables' bucketing versions and for
// bucket map join the big table's bucketing version is considered.
- joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList, numReduceSinks));
+ joinOp.setOpTraits(new OpTraits(bucketColsList, bucketFunctions, -1, bucketColsList, numReduceSinks));
return null;
}
- private List<String> getOutputColNames(JoinOperator joinOp, List<List<String>> parentColNames,
+ private Entry<List<String>, boolean[]> getOutputColNames(JoinOperator joinOp, List<List<String>> parentColNames,
byte pos) {
if (parentColNames == null) {
// no col names in parent
@@ -449,13 +537,16 @@ public class OpTraitsRulesProcFactory {
// a reduce sink always brings down the bucketing cols to a single list.
// may not be true with correlation operators (mux-demux)
List<String> colNames = parentColNames.size() > 0 ? parentColNames.get(0) : new ArrayList<>();
- for (String colName : colNames) {
+ final boolean[] retainedColumns = new boolean[colNames.size()];
+ for (int i = 0; i < colNames.size(); i++) {
+ final String colName = colNames.get(i);
for (ExprNodeDesc exprNode : joinOp.getConf().getExprs().get(pos)) {
if (exprNode instanceof ExprNodeColumnDesc) {
if (((ExprNodeColumnDesc) (exprNode)).getColumn().equals(colName)) {
for (Entry<String, ExprNodeDesc> entry : joinOp.getColumnExprMap().entrySet()) {
if (entry.getValue().isSame(exprNode)) {
bucketColNames.add(entry.getKey());
+ retainedColumns[i] = true;
// we have found the colName
break;
}
@@ -470,7 +561,7 @@ public class OpTraitsRulesProcFactory {
}
}
- return bucketColNames;
+ return new SimpleImmutableEntry<>(bucketColNames, retainedColumns);
}
}
@@ -495,7 +586,7 @@ public class OpTraitsRulesProcFactory {
numReduceSinks = parentOp.getOpTraits().getNumReduceSinks();
}
}
- OpTraits opTraits = new OpTraits(null, -1,
+ OpTraits opTraits = new OpTraits(null, null, -1,
null, numReduceSinks);
operator.setOpTraits(opTraits);
return null;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CustomBucketFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CustomBucketFunction.java
new file mode 100644
index 00000000000..9bf85a8cc17
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CustomBucketFunction.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.plan;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Optional;
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+/**
+ * A bucketing logic based on flexible transformations, e.g. partition transforms of Apache Iceberg.
+ */
+@Unstable
+public interface CustomBucketFunction extends Serializable {
+ /**
+ * Creates a new CustomBucketFunction which accepts only the given columns.
+ *
+ * @param retainedColumns the flags of retained columns
+ * @return a new CustomBucketFunction accepting the retained columns
+ * Optional.empty() if the subset can't generate a bucket id
+ */
+ @Unstable
+ Optional<CustomBucketFunction> select(boolean[] retainedColumns);
+
+ /**
+ * Initializes this bucket function with the given ObjectInspectors.
+ *
+ * @param bucketFieldInspectors the ObjectInspectors which decode the bucketFields
+ */
+ @Unstable
+ void initialize(ObjectInspector[] bucketFieldInspectors);
+
+ /**
+ * Returns the column names in the source table.
+ */
+ @Unstable
+ List<String> getSourceColumnNames();
+
+ /**
+ * Returns the number of buckets.
+ */
+ @Unstable
+ int getNumBuckets();
+
+ /**
+ * Produces a hash code from the given arguments.
+ *
+ * @param bucketFields the source fields
+ * @return the bucket hash code
+ */
+ @Unstable
+ int getBucketHashCode(Object[] bucketFields);
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
index 246c0896b02..fdb623cc1dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
@@ -18,18 +18,28 @@
package org.apache.hadoop.hive.ql.plan;
+import com.google.common.base.Preconditions;
import java.util.List;
+import java.util.Objects;
public class OpTraits {
- private List<List<String>> bucketColNames;
+ private final List<List<String>> bucketColNames;
+ private final List<CustomBucketFunction> customBucketFunctions;
private List<List<String>> sortColNames;
private int numBuckets;
private int numReduceSinks;
- public OpTraits(List<List<String>> bucketColNames, int numBuckets,
- List<List<String>> sortColNames, int numReduceSinks) {
+ public OpTraits(List<List<String>> bucketColNames, List<CustomBucketFunction> customBucketFunctions, int numBuckets,
+ List<List<String>> sortColNames,
+ int numReduceSinks) {
+ if (bucketColNames == null) {
+ Preconditions.checkArgument(customBucketFunctions == null);
+ } else {
+ Preconditions.checkArgument(bucketColNames.size() == customBucketFunctions.size());
+ }
this.bucketColNames = bucketColNames;
+ this.customBucketFunctions = customBucketFunctions;
this.numBuckets = numBuckets;
this.sortColNames = sortColNames;
this.numReduceSinks = numReduceSinks;
@@ -43,10 +53,6 @@ public class OpTraits {
return numBuckets;
}
- public void setBucketColNames(List<List<String>> bucketColNames) {
- this.bucketColNames = bucketColNames;
- }
-
public void setNumBuckets(int numBuckets) {
this.numBuckets = numBuckets;
}
@@ -59,6 +65,13 @@ public class OpTraits {
return sortColNames;
}
+ public List<CustomBucketFunction> getCustomBucketFunctions() {
+ return customBucketFunctions;
+ }
+
+ public boolean hasCustomBucketFunction() {
+ return customBucketFunctions != null && customBucketFunctions.stream().anyMatch(Objects::nonNull);
+ }
public void setNumReduceSinks(int numReduceSinks) {
this.numReduceSinks = numReduceSinks;
@@ -70,7 +83,8 @@ public class OpTraits {
@Override
public String toString() {
- return "{ bucket column names: " + bucketColNames + "; sort column names: "
- + sortColNames + "; bucket count: " + numBuckets + "}";
+ return "{ bucket column names: " + bucketColNames + "; custom partition functions: " + customBucketFunctions
+ + "; sort column names: " + sortColNames + "; bucket count: " + numBuckets + "; num reduce sinks: "
+ + numReduceSinks + "}";
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionAwareOptimizationCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionAwareOptimizationCtx.java
new file mode 100644
index 00000000000..f9827aed085
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionAwareOptimizationCtx.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.plan;
+
+import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable;
+
+/**
+ * A context to support Partition-Aware Optimization leveraging storage layouts.
+ */
+@Unstable
+public class PartitionAwareOptimizationCtx {
+ private final CustomBucketFunction bucketFunction;
+
+ /**
+ * Constructs a new PartitionAwareOptimizationCtx.
+ *
+ * @param bucketFunction the bucket function
+ */
+ @Unstable
+ public PartitionAwareOptimizationCtx(CustomBucketFunction bucketFunction) {
+ this.bucketFunction = bucketFunction;
+ }
+
+ /**
+ * Returns the bucket function for Partition-Aware Optimization.
+ */
+ @Unstable
+ public CustomBucketFunction getBucketFunction() {
+ return bucketFunction;
+ }
+}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index b120ae951fc..1a643ddd93a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Objects;
import java.util.Set;
+import java.util.function.ToIntFunction;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -36,6 +37,10 @@ import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc.ReduceSinkKeyType;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
@@ -45,6 +50,8 @@ import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc.ReduceSinkKeyType;
@Explain(displayName = "Reduce Output Operator", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
public class ReduceSinkDesc extends AbstractOperatorDesc {
private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(ReduceSinkDesc.class);
+
/**
* Key columns are passed to reducer in the "key".
*/
@@ -87,6 +94,12 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
* Partition columns are not passed to reducer.
*/
private List<ExprNodeDesc> partitionCols;
+ /**
+ * The hash function that generates custom hash codes for partitioning. This is configured when Partition-Aware
+ * Optimization requires a special hash function, e.g. the bucket function of Apache Iceberg. Otherwise, this is NULL
+ * and this operator uses the standard hash function of Hive.
+ */
+ private CustomBucketFunction customBucketFunction;
private int numReducers;
@@ -293,6 +306,28 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return false;
}
+ public void setCustomPartitionFunction(CustomBucketFunction customBucketFunction) {
+ this.customBucketFunction = customBucketFunction;
+ }
+
+ public ToIntFunction<Object[]> getPartitionFunction(ObjectInspector[] objectInspectors) {
+ // acidOp flag has to be checked to use JAVA hash which works like
+ // identity function for integers, necessary to read RecordIdentifier
+ // incase of ACID updates/deletes.
+ final boolean acidOp = writeType == AcidUtils.Operation.UPDATE || writeType == AcidUtils.Operation.DELETE;
+ if (customBucketFunction != null) {
+ LOG.info("Use {} for partitioning", customBucketFunction);
+ customBucketFunction.initialize(objectInspectors);
+ return customBucketFunction::getBucketHashCode;
+ } else if (getBucketingVersion() == 2 && !acidOp) {
+ LOG.info("Use v2 hash for partitioning");
+ return fields -> ObjectInspectorUtils.getBucketHashCode(fields, objectInspectors);
+ } else {
+ LOG.info("Use v1 hash for partitioning");
+ return fields -> ObjectInspectorUtils.getBucketHashCodeOld(fields, objectInspectors);
+ }
+ }
+
@Signature
@Explain(displayName = "tag", explainLevels = { Level.EXTENDED })
public int getTag() {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index d6e8935a3e9..60b14fa90f7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -61,6 +61,9 @@ public class TableScanDesc extends AbstractOperatorDesc implements IStatsGatherD
*/
private List<String> partColumns;
+ private List<String> groupingPartitionColumns;
+ private Integer groupingNumBuckets;
+
/**
* Used for split sampling (row count per split)
* For example,
@@ -106,6 +109,12 @@ public class TableScanDesc extends AbstractOperatorDesc implements IStatsGatherD
public static final String PARTITION_PRUNING_FILTER =
"hive.io.pruning.filter";
+ public static final String GROUPING_PARTITION_COLUMNS =
+ "hive.io.grouping.partition.columns";
+
+ public static final String GROUPING_NUM_BUCKETS =
+ "hive.io.grouping.num.buckets";
+
public static final String AS_OF_TIMESTAMP =
"hive.io.as.of.timestamp";
@@ -350,6 +359,26 @@ public class TableScanDesc extends AbstractOperatorDesc implements IStatsGatherD
return partColumns;
}
+ public void setGroupingPartitionColumns(List<String> groupingPartitionColumns) {
+ this.groupingPartitionColumns = groupingPartitionColumns;
+ }
+
+ @Explain(displayName = "Grouping Partition Columns", explainLevels = { Level.USER, Level.EXTENDED })
+ @Signature
+ public List<String> getGroupingPartitionColumns() {
+ return groupingPartitionColumns;
+ }
+
+ public void setGroupingNumBuckets(int groupingNumBuckets) {
+ this.groupingNumBuckets = groupingNumBuckets;
+ }
+
+ @Explain(displayName = "Grouping Num Buckets", explainLevels = { Level.USER, Level.EXTENDED })
+ @Signature
+ public Integer getGroupingNumBuckets() {
+ return groupingNumBuckets;
+ }
+
public void setGatherStats(boolean gatherStats) {
this.gatherStats = gatherStats;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
index 323dcba4239..ba02b3ed81f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
@@ -100,6 +100,7 @@ public abstract class GenericUDF implements Closeable {
@Override
public void prepare(int version) throws HiveException {
+ // This implementation doesn't need to be prepared
}
@Override
@@ -108,6 +109,27 @@ public abstract class GenericUDF implements Closeable {
}
}
+ /**
+ * A basic implementation of DeferredObject which allows to set or get a Java Object reference.
+ */
+ public static class SettableDeferredJavaObject implements DeferredObject {
+ private Object value;
+
+ @Override
+ public void prepare(int version) throws HiveException {
+ // This implementation doesn't need to be prepared
+ }
+
+ @Override
+ public Object get() throws HiveException {
+ return value;
+ }
+
+ public void set(Object value) {
+ this.value = value;
+ }
+ }
+
/**
* The constructor.
*/