You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2015/09/01 01:49:35 UTC
[1/4] incubator-trafodion git commit: More insert costing changes
Repository: incubator-trafodion
Updated Branches:
refs/heads/master c5c516e05 -> 7610b1ea1
More insert costing changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/4f654ece
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/4f654ece
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/4f654ece
Branch: refs/heads/master
Commit: 4f654ecef4f3b3fcda2a844b3a1581bb0b97ce4f
Parents: 68b84db
Author: Dave Birdsall <db...@apache.org>
Authored: Wed Aug 26 22:01:31 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Wed Aug 26 22:02:13 2015 +0000
----------------------------------------------------------------------
core/sql/optimizer/OptPhysRelExpr.cpp | 3 +--
core/sql/optimizer/opt.cpp | 10 ++++++++--
2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4f654ece/core/sql/optimizer/OptPhysRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp
index 2786749..0674c33 100644
--- a/core/sql/optimizer/OptPhysRelExpr.cpp
+++ b/core/sql/optimizer/OptPhysRelExpr.cpp
@@ -16660,8 +16660,7 @@ HbaseInsert::costMethod() const
static THREAD_P CostMethodHbaseInsert *m = NULL;
if (m == NULL)
m = new (GetCliGlobals()->exCollHeap()) CostMethodHbaseInsert();
- // return m; TODO: uncomment this line
- return RelExpr::costMethod(); // TODO: remove this testing line
+ return m;
} // HbaseInsert::costMethod()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4f654ece/core/sql/optimizer/opt.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/opt.cpp b/core/sql/optimizer/opt.cpp
index d1fce2b..a417d63 100644
--- a/core/sql/optimizer/opt.cpp
+++ b/core/sql/optimizer/opt.cpp
@@ -998,12 +998,18 @@ NAString Context::getRequirementsString() const
{
NAString IPPString("", CmpCommon::statementHeap());
- char thisptr[7 + sizeof(void *)];
+ // will receive the string "ptr=%p, ", where %p is a hex address
+ // example: "ptr=0x7fffffff23db, "
+ // so we need 4 bytes for "ptr=", 2 bytes for the "0x", up to
+ // 2 * sizeof(void *) bytes for the hex (each nibble goes to
+ // one ASCII character), 2 bytes for the ", " and 1 byte for
+ // the trailing null
+ char thisptr[4 + 2 + 2*sizeof(void *) + 2 + 1];
if ( CmpCommon::getDefault( NSK_DBG_PRINT_CONTEXT_POINTER ) == DF_ON )
{
sprintf(thisptr, "ptr=%p, ",this);
- thisptr[6 + sizeof(void *)]='\0';
+ thisptr[sizeof(thisptr)-1]='\0'; // shouldn't be needed, but be safe
}
else
thisptr[0]='\0';
[4/4] incubator-trafodion git commit: Merge [TRAFODION-25] Part 1
Insert costing changes
Posted by db...@apache.org.
Merge [TRAFODION-25] Part 1 Insert costing changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/7610b1ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/7610b1ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/7610b1ea
Branch: refs/heads/master
Commit: 7610b1ea187766ac06536e4499cd48ab6239afd7
Parents: c5c516e 4c4a2c7
Author: Dave Birdsall <db...@apache.org>
Authored: Mon Aug 31 23:48:24 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Mon Aug 31 23:48:24 2015 +0000
----------------------------------------------------------------------
core/sql/optimizer/CostMethod.h | 64 ++--------
core/sql/optimizer/OptPhysRelExpr.cpp | 11 ++
core/sql/optimizer/RelUpdate.h | 3 +
core/sql/optimizer/ScmCostMethod.cpp | 72 ++++++++++++
core/sql/optimizer/costmethod.cpp | 181 ++++++++++++++++++++++++++++-
core/sql/optimizer/opt.cpp | 10 +-
core/sql/regress/compGeneral/TEST071 | 20 ++++
7 files changed, 299 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
[2/4] incubator-trafodion git commit: First draft insert costing
Posted by db...@apache.org.
First draft insert costing
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/68b84dbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/68b84dbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/68b84dbd
Branch: refs/heads/master
Commit: 68b84dbdc55db5fed1e1c2014f8ba9d2409109b0
Parents: 5928f31
Author: Dave Birdsall <db...@apache.org>
Authored: Tue Aug 11 20:46:27 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Wed Aug 26 22:02:13 2015 +0000
----------------------------------------------------------------------
core/sql/optimizer/CostMethod.h | 64 +------
core/sql/optimizer/OptPhysRelExpr.cpp | 12 ++
core/sql/optimizer/RelUpdate.h | 3 +
core/sql/optimizer/ScmCostMethod.cpp | 72 ++++++++
core/sql/optimizer/costmethod.cpp | 264 ++++++++++++++++++++++++++++-
5 files changed, 355 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/CostMethod.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/CostMethod.h b/core/sql/optimizer/CostMethod.h
index 43c54ed..22f3d13 100644
--- a/core/sql/optimizer/CostMethod.h
+++ b/core/sql/optimizer/CostMethod.h
@@ -65,9 +65,8 @@ class CostMethodNestedJoinFlow;
class CostMethodMergeUnion;
class CostMethodTuple;
class CostMethodTranspose;
-class CostMethodDP2Insert;
-class CostMethodDP2SideTreeInsert;
-class CostMethodDP2VSBBInsert;
+class CostMethodStoredProc;
+class CostMethodHbaseInsert;
class CostMethodDP2Update;
class CostMethodDP2Delete;
class CostMethodUnPackRows;
@@ -1996,14 +1995,14 @@ protected:
//<pb>
/**********************************************************************/
/* */
-/* CostMethodDP2Insert */
+/* CostMethodHbaseInsert */
/* */
/**********************************************************************/
-class CostMethodDP2Insert : public CostMethod
+class CostMethodHbaseInsert : public CostMethod
{
public:
// Constructor
- CostMethodDP2Insert() : CostMethod( "CostMethodDP2Insert" ) {}
+ CostMethodHbaseInsert() : CostMethod( "CostMethodHbaseInsert" ) {}
protected:
// Cost functions
@@ -2012,26 +2011,17 @@ protected:
Lng32& countOfStreams);
virtual Cost* scmComputeOperatorCostInternal(RelExpr* op,
- const Context* myContext,
const PlanWorkSpace* pws,
Lng32& countOfStreams);
virtual void cacheParameters(RelExpr*, const Context *);
- CostScalar getNumberOfInserts();
- virtual void calculateIOCosts();
- virtual CostScalar calculateNumberOfInserts();
virtual void cleanUp();
- virtual void calculateCpuAndIdleCosts();
protected:
- IndexDesc* CIDesc_;
// storage related
- CostScalar indexLevels_;
- CostScalar kbPerBlock_;
- CostScalar rowsPerBlock_;
- CostScalar totalIndexBlocks_;
- CostScalar entriesInIndexBlock_;
+ // TODO: add members to help compute whether inserts will
+ // fit in memstore or not if this seems interesting
// parallelism related
CostScalar activePartitions_;
@@ -2039,47 +2029,9 @@ protected:
CostScalar streamsPerCpu_;
CostScalar countOfAsynchronousStreams_;
- NABoolean probesInOrder_;
-
- // Used for MV logging pushdown
- NABoolean usedForMvLogging_;
- const PushDownProperty* pushDownProperty_;
-
- // I/O and CPU related cost scalars
- CostScalar transferFR_;
- CostScalar transferLR_;
- CostScalar seeksFR_;
- CostScalar seeksLR_;
- CostScalar numInserts_;
- CostScalar cpuFR_;
- CostScalar cpuLR_;
- CostScalar idleFR_;
- CostScalar idleLR_;
-
-}; // class CostMethodDP2Insert
-
-class CostMethodDP2SideTreeInsert : public CostMethodDP2Insert
-{
-public:
- // constructor
- CostMethodDP2SideTreeInsert() : CostMethodDP2Insert () {}
-
-protected:
- virtual void calculateIOCosts();
- virtual CostScalar calculateNumberOfInserts();
-};
-
-class CostMethodDP2VSBBInsert : public CostMethodDP2Insert
-{
-public:
- // constructor
- CostMethodDP2VSBBInsert() : CostMethodDP2Insert () {}
+}; // class CostMethodHbaseInsert
-protected:
- virtual void calculateIOCosts();
- virtual CostScalar calculateNumberOfInserts();
-};
//<pb>
/**********************************************************************/
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/OptPhysRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp
index 22b394b..2786749 100644
--- a/core/sql/optimizer/OptPhysRelExpr.cpp
+++ b/core/sql/optimizer/OptPhysRelExpr.cpp
@@ -16653,6 +16653,18 @@ HiveInsert::synthPhysicalProperty(const Context* myContext,
} // HiveInsert::synthPhysicalProperty()
+
+CostMethod *
+HbaseInsert::costMethod() const
+{
+ static THREAD_P CostMethodHbaseInsert *m = NULL;
+ if (m == NULL)
+ m = new (GetCliGlobals()->exCollHeap()) CostMethodHbaseInsert();
+ // return m; TODO: uncomment this line
+ return RelExpr::costMethod(); // TODO: remove this testing line
+} // HbaseInsert::costMethod()
+
+
PhysicalProperty*
HbaseInsert::synthPhysicalProperty(const Context* myContext,
const Lng32 planNumber,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/RelUpdate.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelUpdate.h b/core/sql/optimizer/RelUpdate.h
index d4b2c7f..008f6d9 100644
--- a/core/sql/optimizer/RelUpdate.h
+++ b/core/sql/optimizer/RelUpdate.h
@@ -1847,6 +1847,9 @@ public:
const Lng32 planNumber,
PlanWorkSpace *pws);
+ // optimizer functions
+ virtual CostMethod* costMethod() const;
+
// get a printable string that identifies the operator
const NAString getText() const;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/ScmCostMethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScmCostMethod.cpp b/core/sql/optimizer/ScmCostMethod.cpp
index 3d5d98b..85e1aa9 100644
--- a/core/sql/optimizer/ScmCostMethod.cpp
+++ b/core/sql/optimizer/ScmCostMethod.cpp
@@ -3629,6 +3629,78 @@ CostMethodMergeUnion::scmComputeOperatorCostInternal(RelExpr *op,
return muCost;
} // CostMethodMergeUnion::scmComputeOperatorInternal()
+
+//<pb>
+
+//**************************************************************
+// This method computes the cost vector of the HbaseInsert operation
+//**************************************************************
+Cost*
+CostMethodHbaseInsert::scmComputeOperatorCostInternal(RelExpr* op,
+ const PlanWorkSpace* pws,
+ Lng32& countOfStreams)
+{
+ // TODO: For now assume we are always doing an HBase insert.
+ // Is it possible we go through this code path for Hive inserts?
+ // If so, figure out what to do.
+
+ HbaseInsert* insOp = (HbaseInsert *)op; // downcast
+
+ // compute some details
+ const Context * myContext = pws->getContext();
+ Cost *costPtr = computeOperatorCostInternal(op, myContext, countOfStreams);
+
+ CostScalar noOfProbesPerStream(csOne);
+
+ // the number of rows to insert (this is "per-stream" costing).
+ CostScalar numOfProbesPerStream =
+ (noOfProbes_ / countOfAsynchronousStreams_).minCsOne();
+
+ CostScalar tuplesProcessed = numOfProbesPerStream;
+ CostScalar tuplesProduced = numOfProbesPerStream;
+
+ CostScalar ioRand = csZero; // we don't bother estimating this
+ CostScalar ioSeq = csZero; // we don't bother estimating this
+
+ // Factor in row sizes.
+ CostScalar rowSize = ((IndexDesc *)insOp->getIndexDesc())->getNAFileSet()->getRecordLength();
+ CostScalar rowSizeFactor = scmRowSizeFactor(rowSize);
+ tuplesProcessed *= rowSizeFactor;
+ tuplesProduced *= rowSizeFactor;
+
+ // there doesn't seem to be an estRowsAccessed_ member or
+ // related methods in hbaseInsert at the moment... add it
+ // when the need becomes apparent
+ //insOp->setEstRowsAccessed(noOfProbes_);
+
+ //----------------------------------------
+ // Synthesize and return the cost object.
+ //----------------------------------------
+ Cost *hbaseInsertCost =
+ scmCost(tuplesProcessed, tuplesProduced, csZero, ioRand, ioSeq, noOfProbesPerStream_,
+ rowSize, csZero, rowSize, csZero);
+
+#ifndef NDEBUG
+ NABoolean printCost =
+ (CmpCommon::getDefault(OPTIMIZER_PRINT_COST) == DF_ON);
+ if (printCost)
+ {
+ pfp = stdout;
+ fprintf(pfp, "DP2INSERT::scmComputeOperatorCostInternal()\n");
+ hbaseInsertCost->getScmCplr().print(pfp);
+ fprintf(pfp, "DP2Insert elapsed time: ");
+ fprintf(pfp, "%f", hbaseInsertCost->convertToElapsedTime(myContext->getReqdPhysicalProperty()).value());
+ fprintf(pfp, "\n");
+ }
+#endif
+
+ // We use the call to computeOperatorCostInternal() to compute the various costs, but we
+ // we do not need the cost object computed by this method, we generate and return
+ // a different New Cost Model (NCM) cost object.
+ delete costPtr;
+
+ return hbaseInsertCost;
+}
//<pb>
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/costmethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/costmethod.cpp b/core/sql/optimizer/costmethod.cpp
index 94cc37b..57bd4fd 100644
--- a/core/sql/optimizer/costmethod.cpp
+++ b/core/sql/optimizer/costmethod.cpp
@@ -19,7 +19,7 @@
// under the License.
//
// @@@ END COPYRIGHT @@@
-**********************************************************************/
+/**********************************************************************/
/* -*-C++-*-
**************************************************************************
*
@@ -13997,15 +13997,271 @@ CostMethodFastExtract::computeOperatorCostInternal(RelExpr* op,
/**********************************************************************/
//<pb>
-// ----QUICKSEARCH FOR DP2Insert........................................
+// ----QUICKSEARCH FOR HbaseInsert........................................
/**********************************************************************/
/* */
-/* CostMethodDP2Insert */
+/* CostMethodHbaseInsert */
/* */
/**********************************************************************/
-#pragma nowarn(262) // warning elimination
+// -----------------------------------------------------------------------
+// CostMethodHbaseInsert::cacheParameters()
+// -----------------------------------------------------------------------
+void CostMethodHbaseInsert::cacheParameters(RelExpr* op, const Context * myContext)
+{
+ CostMethod::cacheParameters(op, myContext);
+
+ HbaseInsert* insOp = (HbaseInsert *)op;
+
+ CMPASSERT(partFunc_ != NULL);
+ NodeMap * nodeMap = (NodeMap *)partFunc_->getNodeMap();
+ if (nodeMap)
+ activePartitions_ = (CostScalar)nodeMap->getNumActivePartitions();
+ else
+ // Occasionally (e.g., regress/fullstack2/test023, the insert/select
+ // from t023t1 into t023t2 using a transpose operator), we get
+ // a ReplicateNoBroadcastPartitioningFunction lacking a node map.
+ // In this case we'll just use the number of partitions from the
+ // partitioning function itself -- which is probably an ESP count.
+ activePartitions_ = (CostScalar)partFunc_->getCountOfPartitions();
+
+ // The number of asynchronous streams is USUALLY the # of active parts.
+ countOfAsynchronousStreams_ = activePartitions_;
+} // CostMethodHbaseInsert::cacheParameters()
+
+
+
+// -----------------------------------------------------------------------
+// CostMethodHbaseInsert::computeOperatorCostInternal()
+// -----------------------------------------------------------------------
+Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
+ const Context* myContext,
+ Lng32& countOfStreams)
+{
+ cacheParameters(op, myContext);
+ estimateDegreeOfParallelism();
+
+ const InputPhysicalProperty* ippForMe =
+ myContext->getInputPhysicalProperty();
+
+ // ------------------------------------------------------
+ // Save off our current estimated degree of parallelism.
+ // in the 'out' parameter; we might revise it below
+ // ------------------------------------------------------
+ countOfStreams = countOfStreams_;
+
+ HbaseInsert* insOp = (HbaseInsert *)op; // downcast
+
+ if (ippForMe != NULL) // input physical properties exist?
+ {
+ // ----------------------------------------------------------
+ // This long block of code is devoted to figuring out
+ // if there are constraints to parallelism. For example,
+ // if a layer of ESPs is doing the inserts, and the
+ // degree of parallelism (DoP) at that layer is less
+ // than the number of partitions in the table, and the
+ // inserts arrive in partition order, then the regions
+ // accessed by a given ESP are accessed serially. In
+ // that case, we want to reduce the returned "countOfStreams"
+ // to be the ESP layer DoP rather than the number of
+ // partitions.
+ // ----------------------------------------------------------
+
+ // See if the probes are in order.
+
+ // For insert, a partial order is of no help and so we can't use it.
+ NABoolean partiallyInOrderOK = FALSE;
+ NABoolean probesForceSynchronousAccess = FALSE;
+ IndexDesc * CIDesc = (IndexDesc *)insOp->getIndexDesc();
+ ValueIdList targetSortKey = CIDesc->getOrderOfKeyValues();
+ ValueIdSet sourceCharInputs =
+ insOp->getGroupAttr()->getCharacteristicInputs();
+
+ ValueIdSet targetCharInputs;
+ // The char inputs are still in terms of the source. Map them to the target.
+ // Note: The source char outputs in the ipp have already been mapped to
+ // the target. CharOutputs are a set, meaning they do not have duplicates
+ // But we could have cases where two columns of the target are matched to the
+ // same source column, for example, if we have
+ // INSERT INTO b6table1
+ // ( SELECT f, h_to_f, f, 8.4
+ // FROM btre211
+ // );
+ // Hence we use lists here instead of sets.
+ // Check to see if there are any duplicates in the source Characteristics inputs
+ // if no, we shall perform set operations, as these are faster
+
+ ValueIdList bottomValues = insOp->updateToSelectMap().getBottomValues();
+ ValueIdSet bottomValuesSet(bottomValues);
+ NABoolean useListInsteadOfSet = FALSE;
+
+ CascadesGroup* group1 = (*CURRSTMT_OPTGLOBALS->memo)[insOp->getGroupId()];
+
+ GenericUpdate* upOperator = (GenericUpdate *)group1->getFirstLogExpr();
+
+ if (((upOperator->getTableName().getSpecialType() == ExtendedQualName::NORMAL_TABLE) || (upOperator->getTableName().getSpecialType() == ExtendedQualName::GHOST_TABLE)) &&
+ (bottomValuesSet.entries() != bottomValues.entries()))
+ {
+
+ ValueIdList targetInputList;
+ // from here get all the bottom values that appear in the sourceCharInputs
+ bottomValues.findCommonElements(sourceCharInputs);
+ bottomValuesSet = bottomValues;
+
+ // we can use the bottomValues only if these contain some duplicate columns of
+ // characteristics inputs, otherwise we shall use the characteristics inputs.
+ if (bottomValuesSet == sourceCharInputs)
+ {
+ useListInsteadOfSet = TRUE;
+ insOp->updateToSelectMap().rewriteValueIdListUpWithIndex(
+ targetInputList,
+ bottomValues);
+ targetCharInputs = targetInputList;
+ }
+ }
+
+ if (!useListInsteadOfSet)
+ {
+ insOp->updateToSelectMap().rewriteValueIdSetUp(
+ targetCharInputs,
+ sourceCharInputs);
+ }
+
+ // If a target key column is covered by a constant on the source side,
+ // then we need to remove that column from the target sort key
+ removeConstantsFromTargetSortKey(&targetSortKey,
+ &(insOp->updateToSelectMap()));
+ NABoolean orderedNJ = TRUE;
+ // Don't call ordersMatch if njOuterOrder_ is null.
+ if (ippForMe->getAssumeSortedForCosting())
+ orderedNJ = FALSE;
+ else
+ // if leading keys are not same then don't try ordered NJ.
+ orderedNJ =
+ isOrderedNJFeasible(*(ippForMe->getNjOuterOrder()), targetSortKey);
+
+ if (orderedNJ AND
+ ordersMatch(ippForMe,
+ CIDesc,
+ &targetSortKey,
+ targetCharInputs,
+ partiallyInOrderOK,
+ probesForceSynchronousAccess))
+ {
+ if (probesForceSynchronousAccess)
+ {
+ // The probes form a complete order across all partitions and
+ // the clustering key and partitioning key are the same. So, the
+ // only asynchronous I/O we will see will be due to ESPs. So,
+ // limit the count of streams by the count of streams in ESP.
+
+ // Get the logPhysPartitioningFunction, which we will use
+ // to get the logical partitioning function. If it's NULL,
+ // it means the table was not partitioned at all, so we don't
+ // need to limit anything since there already is no asynch I/O.
+ const LogPhysPartitioningFunction* lppf =
+ partFunc_->castToLogPhysPartitioningFunction();
+ if (lppf != NULL)
+ {
+ PartitioningFunction* logPartFunc =
+ lppf->getLogPartitioningFunction();
+ // Get the number of ESPs:
+ CostScalar numParts = logPartFunc->getCountOfPartitions();
+
+ countOfAsynchronousStreams_ = MINOF(numParts,
+ countOfAsynchronousStreams_);
+ } // lppf != NULL
+ } // probesForceSynchronousAccess
+ } // probes are in order
+ } // if input physical properties exist
+
+ CostScalar currentCpus =
+ (CostScalar)myContext->getPlan()->getPhysicalProperty()->getCurrentCountOfCPUs();
+ activeCpus_ = MINOF(countOfAsynchronousStreams_, currentCpus);
+
+ // update count of streams; the caller of the method uses this value
+ if ((countOfAsynchronousStreams_ > 0) &&
+ (countOfAsynchronousStreams_ < countOfStreams)
+ )
+ countOfStreams = (Lng32)countOfAsynchronousStreams_.getValue();
+
+
+ streamsPerCpu_ =
+ (countOfAsynchronousStreams_ / activeCpus_).getCeiling();
+
+ CostScalar noOfProbesPerPartition(csOne);
+
+ // Determine the number of probes per stream. Use this number as
+ // the number of rows to insert (this is "per-stream" costing).
+
+ noOfProbesPerPartition =
+ (noOfProbes_ / countOfAsynchronousStreams_).minCsOne();
+
+ // ************************************************************
+ // Compute the write/read cost for the insert
+ //
+ // ************************************************************
+
+ // ---------------------------------------------------------------------
+ // Synthesize the cost vectors.
+ // ---------------------------------------------------------------------
+ SimpleCostVector cvFR;
+ SimpleCostVector cvLR;
+
+ // we don't bother to estimate CPU time, I/O time, transfer time or idle
+ // time, since we really are only supporting the new cost model
+
+ cvFR.setNumProbes(noOfProbesPerPartition);
+ cvLR.setNumProbes(noOfProbesPerPartition);
+
+ // ---------------------------------------------------------------------
+ // Synthesize and return cost object.
+ // ---------------------------------------------------------------------
+
+ Cost *costPtr = new STMTHEAP
+ Cost(&cvFR
+ , &cvLR
+ , NULL
+ , Lng32(activeCpus_.getValue())
+ , Lng32(streamsPerCpu_.getValue())
+ );
+
+#ifndef NDEBUG
+ if (CmpCommon::getDefault(OPTIMIZER_PRINT_COST) == DF_ON)
+ {
+ pfp = stdout;
+ fprintf(pfp, "HbaseInsert elapsed time: ");
+ fprintf(pfp, "%f", costPtr->
+ convertToElapsedTime(
+ myContext->getReqdPhysicalProperty()).
+ value());
+ fprintf(pfp, "\n");
+ }
+#endif
+
+ return costPtr;
+} // CostMethodHbaseInsert::computeOperatorCostInternal
+
+// -----------------------------------------------------------------------
+// CostMethodHbaseInsert::cleanUp()
+//
+// The method cleans up cached parameters which need deallocation and
+// should be called after a costing session is done.
+// -----------------------------------------------------------------------
+void CostMethodHbaseInsert::cleanUp()
+{
+ activePartitions_ = csOne;
+ activeCpus_ = csOne;
+ streamsPerCpu_ = csOne;
+ countOfAsynchronousStreams_ = csOne;
+
+ // Clean up fields in base class
+ CostMethod::cleanUp();
+
+} // CostMethodHbaseInsert::cleanUp().
+
+//<pb>
[3/4] incubator-trafodion git commit: [TRAFODION-25] Second attempt
at INSERT costing
Posted by db...@apache.org.
[TRAFODION-25] Second attempt at INSERT costing
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/4c4a2c77
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/4c4a2c77
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/4c4a2c77
Branch: refs/heads/master
Commit: 4c4a2c77e3a477f1a38c6eddff766ac840c642bd
Parents: 4f654ec
Author: Dave Birdsall <db...@apache.org>
Authored: Thu Aug 27 20:38:21 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Thu Aug 27 20:38:21 2015 +0000
----------------------------------------------------------------------
core/sql/optimizer/costmethod.cpp | 185 ++++++++----------------------
core/sql/regress/compGeneral/TEST071 | 20 ++++
2 files changed, 71 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4c4a2c77/core/sql/optimizer/costmethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/costmethod.cpp b/core/sql/optimizer/costmethod.cpp
index 57bd4fd..e82ff7a 100644
--- a/core/sql/optimizer/costmethod.cpp
+++ b/core/sql/optimizer/costmethod.cpp
@@ -14042,140 +14042,12 @@ Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
cacheParameters(op, myContext);
estimateDegreeOfParallelism();
- const InputPhysicalProperty* ippForMe =
- myContext->getInputPhysicalProperty();
-
// ------------------------------------------------------
// Save off our current estimated degree of parallelism.
// in the 'out' parameter; we might revise it below
// ------------------------------------------------------
countOfStreams = countOfStreams_;
- HbaseInsert* insOp = (HbaseInsert *)op; // downcast
-
- if (ippForMe != NULL) // input physical properties exist?
- {
- // ----------------------------------------------------------
- // This long block of code is devoted to figuring out
- // if there are constraints to parallelism. For example,
- // if a layer of ESPs is doing the inserts, and the
- // degree of parallelism (DoP) at that layer is less
- // than the number of partitions in the table, and the
- // inserts arrive in partition order, then the regions
- // accessed by a given ESP are accessed serially. In
- // that case, we want to reduce the returned "countOfStreams"
- // to be the ESP layer DoP rather than the number of
- // partitions.
- // ----------------------------------------------------------
-
- // See if the probes are in order.
-
- // For insert, a partial order is of no help and so we can't use it.
- NABoolean partiallyInOrderOK = FALSE;
- NABoolean probesForceSynchronousAccess = FALSE;
- IndexDesc * CIDesc = (IndexDesc *)insOp->getIndexDesc();
- ValueIdList targetSortKey = CIDesc->getOrderOfKeyValues();
- ValueIdSet sourceCharInputs =
- insOp->getGroupAttr()->getCharacteristicInputs();
-
- ValueIdSet targetCharInputs;
- // The char inputs are still in terms of the source. Map them to the target.
- // Note: The source char outputs in the ipp have already been mapped to
- // the target. CharOutputs are a set, meaning they do not have duplicates
- // But we could have cases where two columns of the target are matched to the
- // same source column, for example, if we have
- // INSERT INTO b6table1
- // ( SELECT f, h_to_f, f, 8.4
- // FROM btre211
- // );
- // Hence we use lists here instead of sets.
- // Check to see if there are any duplicates in the source Characteristics inputs
- // if no, we shall perform set operations, as these are faster
-
- ValueIdList bottomValues = insOp->updateToSelectMap().getBottomValues();
- ValueIdSet bottomValuesSet(bottomValues);
- NABoolean useListInsteadOfSet = FALSE;
-
- CascadesGroup* group1 = (*CURRSTMT_OPTGLOBALS->memo)[insOp->getGroupId()];
-
- GenericUpdate* upOperator = (GenericUpdate *)group1->getFirstLogExpr();
-
- if (((upOperator->getTableName().getSpecialType() == ExtendedQualName::NORMAL_TABLE) || (upOperator->getTableName().getSpecialType() == ExtendedQualName::GHOST_TABLE)) &&
- (bottomValuesSet.entries() != bottomValues.entries()))
- {
-
- ValueIdList targetInputList;
- // from here get all the bottom values that appear in the sourceCharInputs
- bottomValues.findCommonElements(sourceCharInputs);
- bottomValuesSet = bottomValues;
-
- // we can use the bottomValues only if these contain some duplicate columns of
- // characteristics inputs, otherwise we shall use the characteristics inputs.
- if (bottomValuesSet == sourceCharInputs)
- {
- useListInsteadOfSet = TRUE;
- insOp->updateToSelectMap().rewriteValueIdListUpWithIndex(
- targetInputList,
- bottomValues);
- targetCharInputs = targetInputList;
- }
- }
-
- if (!useListInsteadOfSet)
- {
- insOp->updateToSelectMap().rewriteValueIdSetUp(
- targetCharInputs,
- sourceCharInputs);
- }
-
- // If a target key column is covered by a constant on the source side,
- // then we need to remove that column from the target sort key
- removeConstantsFromTargetSortKey(&targetSortKey,
- &(insOp->updateToSelectMap()));
- NABoolean orderedNJ = TRUE;
- // Don't call ordersMatch if njOuterOrder_ is null.
- if (ippForMe->getAssumeSortedForCosting())
- orderedNJ = FALSE;
- else
- // if leading keys are not same then don't try ordered NJ.
- orderedNJ =
- isOrderedNJFeasible(*(ippForMe->getNjOuterOrder()), targetSortKey);
-
- if (orderedNJ AND
- ordersMatch(ippForMe,
- CIDesc,
- &targetSortKey,
- targetCharInputs,
- partiallyInOrderOK,
- probesForceSynchronousAccess))
- {
- if (probesForceSynchronousAccess)
- {
- // The probes form a complete order across all partitions and
- // the clustering key and partitioning key are the same. So, the
- // only asynchronous I/O we will see will be due to ESPs. So,
- // limit the count of streams by the count of streams in ESP.
-
- // Get the logPhysPartitioningFunction, which we will use
- // to get the logical partitioning function. If it's NULL,
- // it means the table was not partitioned at all, so we don't
- // need to limit anything since there already is no asynch I/O.
- const LogPhysPartitioningFunction* lppf =
- partFunc_->castToLogPhysPartitioningFunction();
- if (lppf != NULL)
- {
- PartitioningFunction* logPartFunc =
- lppf->getLogPartitioningFunction();
- // Get the number of ESPs:
- CostScalar numParts = logPartFunc->getCountOfPartitions();
-
- countOfAsynchronousStreams_ = MINOF(numParts,
- countOfAsynchronousStreams_);
- } // lppf != NULL
- } // probesForceSynchronousAccess
- } // probes are in order
- } // if input physical properties exist
-
CostScalar currentCpus =
(CostScalar)myContext->getPlan()->getPhysicalProperty()->getCurrentCountOfCPUs();
activeCpus_ = MINOF(countOfAsynchronousStreams_, currentCpus);
@@ -14190,12 +14062,12 @@ Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
streamsPerCpu_ =
(countOfAsynchronousStreams_ / activeCpus_).getCeiling();
- CostScalar noOfProbesPerPartition(csOne);
+ CostScalar noOfProbesPerStream(csOne);
// Determine the number of probes per stream. Use this number as
// the number of rows to insert (this is "per-stream" costing).
- noOfProbesPerPartition =
+ noOfProbesPerStream =
(noOfProbes_ / countOfAsynchronousStreams_).minCsOne();
// ************************************************************
@@ -14209,11 +14081,56 @@ Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
SimpleCostVector cvFR;
SimpleCostVector cvLR;
- // we don't bother to estimate CPU time, I/O time, transfer time or idle
- // time, since we really are only supporting the new cost model
+ // For now, we don't bother to estimate CPU time, I/O time, transfer
+ // time or idle time, since we really are only supporting the new
+ // cost model.
+ //
+ // Future possible improvements:
+ //
+ // 1. Take into account HBase memstore insertion cost. The memstore
+ // uses a Red-Black tree which has o(n * log(n)) insertion cost. To
+ // model this correctly, we'd need to take into account the number
+ // of HBase regions rather than the number of ESPs, that is, to
+ // divide the number of probes by the number of HBase regions to find
+ // n. This cost will be paid no matter how many inserting streams
+ // there are so by itself this may not be interesting. It would only
+ // be interesting if there were a choice in the plan between inserting
+ // and not inserting (e.g. if we were considering bypassing the
+ // memstore, or if we were considering storing an intermediate result,
+ // neither of which are choices we examine today).
+ //
+ // 2. Take into account whether the probes are in key order. There
+ // is anecdotal evidence that if the probes are in key order, then
+ // memstore insertion cost is less. Possibly this is true only if
+ // inserting at the end or the beginning of the key range in a
+ // partition; intuitively inserting in the middle would seem to incur
+ // the full insertion cost. This is worthwhile taking into
+ // consideration as it opens the possibility of choosing between a
+ // plan that sorts rows in Trafodion before passing them to HBase
+ // vs. a plan that does not. To make this calculation we must know
+ // the memstore insertion cost (point 1 above), the order of the
+ // probes, whether the ESPs are aligned to the Regions of the
+ // target table, and whether we are inserting at the beginning or
+ // end of the key range. A first approximation to the last item
+ // would be whether the target table is empty. This is interesting
+ // because the case that we are doing an INSERT/SELECT into a new
+ // table is likely to be common.
+ //
+ // 3. Take into account memstore flush cost. We could add I/O time
+ // for flushes. For example, we could compare the number of probes
+ // per HBase Region with the number of rows that would cause a
+ // flush (the latter can be obtained from
+ // HbaseClient::estimateMemStoreRows() and is a function of the
+ // HBase parameter hbase.hregion.memstore.flush.size). Again, this
+ // cost will be paid no matter the plan choice so this is not
+ // interesting today. As with point 1, it becomes interesting only
+ // if there is a plan choice between inserting via memstore or not.
+ //
+ // In the interest of time, we move forward without these
+ // improvements for now.
- cvFR.setNumProbes(noOfProbesPerPartition);
- cvLR.setNumProbes(noOfProbesPerPartition);
+ cvFR.setNumProbes(noOfProbesPerStream);
+ cvLR.setNumProbes(noOfProbesPerStream);
// ---------------------------------------------------------------------
// Synthesize and return cost object.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4c4a2c77/core/sql/regress/compGeneral/TEST071
----------------------------------------------------------------------
diff --git a/core/sql/regress/compGeneral/TEST071 b/core/sql/regress/compGeneral/TEST071
index 3ee7546..41dac6b 100644
--- a/core/sql/regress/compGeneral/TEST071
+++ b/core/sql/regress/compGeneral/TEST071
@@ -4,6 +4,26 @@
-- Simple test case to set up a table with computed col key prefix,
-- do simple inserts, updates, deletes, merges.
-------------------------------------------------------------------------
+-- @@@ START COPYRIGHT @@@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+-- @@@ END COPYRIGHT @@@
obey TEST071(clean_up);
log LOG071 clear;