You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2015/09/01 01:49:35 UTC

[1/4] incubator-trafodion git commit: More insert costing changes

Repository: incubator-trafodion
Updated Branches:
  refs/heads/master c5c516e05 -> 7610b1ea1


More insert costing changes


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/4f654ece
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/4f654ece
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/4f654ece

Branch: refs/heads/master
Commit: 4f654ecef4f3b3fcda2a844b3a1581bb0b97ce4f
Parents: 68b84db
Author: Dave Birdsall <db...@apache.org>
Authored: Wed Aug 26 22:01:31 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Wed Aug 26 22:02:13 2015 +0000

----------------------------------------------------------------------
 core/sql/optimizer/OptPhysRelExpr.cpp |  3 +--
 core/sql/optimizer/opt.cpp            | 10 ++++++++--
 2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4f654ece/core/sql/optimizer/OptPhysRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp
index 2786749..0674c33 100644
--- a/core/sql/optimizer/OptPhysRelExpr.cpp
+++ b/core/sql/optimizer/OptPhysRelExpr.cpp
@@ -16660,8 +16660,7 @@ HbaseInsert::costMethod() const
   static THREAD_P CostMethodHbaseInsert *m = NULL;
   if (m == NULL)
     m = new (GetCliGlobals()->exCollHeap()) CostMethodHbaseInsert();
-  // return m;  TODO: uncomment this line
-  return RelExpr::costMethod();  // TODO: remove this testing line
+  return m;
 } // HbaseInsert::costMethod()
 
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4f654ece/core/sql/optimizer/opt.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/opt.cpp b/core/sql/optimizer/opt.cpp
index d1fce2b..a417d63 100644
--- a/core/sql/optimizer/opt.cpp
+++ b/core/sql/optimizer/opt.cpp
@@ -998,12 +998,18 @@ NAString Context::getRequirementsString() const
 {
   NAString IPPString("", CmpCommon::statementHeap());
 
-  char thisptr[7 + sizeof(void *)];
+  // will receive the string "ptr=%p, ", where %p is a hex address
+  // example:  "ptr=0x7fffffff23db, "
+  // so we need 4 bytes for "ptr=", 2 bytes for the "0x", up to
+  // 2 * sizeof(void *) bytes for the hex (each nibble goes to
+  // one ASCII character), 2 bytes for the ", " and 1 byte for
+  // the trailing null 
+  char thisptr[4 + 2 + 2*sizeof(void *) + 2 + 1];
 
   if ( CmpCommon::getDefault( NSK_DBG_PRINT_CONTEXT_POINTER ) == DF_ON )
   {
     sprintf(thisptr, "ptr=%p, ",this);
-    thisptr[6 + sizeof(void *)]='\0';
+    thisptr[sizeof(thisptr)-1]='\0';  // shouldn't be needed, but be safe
   }
   else
     thisptr[0]='\0';


[4/4] incubator-trafodion git commit: Merge [TRAFODION-25] Part 1 Insert costing changes

Posted by db...@apache.org.
Merge [TRAFODION-25] Part 1 Insert costing changes


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/7610b1ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/7610b1ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/7610b1ea

Branch: refs/heads/master
Commit: 7610b1ea187766ac06536e4499cd48ab6239afd7
Parents: c5c516e 4c4a2c7
Author: Dave Birdsall <db...@apache.org>
Authored: Mon Aug 31 23:48:24 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Mon Aug 31 23:48:24 2015 +0000

----------------------------------------------------------------------
 core/sql/optimizer/CostMethod.h       |  64 ++--------
 core/sql/optimizer/OptPhysRelExpr.cpp |  11 ++
 core/sql/optimizer/RelUpdate.h        |   3 +
 core/sql/optimizer/ScmCostMethod.cpp  |  72 ++++++++++++
 core/sql/optimizer/costmethod.cpp     | 181 ++++++++++++++++++++++++++++-
 core/sql/optimizer/opt.cpp            |  10 +-
 core/sql/regress/compGeneral/TEST071  |  20 ++++
 7 files changed, 299 insertions(+), 62 deletions(-)
----------------------------------------------------------------------



[2/4] incubator-trafodion git commit: First draft insert costing

Posted by db...@apache.org.
First draft insert costing


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/68b84dbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/68b84dbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/68b84dbd

Branch: refs/heads/master
Commit: 68b84dbdc55db5fed1e1c2014f8ba9d2409109b0
Parents: 5928f31
Author: Dave Birdsall <db...@apache.org>
Authored: Tue Aug 11 20:46:27 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Wed Aug 26 22:02:13 2015 +0000

----------------------------------------------------------------------
 core/sql/optimizer/CostMethod.h       |  64 +------
 core/sql/optimizer/OptPhysRelExpr.cpp |  12 ++
 core/sql/optimizer/RelUpdate.h        |   3 +
 core/sql/optimizer/ScmCostMethod.cpp  |  72 ++++++++
 core/sql/optimizer/costmethod.cpp     | 264 ++++++++++++++++++++++++++++-
 5 files changed, 355 insertions(+), 60 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/CostMethod.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/CostMethod.h b/core/sql/optimizer/CostMethod.h
index 43c54ed..22f3d13 100644
--- a/core/sql/optimizer/CostMethod.h
+++ b/core/sql/optimizer/CostMethod.h
@@ -65,9 +65,8 @@ class CostMethodNestedJoinFlow;
 class CostMethodMergeUnion;
 class CostMethodTuple;
 class CostMethodTranspose;
-class CostMethodDP2Insert;
-class CostMethodDP2SideTreeInsert;
-class CostMethodDP2VSBBInsert;
+class CostMethodStoredProc;
+class CostMethodHbaseInsert;
 class CostMethodDP2Update;
 class CostMethodDP2Delete;
 class CostMethodUnPackRows;
@@ -1996,14 +1995,14 @@ protected:
 //<pb>
 /**********************************************************************/
 /*                                                                    */
-/*                         CostMethodDP2Insert                        */
+/*                         CostMethodHbaseInsert                      */
 /*                                                                    */
 /**********************************************************************/
-class CostMethodDP2Insert : public CostMethod
+class CostMethodHbaseInsert : public CostMethod
 {
 public:
   // Constructor
-  CostMethodDP2Insert() : CostMethod( "CostMethodDP2Insert" ) {}
+  CostMethodHbaseInsert() : CostMethod( "CostMethodHbaseInsert" ) {}
 
 protected:
   // Cost functions
@@ -2012,26 +2011,17 @@ protected:
                                             Lng32& countOfStreams);
 
   virtual Cost* scmComputeOperatorCostInternal(RelExpr* op,
-					       const Context* myContext,
                                                const PlanWorkSpace* pws,
 					       Lng32& countOfStreams);
 
   virtual void cacheParameters(RelExpr*, const Context *);
-  CostScalar getNumberOfInserts();
-  virtual void calculateIOCosts();
-  virtual CostScalar calculateNumberOfInserts();
   virtual void cleanUp();
-  virtual void calculateCpuAndIdleCosts();
 
 protected:
-  IndexDesc* CIDesc_;
   
   // storage related
-  CostScalar indexLevels_;
-  CostScalar kbPerBlock_;
-  CostScalar rowsPerBlock_;
-  CostScalar totalIndexBlocks_;
-  CostScalar entriesInIndexBlock_;
+  // TODO: add members to help compute whether inserts will
+  // fit in memstore or not if this seems interesting
 
   // parallelism related
   CostScalar activePartitions_;
@@ -2039,47 +2029,9 @@ protected:
   CostScalar streamsPerCpu_;
   CostScalar countOfAsynchronousStreams_;
 
-  NABoolean probesInOrder_;
-
-  // Used for MV logging pushdown
-  NABoolean usedForMvLogging_;
-  const PushDownProperty* pushDownProperty_;
-
-  // I/O and CPU related cost scalars
-  CostScalar transferFR_;
-  CostScalar transferLR_;
-  CostScalar seeksFR_;
-  CostScalar seeksLR_;
-  CostScalar numInserts_;
-  CostScalar cpuFR_;
-  CostScalar cpuLR_;
-  CostScalar idleFR_;
-  CostScalar idleLR_;
-
-}; // class CostMethodDP2Insert
-
-class CostMethodDP2SideTreeInsert : public CostMethodDP2Insert
-{
-public:
-  // constructor
-  CostMethodDP2SideTreeInsert() : CostMethodDP2Insert () {}
-
-protected:
-  virtual void calculateIOCosts();
-  virtual CostScalar calculateNumberOfInserts();
-};
-
-class CostMethodDP2VSBBInsert : public CostMethodDP2Insert
-{
-public:
-  // constructor
-  CostMethodDP2VSBBInsert() : CostMethodDP2Insert () {}
+}; // class CostMethodHbaseInsert
 
 
-protected:
-  virtual void calculateIOCosts();
-  virtual CostScalar calculateNumberOfInserts();
-};
 
 //<pb>
 /**********************************************************************/

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/OptPhysRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp
index 22b394b..2786749 100644
--- a/core/sql/optimizer/OptPhysRelExpr.cpp
+++ b/core/sql/optimizer/OptPhysRelExpr.cpp
@@ -16653,6 +16653,18 @@ HiveInsert::synthPhysicalProperty(const Context* myContext,
 
 } // HiveInsert::synthPhysicalProperty()
 
+
+CostMethod *
+HbaseInsert::costMethod() const
+{
+  static THREAD_P CostMethodHbaseInsert *m = NULL;
+  if (m == NULL)
+    m = new (GetCliGlobals()->exCollHeap()) CostMethodHbaseInsert();
+  // return m;  TODO: uncomment this line
+  return RelExpr::costMethod();  // TODO: remove this testing line
+} // HbaseInsert::costMethod()
+
+
 PhysicalProperty*
 HbaseInsert::synthPhysicalProperty(const Context* myContext,
                                    const Lng32     planNumber,

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/RelUpdate.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelUpdate.h b/core/sql/optimizer/RelUpdate.h
index d4b2c7f..008f6d9 100644
--- a/core/sql/optimizer/RelUpdate.h
+++ b/core/sql/optimizer/RelUpdate.h
@@ -1847,6 +1847,9 @@ public:
                                                   const Lng32    planNumber,
                                                   PlanWorkSpace  *pws);
 
+  // optimizer functions
+  virtual CostMethod* costMethod() const;
+
   // get a printable string that identifies the operator
   const NAString getText() const;
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/ScmCostMethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScmCostMethod.cpp b/core/sql/optimizer/ScmCostMethod.cpp
index 3d5d98b..85e1aa9 100644
--- a/core/sql/optimizer/ScmCostMethod.cpp
+++ b/core/sql/optimizer/ScmCostMethod.cpp
@@ -3629,6 +3629,78 @@ CostMethodMergeUnion::scmComputeOperatorCostInternal(RelExpr *op,
 
   return muCost;
 }      // CostMethodMergeUnion::scmComputeOperatorInternal()
+	   
+//<pb>
+
+//**************************************************************
+// This method computes the cost vector of the HbaseInsert operation
+//**************************************************************
+Cost*
+CostMethodHbaseInsert::scmComputeOperatorCostInternal(RelExpr* op,
+  const PlanWorkSpace* pws,
+  Lng32& countOfStreams)
+{
+  // TODO: For now assume we are always doing an HBase insert.
+  // Is it possible we go through this code path for Hive inserts?
+  // If so, figure out what to do.
+
+  HbaseInsert* insOp = (HbaseInsert *)op;   // downcast
+
+  // compute some details
+  const Context * myContext = pws->getContext();
+  Cost *costPtr = computeOperatorCostInternal(op, myContext, countOfStreams);
+
+  CostScalar noOfProbesPerStream(csOne);
+
+  // the number of rows to insert (this is "per-stream" costing).
+  CostScalar numOfProbesPerStream =
+    (noOfProbes_ / countOfAsynchronousStreams_).minCsOne();
+
+  CostScalar tuplesProcessed = numOfProbesPerStream;
+  CostScalar tuplesProduced = numOfProbesPerStream;
+
+  CostScalar ioRand = csZero;  // we don't bother estimating this
+  CostScalar ioSeq = csZero;  // we don't bother estimating this
+
+  // Factor in row sizes.
+  CostScalar rowSize = ((IndexDesc *)insOp->getIndexDesc())->getNAFileSet()->getRecordLength();
+  CostScalar rowSizeFactor = scmRowSizeFactor(rowSize);
+  tuplesProcessed *= rowSizeFactor;
+  tuplesProduced *= rowSizeFactor;
+
+  // there doesn't seem to be an estRowsAccessed_ member or
+  // related methods in hbaseInsert at the moment... add it
+  // when the need becomes apparent
+  //insOp->setEstRowsAccessed(noOfProbes_);
+
+  //----------------------------------------
+  //  Synthesize and return the cost object.
+  //----------------------------------------
+  Cost *hbaseInsertCost =
+    scmCost(tuplesProcessed, tuplesProduced, csZero, ioRand, ioSeq, noOfProbesPerStream_,
+            rowSize, csZero, rowSize, csZero);
+
+#ifndef NDEBUG
+  NABoolean printCost =
+    (CmpCommon::getDefault(OPTIMIZER_PRINT_COST) == DF_ON);
+  if (printCost)
+    {
+      pfp = stdout;
+      fprintf(pfp, "DP2INSERT::scmComputeOperatorCostInternal()\n");
+      hbaseInsertCost->getScmCplr().print(pfp);
+      fprintf(pfp, "DP2Insert elapsed time: ");
+      fprintf(pfp, "%f", hbaseInsertCost->convertToElapsedTime(myContext->getReqdPhysicalProperty()).value());
+      fprintf(pfp, "\n");
+    }
+#endif
+
+  // We use the call to computeOperatorCostInternal() to compute the various costs, but we
+  // we do not need the cost object computed by this method, we generate and return
+  // a different New Cost Model (NCM) cost object.
+  delete costPtr;
+
+  return hbaseInsertCost;
+}
 
 //<pb>
 

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68b84dbd/core/sql/optimizer/costmethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/costmethod.cpp b/core/sql/optimizer/costmethod.cpp
index 94cc37b..57bd4fd 100644
--- a/core/sql/optimizer/costmethod.cpp
+++ b/core/sql/optimizer/costmethod.cpp
@@ -19,7 +19,7 @@
 // under the License.
 //
 // @@@ END COPYRIGHT @@@
-**********************************************************************/
+/**********************************************************************/
 /* -*-C++-*-
 **************************************************************************
 *
@@ -13997,15 +13997,271 @@ CostMethodFastExtract::computeOperatorCostInternal(RelExpr* op,
 /**********************************************************************/
 
 //<pb>
-// ----QUICKSEARCH FOR DP2Insert........................................
+// ----QUICKSEARCH FOR HbaseInsert........................................
 
 /**********************************************************************/
 /*                                                                    */
-/*                      CostMethodDP2Insert                           */
+/*                      CostMethodHbaseInsert                         */
 /*                                                                    */
 /**********************************************************************/
 
-#pragma nowarn(262)   // warning elimination
+// -----------------------------------------------------------------------
+// CostMethodHbaseInsert::cacheParameters()
+// -----------------------------------------------------------------------
+void CostMethodHbaseInsert::cacheParameters(RelExpr* op, const Context * myContext)
+{
+  CostMethod::cacheParameters(op, myContext);
+
+  HbaseInsert* insOp = (HbaseInsert *)op;
+
+  CMPASSERT(partFunc_ != NULL);
+  NodeMap * nodeMap = (NodeMap *)partFunc_->getNodeMap();
+  if (nodeMap)
+    activePartitions_ = (CostScalar)nodeMap->getNumActivePartitions();
+  else
+    // Occasionally (e.g., regress/fullstack2/test023, the insert/select
+    // from t023t1 into t023t2 using a transpose operator), we get
+    // a ReplicateNoBroadcastPartitioningFunction lacking a node map.
+    // In this case we'll just use the number of partitions from the
+    // partitioning function itself -- which is probably an ESP count.
+    activePartitions_ = (CostScalar)partFunc_->getCountOfPartitions();
+
+  // The number of asynchronous streams is USUALLY the # of active parts.
+  countOfAsynchronousStreams_ = activePartitions_;
+} // CostMethodHbaseInsert::cacheParameters()
+
+
+
+// -----------------------------------------------------------------------
+// CostMethodHbaseInsert::computeOperatorCostInternal()
+// -----------------------------------------------------------------------
+Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
+  const Context* myContext,
+  Lng32& countOfStreams)
+{
+  cacheParameters(op, myContext);
+  estimateDegreeOfParallelism();
+
+  const InputPhysicalProperty* ippForMe =
+    myContext->getInputPhysicalProperty();
+
+  // ------------------------------------------------------
+  // Save off our current estimated degree of parallelism.
+  // in the 'out' parameter; we might revise it below
+  // ------------------------------------------------------
+  countOfStreams = countOfStreams_;
+
+  HbaseInsert* insOp = (HbaseInsert *)op;   // downcast
+
+  if (ippForMe != NULL)  // input physical properties exist?
+    {
+      // ----------------------------------------------------------
+      // This long block of code is devoted to figuring out
+      // if there are constraints to parallelism. For example,
+      // if a layer of ESPs is doing the inserts, and the
+      // degree of parallelism (DoP) at that layer is less 
+      // than the number of partitions in the table, and the
+      // inserts arrive in partition order, then the regions
+      // accessed by a given ESP are accessed serially. In
+      // that case, we want to reduce the returned "countOfStreams"
+      // to be the ESP layer DoP rather than the number of
+      // partitions.
+      // ---------------------------------------------------------- 
+
+      // See if the probes are in order.
+
+      // For insert, a partial order is of no help and so we can't use it.
+      NABoolean partiallyInOrderOK = FALSE;
+      NABoolean probesForceSynchronousAccess = FALSE;
+      IndexDesc * CIDesc = (IndexDesc *)insOp->getIndexDesc();
+      ValueIdList targetSortKey = CIDesc->getOrderOfKeyValues();
+      ValueIdSet sourceCharInputs =
+        insOp->getGroupAttr()->getCharacteristicInputs();
+
+      ValueIdSet targetCharInputs;
+      // The char inputs are still in terms of the source. Map them to the target.
+      // Note: The source char outputs in the ipp have already been mapped to
+      // the target. CharOutputs are a set, meaning they do not have duplicates
+      // But we could have cases where two columns of the target are matched to the
+      // same source column, for example, if we have
+      // INSERT INTO b6table1
+      //                  ( SELECT f, h_to_f, f, 8.4
+      //            FROM btre211
+      //            );
+      // Hence we use lists here instead of sets.
+      // Check to see if there are any duplicates in the source Characteristics inputs
+      // if no, we shall perform set operations, as these are faster
+
+      ValueIdList bottomValues = insOp->updateToSelectMap().getBottomValues();
+      ValueIdSet bottomValuesSet(bottomValues);
+      NABoolean useListInsteadOfSet = FALSE;
+
+      CascadesGroup* group1 = (*CURRSTMT_OPTGLOBALS->memo)[insOp->getGroupId()];
+
+      GenericUpdate* upOperator = (GenericUpdate *)group1->getFirstLogExpr();
+
+      if (((upOperator->getTableName().getSpecialType() == ExtendedQualName::NORMAL_TABLE) || (upOperator->getTableName().getSpecialType() == ExtendedQualName::GHOST_TABLE)) &&
+          (bottomValuesSet.entries() != bottomValues.entries()))
+        {
+
+          ValueIdList targetInputList;
+          // from here get all the bottom values that appear in the sourceCharInputs
+          bottomValues.findCommonElements(sourceCharInputs);
+          bottomValuesSet = bottomValues;
+
+          // we can use the bottomValues only if these contain some duplicate columns of
+          // characteristics inputs, otherwise we shall use the characteristics inputs.
+          if (bottomValuesSet == sourceCharInputs)
+            {
+              useListInsteadOfSet = TRUE;
+              insOp->updateToSelectMap().rewriteValueIdListUpWithIndex(
+                                                                       targetInputList,
+                                                                       bottomValues);
+              targetCharInputs = targetInputList;
+            }
+        }
+
+      if (!useListInsteadOfSet)
+        {
+          insOp->updateToSelectMap().rewriteValueIdSetUp(
+                                                         targetCharInputs,
+                                                         sourceCharInputs);
+        }
+
+      // If a target key column is covered by a constant on the source side,
+      // then we need to remove that column from the target sort key
+      removeConstantsFromTargetSortKey(&targetSortKey,
+                                       &(insOp->updateToSelectMap()));
+      NABoolean orderedNJ = TRUE;
+      // Don't call ordersMatch if njOuterOrder_ is null.
+      if (ippForMe->getAssumeSortedForCosting())
+        orderedNJ = FALSE;
+      else
+        // if leading keys are not same then don't try ordered NJ.
+        orderedNJ =
+          isOrderedNJFeasible(*(ippForMe->getNjOuterOrder()), targetSortKey);
+
+      if (orderedNJ AND
+          ordersMatch(ippForMe,
+                      CIDesc,
+                      &targetSortKey,
+                      targetCharInputs,
+                      partiallyInOrderOK,
+                      probesForceSynchronousAccess))
+        {
+          if (probesForceSynchronousAccess)
+            {
+              // The probes form a complete order across all partitions and
+              // the clustering key and partitioning key are the same. So, the
+              // only asynchronous I/O we will see will be due to ESPs. So,
+              // limit the count of streams by the count of streams in ESP.
+
+              // Get the logPhysPartitioningFunction, which we will use
+              // to get the logical partitioning function. If it's NULL,
+              // it means the table was not partitioned at all, so we don't
+              // need to limit anything since there already is no asynch I/O.
+              const LogPhysPartitioningFunction* lppf =
+                partFunc_->castToLogPhysPartitioningFunction();
+              if (lppf != NULL)
+                {
+                  PartitioningFunction* logPartFunc =
+                    lppf->getLogPartitioningFunction();
+                  // Get the number of ESPs:
+                  CostScalar numParts = logPartFunc->getCountOfPartitions();
+
+                  countOfAsynchronousStreams_ = MINOF(numParts,
+                                                      countOfAsynchronousStreams_);
+                } // lppf != NULL
+            } // probesForceSynchronousAccess
+        } // probes are in order
+    } // if input physical properties exist
+
+  CostScalar currentCpus =
+    (CostScalar)myContext->getPlan()->getPhysicalProperty()->getCurrentCountOfCPUs();
+  activeCpus_ = MINOF(countOfAsynchronousStreams_, currentCpus);
+
+  // update count of streams; the caller of the method uses this value
+  if ((countOfAsynchronousStreams_ > 0) &&
+      (countOfAsynchronousStreams_ < countOfStreams)
+      )
+    countOfStreams = (Lng32)countOfAsynchronousStreams_.getValue();
+
+
+  streamsPerCpu_ =
+    (countOfAsynchronousStreams_ / activeCpus_).getCeiling();
+
+  CostScalar noOfProbesPerPartition(csOne);
+
+  // Determine the number of probes per stream. Use this number as
+  // the number of rows to insert (this is "per-stream" costing).
+
+  noOfProbesPerPartition =
+    (noOfProbes_ / countOfAsynchronousStreams_).minCsOne();
+
+  // ************************************************************
+  // Compute the write/read cost for the insert
+  //
+  // ************************************************************
+
+  // ---------------------------------------------------------------------
+  // Synthesize the cost vectors.
+  // ---------------------------------------------------------------------
+  SimpleCostVector cvFR;
+  SimpleCostVector cvLR;
+
+  // we don't bother to estimate CPU time, I/O time, transfer time or idle 
+  // time, since we really are only supporting the new cost model
+
+  cvFR.setNumProbes(noOfProbesPerPartition);
+  cvLR.setNumProbes(noOfProbesPerPartition);
+
+  // ---------------------------------------------------------------------
+  // Synthesize and return cost object.
+  // ---------------------------------------------------------------------
+
+  Cost *costPtr = new STMTHEAP
+    Cost(&cvFR
+         , &cvLR
+         , NULL
+         , Lng32(activeCpus_.getValue())
+         , Lng32(streamsPerCpu_.getValue())
+         );
+
+#ifndef NDEBUG
+  if (CmpCommon::getDefault(OPTIMIZER_PRINT_COST) == DF_ON)
+    {
+      pfp = stdout;
+      fprintf(pfp, "HbaseInsert elapsed time: ");
+      fprintf(pfp, "%f", costPtr->
+              convertToElapsedTime(
+                                   myContext->getReqdPhysicalProperty()).
+              value());
+      fprintf(pfp, "\n");
+    }
+#endif
+
+  return costPtr;
+} // CostMethodHbaseInsert::computeOperatorCostInternal
+
+// -----------------------------------------------------------------------
+// CostMethodHbaseInsert::cleanUp()
+//
+// The method cleans up cached parameters which need deallocation and
+// should be called after a costing session is done.
+// -----------------------------------------------------------------------
+void CostMethodHbaseInsert::cleanUp()
+{
+  activePartitions_ = csOne;
+  activeCpus_ = csOne;
+  streamsPerCpu_ = csOne;
+  countOfAsynchronousStreams_ = csOne;
+
+  // Clean up fields in base class
+  CostMethod::cleanUp();
+
+}  // CostMethodHbaseInsert::cleanUp().
+
+//<pb>
 
 
 


[3/4] incubator-trafodion git commit: [TRAFODION-25] Second attempt at INSERT costing

Posted by db...@apache.org.
[TRAFODION-25] Second attempt at INSERT costing


Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/4c4a2c77
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/4c4a2c77
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/4c4a2c77

Branch: refs/heads/master
Commit: 4c4a2c77e3a477f1a38c6eddff766ac840c642bd
Parents: 4f654ec
Author: Dave Birdsall <db...@apache.org>
Authored: Thu Aug 27 20:38:21 2015 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Thu Aug 27 20:38:21 2015 +0000

----------------------------------------------------------------------
 core/sql/optimizer/costmethod.cpp    | 185 ++++++++----------------------
 core/sql/regress/compGeneral/TEST071 |  20 ++++
 2 files changed, 71 insertions(+), 134 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4c4a2c77/core/sql/optimizer/costmethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/costmethod.cpp b/core/sql/optimizer/costmethod.cpp
index 57bd4fd..e82ff7a 100644
--- a/core/sql/optimizer/costmethod.cpp
+++ b/core/sql/optimizer/costmethod.cpp
@@ -14042,140 +14042,12 @@ Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
   cacheParameters(op, myContext);
   estimateDegreeOfParallelism();
 
-  const InputPhysicalProperty* ippForMe =
-    myContext->getInputPhysicalProperty();
-
   // ------------------------------------------------------
   // Save off our current estimated degree of parallelism.
   // in the 'out' parameter; we might revise it below
   // ------------------------------------------------------
   countOfStreams = countOfStreams_;
 
-  HbaseInsert* insOp = (HbaseInsert *)op;   // downcast
-
-  if (ippForMe != NULL)  // input physical properties exist?
-    {
-      // ----------------------------------------------------------
-      // This long block of code is devoted to figuring out
-      // if there are constraints to parallelism. For example,
-      // if a layer of ESPs is doing the inserts, and the
-      // degree of parallelism (DoP) at that layer is less 
-      // than the number of partitions in the table, and the
-      // inserts arrive in partition order, then the regions
-      // accessed by a given ESP are accessed serially. In
-      // that case, we want to reduce the returned "countOfStreams"
-      // to be the ESP layer DoP rather than the number of
-      // partitions.
-      // ---------------------------------------------------------- 
-
-      // See if the probes are in order.
-
-      // For insert, a partial order is of no help and so we can't use it.
-      NABoolean partiallyInOrderOK = FALSE;
-      NABoolean probesForceSynchronousAccess = FALSE;
-      IndexDesc * CIDesc = (IndexDesc *)insOp->getIndexDesc();
-      ValueIdList targetSortKey = CIDesc->getOrderOfKeyValues();
-      ValueIdSet sourceCharInputs =
-        insOp->getGroupAttr()->getCharacteristicInputs();
-
-      ValueIdSet targetCharInputs;
-      // The char inputs are still in terms of the source. Map them to the target.
-      // Note: The source char outputs in the ipp have already been mapped to
-      // the target. CharOutputs are a set, meaning they do not have duplicates
-      // But we could have cases where two columns of the target are matched to the
-      // same source column, for example, if we have
-      // INSERT INTO b6table1
-      //                  ( SELECT f, h_to_f, f, 8.4
-      //            FROM btre211
-      //            );
-      // Hence we use lists here instead of sets.
-      // Check to see if there are any duplicates in the source Characteristics inputs
-      // if no, we shall perform set operations, as these are faster
-
-      ValueIdList bottomValues = insOp->updateToSelectMap().getBottomValues();
-      ValueIdSet bottomValuesSet(bottomValues);
-      NABoolean useListInsteadOfSet = FALSE;
-
-      CascadesGroup* group1 = (*CURRSTMT_OPTGLOBALS->memo)[insOp->getGroupId()];
-
-      GenericUpdate* upOperator = (GenericUpdate *)group1->getFirstLogExpr();
-
-      if (((upOperator->getTableName().getSpecialType() == ExtendedQualName::NORMAL_TABLE) || (upOperator->getTableName().getSpecialType() == ExtendedQualName::GHOST_TABLE)) &&
-          (bottomValuesSet.entries() != bottomValues.entries()))
-        {
-
-          ValueIdList targetInputList;
-          // from here get all the bottom values that appear in the sourceCharInputs
-          bottomValues.findCommonElements(sourceCharInputs);
-          bottomValuesSet = bottomValues;
-
-          // we can use the bottomValues only if these contain some duplicate columns of
-          // characteristics inputs, otherwise we shall use the characteristics inputs.
-          if (bottomValuesSet == sourceCharInputs)
-            {
-              useListInsteadOfSet = TRUE;
-              insOp->updateToSelectMap().rewriteValueIdListUpWithIndex(
-                                                                       targetInputList,
-                                                                       bottomValues);
-              targetCharInputs = targetInputList;
-            }
-        }
-
-      if (!useListInsteadOfSet)
-        {
-          insOp->updateToSelectMap().rewriteValueIdSetUp(
-                                                         targetCharInputs,
-                                                         sourceCharInputs);
-        }
-
-      // If a target key column is covered by a constant on the source side,
-      // then we need to remove that column from the target sort key
-      removeConstantsFromTargetSortKey(&targetSortKey,
-                                       &(insOp->updateToSelectMap()));
-      NABoolean orderedNJ = TRUE;
-      // Don't call ordersMatch if njOuterOrder_ is null.
-      if (ippForMe->getAssumeSortedForCosting())
-        orderedNJ = FALSE;
-      else
-        // if leading keys are not same then don't try ordered NJ.
-        orderedNJ =
-          isOrderedNJFeasible(*(ippForMe->getNjOuterOrder()), targetSortKey);
-
-      if (orderedNJ AND
-          ordersMatch(ippForMe,
-                      CIDesc,
-                      &targetSortKey,
-                      targetCharInputs,
-                      partiallyInOrderOK,
-                      probesForceSynchronousAccess))
-        {
-          if (probesForceSynchronousAccess)
-            {
-              // The probes form a complete order across all partitions and
-              // the clustering key and partitioning key are the same. So, the
-              // only asynchronous I/O we will see will be due to ESPs. So,
-              // limit the count of streams by the count of streams in ESP.
-
-              // Get the logPhysPartitioningFunction, which we will use
-              // to get the logical partitioning function. If it's NULL,
-              // it means the table was not partitioned at all, so we don't
-              // need to limit anything since there already is no asynch I/O.
-              const LogPhysPartitioningFunction* lppf =
-                partFunc_->castToLogPhysPartitioningFunction();
-              if (lppf != NULL)
-                {
-                  PartitioningFunction* logPartFunc =
-                    lppf->getLogPartitioningFunction();
-                  // Get the number of ESPs:
-                  CostScalar numParts = logPartFunc->getCountOfPartitions();
-
-                  countOfAsynchronousStreams_ = MINOF(numParts,
-                                                      countOfAsynchronousStreams_);
-                } // lppf != NULL
-            } // probesForceSynchronousAccess
-        } // probes are in order
-    } // if input physical properties exist
-
   CostScalar currentCpus =
     (CostScalar)myContext->getPlan()->getPhysicalProperty()->getCurrentCountOfCPUs();
   activeCpus_ = MINOF(countOfAsynchronousStreams_, currentCpus);
@@ -14190,12 +14062,12 @@ Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
   streamsPerCpu_ =
     (countOfAsynchronousStreams_ / activeCpus_).getCeiling();
 
-  CostScalar noOfProbesPerPartition(csOne);
+  CostScalar noOfProbesPerStream(csOne);
 
   // Determine the number of probes per stream. Use this number as
   // the number of rows to insert (this is "per-stream" costing).
 
-  noOfProbesPerPartition =
+  noOfProbesPerStream =
     (noOfProbes_ / countOfAsynchronousStreams_).minCsOne();
 
   // ************************************************************
@@ -14209,11 +14081,56 @@ Cost* CostMethodHbaseInsert::computeOperatorCostInternal(RelExpr* op,
   SimpleCostVector cvFR;
   SimpleCostVector cvLR;
 
-  // we don't bother to estimate CPU time, I/O time, transfer time or idle 
-  // time, since we really are only supporting the new cost model
+  // For now, we don't bother to estimate CPU time, I/O time, transfer 
+  // time or idle time, since we really are only supporting the new 
+  // cost model.
+  //
+  // Future possible improvements:
+  //
+  // 1. Take into account HBase memstore insertion cost. The memstore
+  // uses a Red-Black tree which has o(n * log(n)) insertion cost. To
+  // model this correctly, we'd need to take into account the number
+  // of HBase regions rather than the number of ESPs, that is, to
+  // divide the number of probes by the number of HBase regions to find
+  // n. This cost will be paid no matter how many inserting streams
+  // there are so by itself this may not be interesting. It would only
+  // be interesting if there were a choice in the plan between inserting
+  // and not inserting (e.g. if we were considering bypassing the
+  // memstore, or if we were considering storing an intermediate result,
+  // neither of which are choices we examine today).
+  //
+  // 2. Take into account whether the probes are in key order. There
+  // is anecdotal evidence that if the probes are in key order, then
+  // memstore insertion cost is less. Possibly this is true only if
+  // inserting at the end or the beginning of the key range in a 
+  // partition; intuitively inserting in the middle would seem to incur
+  // the full insertion cost. This is worthwhile taking into 
+  // consideration as it opens the possibility of choosing between a
+  // plan that sorts rows in Trafodion before passing them to HBase
+  // vs. a plan that does not. To make this calculation we must know
+  // the memstore insertion cost (point 1 above), the order of the 
+  // probes, whether the ESPs are aligned to the Regions of the 
+  // target table, and whether we are inserting at the beginning or
+  // end of the key range. A first approximation to the last item
+  // would be whether the target table is empty. This is interesting
+  // because the case that we are doing an INSERT/SELECT into a new
+  // table is likely to be common.
+  //
+  // 3. Take into account memstore flush cost. We could add I/O time
+  // for flushes. For example, we could compare the number of probes
+  // per HBase Region with the number of rows that would cause a
+  // flush (the latter can be obtained from 
+  // HbaseClient::estimateMemStoreRows() and is a function of the
+  // HBase parameter hbase.hregion.memstore.flush.size). Again, this
+  // cost will be paid no matter the plan choice so this is not
+  // interesting today. As with point 1, it becomes interesting only
+  // if there is a plan choice between inserting via memstore or not.
+  //
+  // In the interest of time, we move forward without these 
+  // improvements for now.
 
-  cvFR.setNumProbes(noOfProbesPerPartition);
-  cvLR.setNumProbes(noOfProbesPerPartition);
+  cvFR.setNumProbes(noOfProbesPerStream);
+  cvLR.setNumProbes(noOfProbesPerStream);
 
   // ---------------------------------------------------------------------
   // Synthesize and return cost object.

http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/4c4a2c77/core/sql/regress/compGeneral/TEST071
----------------------------------------------------------------------
diff --git a/core/sql/regress/compGeneral/TEST071 b/core/sql/regress/compGeneral/TEST071
index 3ee7546..41dac6b 100644
--- a/core/sql/regress/compGeneral/TEST071
+++ b/core/sql/regress/compGeneral/TEST071
@@ -4,6 +4,26 @@
 -- Simple test case to set up a table with computed col key prefix,
 -- do simple inserts, updates, deletes, merges.
 -------------------------------------------------------------------------
+-- @@@ START COPYRIGHT @@@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+-- @@@ END COPYRIGHT @@@
 
 obey TEST071(clean_up);
 log LOG071 clear;