You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2017/04/04 15:30:42 UTC
[1/4] incubator-trafodion git commit: [TRAFODION-25] Temporary code
for UPDATE costing
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 2e1e3ffa7 -> 1926da9c3
[TRAFODION-25] Temporary code for UPDATE costing
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/55e65d2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/55e65d2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/55e65d2b
Branch: refs/heads/master
Commit: 55e65d2b2a65153c6d5ef58d994980795560f72f
Parents: 1bd6c78
Author: Dave Birdsall <db...@apache.org>
Authored: Mon Apr 3 20:15:46 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Mon Apr 3 20:15:46 2017 +0000
----------------------------------------------------------------------
core/sql/optimizer/ScmCostMethod.cpp | 332 +++++++++++++++++++++++++++++-
1 file changed, 330 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/55e65d2b/core/sql/optimizer/ScmCostMethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScmCostMethod.cpp b/core/sql/optimizer/ScmCostMethod.cpp
index 16a0d87..c7a82c0 100644
--- a/core/sql/optimizer/ScmCostMethod.cpp
+++ b/core/sql/optimizer/ScmCostMethod.cpp
@@ -3845,8 +3845,336 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
const PlanWorkSpace* pws,
Lng32& countOfStreams)
{
- // TODO: Write this method; the line below is a stub
- return CostMethod::scmComputeOperatorCostInternal(op,pws,countOfStreams);
+ // TODO: Write this method; the code below is a copy of the Delete
+ // method which we'll use for the moment. This is better than just
+ // a simple constant stub; we will get parallel Update plans with
+ // this code, for example, that we won't get with constant cost.
+
+ // The theory of operation of Update is somewhat different (since it
+ // might, for example, do a Delete + an Insert, or might do an Hbase
+ // Update -- is that decided before we get here?), so this code will
+ // underestimate the cost in general.
+
+ const Context * myContext = pws->getContext();
+
+ cacheParameters(op,myContext);
+ estimateDegreeOfParallelism();
+
+ const InputPhysicalProperty* ippForMe =
+ myContext->getInputPhysicalProperty();
+
+ // -----------------------------------------
+ // Save off estimated degree of parallelism.
+ // -----------------------------------------
+ countOfStreams = countOfStreams_;
+
+ HbaseDelete* delOp = (HbaseDelete *)op; // downcast
+
+ CMPASSERT(partFunc_ != NULL);
+
+ // Later, if and when we start using NodeMaps to track active regions for
+ // Trafodion tables in HBase (or native HBase tables), we can use the
+ // following to get active partitions.
+ //CostScalar activePartitions =
+ // (CostScalar)
+ // (((NodeMap *)(partFunc_->getNodeMap()))->getNumActivePartitions());
+ // But for now, we do the following:
+ CostScalar activePartitions = (CostScalar)(partFunc_->getCountOfPartitions());
+
+ const IndexDesc* CIDesc = delOp->getIndexDesc();
+ const CostScalar & recordSizeInKb = CIDesc->getRecordSizeInKb();
+
+ CostScalar tuplesProcessed(csZero);
+ CostScalar tuplesProduced(csZero);
+ CostScalar tuplesSent(csZero); // we use tuplesSent to model sending rowIDs to Hbase
+ CostScalar randomIOs(csZero);
+ CostScalar sequentialIOs(csZero);
+
+ CostScalar countOfAsynchronousStreams = activePartitions;
+
+ // figure out if the probes are in order - if they are, then when
+ // scanning, I/O will tend to be sequential
+
+ NABoolean probesInOrder = FALSE;
+ if (ippForMe != NULL) // input physical properties exist?
+ {
+ // See if the probes are in order.
+
+ // For delete, a partial order is ok.
+ NABoolean partiallyInOrderOK = TRUE;
+ NABoolean probesForceSynchronousAccess = FALSE;
+ ValueIdList targetSortKey = CIDesc->getOrderOfKeyValues();
+ ValueIdSet sourceCharInputs =
+ delOp->getGroupAttr()->getCharacteristicInputs();
+
+ ValueIdSet targetCharInputs;
+ // The char inputs are still in terms of the source. Map them to the target.
+ // Note: The source char outputs in the ipp have already been mapped to
+ // the target. CharOutputs are a set, meaning they do not have duplicates
+ // But we could have cases where two columns of the target are matched to the
+ // same source column, example: Sol: 10-040416-5166, where we have
+ // INSERT INTO b6table1
+ // ( SELECT f, h_to_f, f, 8.4
+ // FROM btre211
+ // );
+ // Hence we use lists here instead of sets.
+ // Check to see if there are any duplicates in the source Characteristics inputs
+ // if no, we shall perform set operations, as these are faster
+ ValueIdList bottomValues = delOp->updateToSelectMap().getBottomValues();
+ ValueIdSet bottomValuesSet(bottomValues);
+ NABoolean useListInsteadOfSet = FALSE;
+
+ CascadesGroup* group1 = (*CURRSTMT_OPTGLOBALS->memo)[delOp->getGroupId()];
+
+ GenericUpdate* upOperator = (GenericUpdate *) group1->getFirstLogExpr();
+
+ if (((upOperator->getTableName().getSpecialType() == ExtendedQualName::NORMAL_TABLE ) || (upOperator->getTableName().getSpecialType() == ExtendedQualName::GHOST_TABLE )) &&
+ (bottomValuesSet.entries() != bottomValues.entries() ) )
+ {
+
+ ValueIdList targetInputList;
+ // from here get all the bottom values that appear in the sourceCharInputs
+ bottomValues.findCommonElements(sourceCharInputs );
+ bottomValuesSet = bottomValues;
+
+ // we can use the bottomValues only if these contain some duplicate columns of
+ // characteristics inputs, otherwise we shall use the characteristics inputs.
+ if (bottomValuesSet == sourceCharInputs)
+ {
+ useListInsteadOfSet = TRUE;
+ delOp->updateToSelectMap().rewriteValueIdListUpWithIndex(
+ targetInputList,
+ bottomValues);
+ targetCharInputs = targetInputList;
+ }
+ }
+
+ if (!useListInsteadOfSet)
+ {
+ delOp->updateToSelectMap().rewriteValueIdSetUp(
+ targetCharInputs,
+ sourceCharInputs);
+ }
+
+ // If a target key column is covered by a constant on the source side,
+ // then we need to remove that column from the target sort key
+ removeConstantsFromTargetSortKey(&targetSortKey,
+ &(delOp->updateToSelectMap()));
+ NABoolean orderedNJ = TRUE;
+ // Don't call ordersMatch if njOuterOrder_ is null.
+ if (ippForMe->getAssumeSortedForCosting())
+ orderedNJ = FALSE;
+ else
+ // if leading keys are not same then don't try ordered NJ.
+ orderedNJ =
+ isOrderedNJFeasible(*(ippForMe->getNjOuterOrder()), targetSortKey);
+
+ if (orderedNJ AND
+ ordersMatch(ippForMe,
+ CIDesc,
+ &targetSortKey,
+ targetCharInputs,
+ partiallyInOrderOK,
+ probesForceSynchronousAccess))
+ {
+ probesInOrder = TRUE;
+ if (probesForceSynchronousAccess)
+ {
+ // The probes form a complete order across all partitions and
+ // the clustering key and partitioning key are the same. So, the
+ // only asynchronous I/O we will see will be due to ESPs. So,
+ // limit the count of streams in DP2 by the count of streams in ESP.
+
+ // Get the logPhysPartitioningFunction, which we will use
+ // to get the logical partitioning function. If it's NULL,
+ // it means the table was not partitioned at all, so we don't
+ // need to limit anything since there already is no asynch I/O.
+
+ // TODO: lppf is always null in Trafodion; figure out what to do instead...
+ const LogPhysPartitioningFunction* lppf =
+ partFunc_->castToLogPhysPartitioningFunction();
+ if (lppf != NULL)
+ {
+ PartitioningFunction* logPartFunc =
+ lppf->getLogPartitioningFunction();
+ // Get the number of ESPs:
+ CostScalar numParts = logPartFunc->getCountOfPartitions();
+
+ countOfAsynchronousStreams = MINOF(numParts,
+ countOfAsynchronousStreams);
+ } // lppf != NULL
+ } // probesForceSynchronousAccess
+ } // probes are in order
+ } // if input physical properties exist
+
+ CostScalar currentCpus =
+ (CostScalar)myContext->getPlan()->getPhysicalProperty()->getCurrentCountOfCPUs();
+ CostScalar activeCpus = MINOF(countOfAsynchronousStreams, currentCpus);
+ CostScalar streamsPerCpu =
+ (countOfAsynchronousStreams / activeCpus).getCeiling();
+
+
+ CostScalar noOfProbesPerPartition(csOne);
+
+ CostScalar numRowsToDelete(csOne);
+ CostScalar numRowsToScan(csOne);
+
+ CostScalar commonComputation;
+
+ // Determine # of rows to scan and to delete
+
+ if (delOp->getSearchKey() && delOp->getSearchKey()->isUnique() &&
+ (noOfProbes_ == 1))
+ {
+ // unique access
+
+ activePartitions = csOne;
+ countOfAsynchronousStreams = csOne;
+ activeCpus = csOne;
+ streamsPerCpu = csOne;
+ numRowsToScan = csOne;
+ // assume the 1 row always satisfies any executor predicates so
+ // we'll always do the Delete
+ numRowsToDelete = csOne;
+ }
+ else
+ {
+ // non-unique access
+
+ numRowsToDelete =
+ ((myRowCount_ / activePartitions).getCeiling()).minCsOne();
+ noOfProbesPerPartition =
+ ((noOfProbes_ / countOfAsynchronousStreams).getCeiling()).minCsOne();
+
+ // need to compute the number of rows that satisfy the key predicates
+ // to compute the I/Os that must be performed
+
+ // need to create a new histogram, since the one from input logical
+ // prop. has the histogram for the table after all executor preds are
+ // applied (i.e. the result cardinality)
+ IndexDescHistograms histograms(*CIDesc,CIDesc->getIndexKey().entries());
+
+ // retrieve all of the key preds in key column order
+ ColumnOrderList keyPredsByCol(CIDesc->getIndexKey());
+ delOp->getSearchKey()->getKeyPredicatesByColumn(keyPredsByCol);
+
+ if ( NOT allKeyColumnsHaveHistogramStatistics( histograms, CIDesc ) )
+ {
+ // All key columns do not have histogram data, the best we can
+ // do is use the number of rows that satisfy all predicates
+ // (i.e. the number of rows we will be updating)
+ numRowsToScan = numRowsToDelete;
+ }
+ else
+ {
+ numRowsToScan = numRowsToScanWhenAllKeyColumnsHaveHistograms(
+ histograms,
+ keyPredsByCol,
+ activePartitions,
+ CIDesc
+ );
+ if (numRowsToScan < numRowsToDelete) // sanity test
+ {
+ // we will scan at least as many rows as we delete
+ numRowsToScan = numRowsToDelete;
+ }
+ }
+ }
+
+ // Notes: At execution time, several different TCBs can be created
+ // for a delete. We can class them three ways: Unique, Subset, and
+ // Rowset. Representative examples of the three classes are:
+ //
+ // ExHbaseUMDtrafUniqueTaskTcb
+ // ExHbaseUMDtrafSubsetTaskTcb
+ // ExHbaseAccessSQRowsetTcb
+ //
+ // The theory of operation of each of these differs somewhat.
+ //
+ // For the Unique variant, we use an HBase "get" to obtain a row, apply
+ // a predicate to it, then do an HBase "delete" to delete it if the
+ // predicate is true. (If there is no predicate, we'll simply do a
+ // "checkAndDelete" so there would be no "get" cost.)
+ //
+ // For the Subset variant, we use an HBase "scan" to obtain a sequence
+ // of rows, apply a predicate to each, then do an HBase "delete" on
+ // each row that passes the predicate.
+ //
+ // For the Rowset variant, we simply pass all the input keys to
+ // HBase in batches in HBase "deleteRows" calls. (In Explain plans,
+ // this TCB shows up as "trafodion_delete_vsbb", while the first two
+ // show up as "trafodion_delete".) There is no "get" cost. In plans
+ // with this TCB, there is a separate Scan TCB to obtain the keys,
+ // which then flow to this Rowset TCB via a tuple flow or nested join.
+ // (Such a separate Scan might exist with the first two TCBs also,
+ // e.g., when an index is used to decide which rows to delete.)
+ // The messaging cost to HBase is also reduced since multiple delete
+ // keys are sent per HBase interaction.
+ //
+ // Unfortunately the decisions as to which TCB will be used are
+ // currently made in the generator code and so aren't easily
+ // available to us here. For the moment then, we make no attempt
+ // to distinguish a separate "get" cost, nor do we take into account
+ // possible reduced message cost in the Rowset case. Should this
+ // choice be refactored in the future to push it into the Optimizer,
+ // then we can do a better job here. We did attempt to distinguish
+ // the unique case here from the others, but even there our criteria
+ // are not quite the same as in the generator. So at best, this attempt
+ // simply sharpens the cost estimate in this one particular case.
+
+
+ // Compute the I/O cost
+
+ computeIOCostsForCursorOperation(
+ randomIOs /* out */,
+ sequentialIOs /* out */,
+ CIDesc,
+ numRowsToScan,
+ probesInOrder
+ );
+
+ // Compute the tuple cost
+
+ tuplesProduced = numRowsToDelete;
+ tuplesProcessed = numRowsToScan;
+ tuplesSent = numRowsToDelete;
+
+ CostScalar rowSize = delOp->getIndexDesc()->getRecordLength();
+ CostScalar rowSizeFactor = scmRowSizeFactor(rowSize);
+ CostScalar outputRowSize = delOp->getGroupAttr()->getRecordLength();
+ CostScalar outputRowSizeFactor = scmRowSizeFactor(outputRowSize);
+
+ tuplesProcessed *= rowSizeFactor;
+ tuplesSent *= rowSizeFactor;
+ tuplesProduced *= outputRowSizeFactor;
+
+
+ // ---------------------------------------------------------------------
+ // Synthesize and return cost object.
+ // ---------------------------------------------------------------------
+
+ CostScalar probeRowSize = delOp->getIndexDesc()->getKeyLength();
+ Cost * updateCost =
+ scmCost(tuplesProcessed, tuplesProduced, tuplesSent, randomIOs, sequentialIOs, noOfProbes_,
+ rowSize, csZero, outputRowSize, probeRowSize);
+
+#ifndef NDEBUG
+if ( CmpCommon::getDefault( OPTIMIZER_PRINT_COST ) == DF_ON )
+ {
+ pfp = stdout;
+ fprintf(pfp, "HbaseDelete::scmComputeOperatorCostInternal()\n");
+ updateCost->getScmCplr().print(pfp);
+ fprintf(pfp, "HBase Delete elapsed time: ");
+ fprintf(pfp,"%f", updateCost->
+ convertToElapsedTime(
+ myContext->getReqdPhysicalProperty()).
+ value());
+ fprintf(pfp,"\n");
+ fprintf(pfp,"CountOfStreams returned %d\n",countOfStreams);
+ }
+#endif
+
+ return updateCost;
}
// ----QUICKSEARCH FOR HbaseDelete........................................
[3/4] incubator-trafodion git commit: Fixing incorrect cast
Posted by db...@apache.org.
Fixing incorrect cast
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/35d5cb81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/35d5cb81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/35d5cb81
Branch: refs/heads/master
Commit: 35d5cb81744326253df0128e942fb114b5e9b0f1
Parents: 20d0fac
Author: Dave Birdsall <db...@apache.org>
Authored: Mon Apr 3 22:29:59 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Mon Apr 3 22:29:59 2017 +0000
----------------------------------------------------------------------
core/sql/optimizer/ScmCostMethod.cpp | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/35d5cb81/core/sql/optimizer/ScmCostMethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScmCostMethod.cpp b/core/sql/optimizer/ScmCostMethod.cpp
index 405829c..ace650e 100644
--- a/core/sql/optimizer/ScmCostMethod.cpp
+++ b/core/sql/optimizer/ScmCostMethod.cpp
@@ -3868,7 +3868,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
// -----------------------------------------
countOfStreams = countOfStreams_;
- HbaseDelete* delOp = (HbaseDelete *)op; // downcast
+ HbaseUpdate* updOp = (HbaseUpdate *)op; // downcast
CMPASSERT(partFunc_ != NULL);
@@ -3881,7 +3881,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
// But for now, we do the following:
CostScalar activePartitions = (CostScalar)(partFunc_->getCountOfPartitions());
- const IndexDesc* CIDesc = delOp->getIndexDesc();
+ const IndexDesc* CIDesc = updOp->getIndexDesc();
const CostScalar & recordSizeInKb = CIDesc->getRecordSizeInKb();
CostScalar tuplesProcessed(csZero);
@@ -3905,7 +3905,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
NABoolean probesForceSynchronousAccess = FALSE;
ValueIdList targetSortKey = CIDesc->getOrderOfKeyValues();
ValueIdSet sourceCharInputs =
- delOp->getGroupAttr()->getCharacteristicInputs();
+ updOp->getGroupAttr()->getCharacteristicInputs();
ValueIdSet targetCharInputs;
// The char inputs are still in terms of the source. Map them to the target.
@@ -3920,11 +3920,11 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
// Hence we use lists here instead of sets.
// Check to see if there are any duplicates in the source Characteristics inputs
// if no, we shall perform set operations, as these are faster
- ValueIdList bottomValues = delOp->updateToSelectMap().getBottomValues();
+ ValueIdList bottomValues = updOp->updateToSelectMap().getBottomValues();
ValueIdSet bottomValuesSet(bottomValues);
NABoolean useListInsteadOfSet = FALSE;
- CascadesGroup* group1 = (*CURRSTMT_OPTGLOBALS->memo)[delOp->getGroupId()];
+ CascadesGroup* group1 = (*CURRSTMT_OPTGLOBALS->memo)[updOp->getGroupId()];
GenericUpdate* upOperator = (GenericUpdate *) group1->getFirstLogExpr();
@@ -3942,7 +3942,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
if (bottomValuesSet == sourceCharInputs)
{
useListInsteadOfSet = TRUE;
- delOp->updateToSelectMap().rewriteValueIdListUpWithIndex(
+ updOp->updateToSelectMap().rewriteValueIdListUpWithIndex(
targetInputList,
bottomValues);
targetCharInputs = targetInputList;
@@ -3951,7 +3951,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
if (!useListInsteadOfSet)
{
- delOp->updateToSelectMap().rewriteValueIdSetUp(
+ updOp->updateToSelectMap().rewriteValueIdSetUp(
targetCharInputs,
sourceCharInputs);
}
@@ -3959,7 +3959,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
// If a target key column is covered by a constant on the source side,
// then we need to remove that column from the target sort key
removeConstantsFromTargetSortKey(&targetSortKey,
- &(delOp->updateToSelectMap()));
+ &(updOp->updateToSelectMap()));
NABoolean orderedNJ = TRUE;
// Don't call ordersMatch if njOuterOrder_ is null.
if (ippForMe->getAssumeSortedForCosting())
@@ -4023,7 +4023,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
// Determine # of rows to scan and to delete
- if (delOp->getSearchKey() && delOp->getSearchKey()->isUnique() &&
+ if (updOp->getSearchKey() && updOp->getSearchKey()->isUnique() &&
(noOfProbes_ == 1))
{
// unique access
@@ -4056,7 +4056,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
// retrieve all of the key preds in key column order
ColumnOrderList keyPredsByCol(CIDesc->getIndexKey());
- delOp->getSearchKey()->getKeyPredicatesByColumn(keyPredsByCol);
+ updOp->getSearchKey()->getKeyPredicatesByColumn(keyPredsByCol);
if ( NOT allKeyColumnsHaveHistogramStatistics( histograms, CIDesc ) )
{
@@ -4139,9 +4139,9 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
tuplesProcessed = numRowsToScan;
tuplesSent = numRowsToDelete;
- CostScalar rowSize = delOp->getIndexDesc()->getRecordLength();
+ CostScalar rowSize = updOp->getIndexDesc()->getRecordLength();
CostScalar rowSizeFactor = scmRowSizeFactor(rowSize);
- CostScalar outputRowSize = delOp->getGroupAttr()->getRecordLength();
+ CostScalar outputRowSize = updOp->getGroupAttr()->getRecordLength();
CostScalar outputRowSizeFactor = scmRowSizeFactor(outputRowSize);
tuplesProcessed *= rowSizeFactor;
@@ -4153,7 +4153,7 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
// Synthesize and return cost object.
// ---------------------------------------------------------------------
- CostScalar probeRowSize = delOp->getIndexDesc()->getKeyLength();
+ CostScalar probeRowSize = updOp->getIndexDesc()->getKeyLength();
Cost * updateCost =
scmCost(tuplesProcessed, tuplesProduced, tuplesSent, randomIOs, sequentialIOs, noOfProbes_,
rowSize, csZero, outputRowSize, probeRowSize);
[4/4] incubator-trafodion git commit: Merge [TRAFODION-25] PR 1043
Temporary UPDATE costing code
Posted by db...@apache.org.
Merge [TRAFODION-25] PR 1043 Temporary UPDATE costing code
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/1926da9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/1926da9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/1926da9c
Branch: refs/heads/master
Commit: 1926da9c3d2c63ad3c112cd39b422616ec5c79b2
Parents: 2e1e3ff 35d5cb8
Author: Dave Birdsall <db...@apache.org>
Authored: Tue Apr 4 15:29:45 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Tue Apr 4 15:29:45 2017 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/analyzeOptimizerTrace.py | 215 +++++++++++++
core/sql/optimizer/ScmCostMethod.cpp | 332 ++++++++++++++++++++-
2 files changed, 545 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[2/4] incubator-trafodion git commit: Fixed typo; added script
Posted by db...@apache.org.
Fixed typo; added script
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/20d0fac4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/20d0fac4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/20d0fac4
Branch: refs/heads/master
Commit: 20d0fac440d95404f035f52dd99c22008ec448ba
Parents: 55e65d2
Author: Dave Birdsall <db...@apache.org>
Authored: Mon Apr 3 21:37:39 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Mon Apr 3 21:37:39 2017 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/analyzeOptimizerTrace.py | 215 +++++++++++++++++++++
core/sql/optimizer/ScmCostMethod.cpp | 4 +-
2 files changed, 217 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/20d0fac4/core/sqf/sql/scripts/analyzeOptimizerTrace.py
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/analyzeOptimizerTrace.py b/core/sqf/sql/scripts/analyzeOptimizerTrace.py
new file mode 100644
index 0000000..f2b0268
--- /dev/null
+++ b/core/sqf/sql/scripts/analyzeOptimizerTrace.py
@@ -0,0 +1,215 @@
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+#
+# This script extracts data from an Optimizer trace, created
+# using the following CQDs:
+#
+# cqd nsk_dbg_log_file 'optimizerTrace.txt';
+#
+# cqd NSK_DBG_GENERIC 'ON';
+# cqd NSK_DBG_PRINT_COST 'ON';
+# cqd NSK_DBG_PRINT_COST_LIMIT 'ON';
+# cqd NSK_DBG_PRINT_LOG_PROP 'ON';
+# cqd NSK_DBG_PRINT_PHYS_PROP 'ON';
+# cqd NSK_DBG_PRINT_TASK 'ON';
+# cqd NSK_DBG_PRINT_TASK_STACK 'ON';
+#
+# cqd NSK_DBG 'ON';
+#
+# The script extracts completed plans and shows their shape
+# and costs, along with whether the plan was chosen. The
+# goal of the script is to present the trace information in
+# a simple and high-level way.
+#
+
+import os
+import sys
+import subprocess
+import sets
+import datetime
+import argparse # requires Python 2.7
+
+
+
+def indentLevel(line):
+ i = 0
+ while i < len(line) and line[i] == ' ':
+ i = i + 1
+ return i
+
+
+class ParseTrace:
+ #
+
+ def __init__(self, OptimizerTraceFileName):
+ self.OptimizerTraceFileName = OptimizerTraceFileName
+
+
+ def parseTrace(self):
+ #
+ # The things we are looking for are:
+ #
+ # "Query to optimize:"
+ # This shows the start of a trace for a particular query
+ #
+ # "Task: 8 ParentTask: 7 Pass: 2 GroupId: 1 7 1 Create plan (0) ..."
+ # Plan fragments show up as a result of "Create plan" tasks being executed.
+ # Plan fragments may result in incomplete plans; we ignore those.
+ #
+ # "*** Qualified plan ***"
+ # This is the beginning of a qualified plan fragment, created by a "Create
+ # plan" task.
+ #
+ # "*** Chosen plan ***" and "*** Non-optimal plan ***"
+ # These indicate the end of a qualified plan fragment, and indicate whether
+ # the particular fragment was chosen for its group.
+ #
+ # "SQL node(<node description>)"
+ # This indicates a node within the plan. The tree structure of the nodes
+ # is indicated by the relative indentation of each of these. Python-like.
+ #
+ # "**** Roll-up Cost ****"
+ # This indicates that cost info for the subtree follows.
+ #
+ # "**** Operator Cost ****"
+ # This indicates that cost info for the node itself follows.
+ #
+ # "elapsed time = <floating point number>"
+ # This gives the cost, reduced to a single number.
+ #
+ # For now, we ignore the other information in the trace.
+ #
+
+ try:
+ f = open(self.OptimizerTraceFileName)
+ saveSQLNode = ''
+ state = 0
+ for line in f:
+ #print 'state = ' + str(state)
+ line = line.rstrip('\n') # get rid of trailing return character
+ if state == 0 and line == "Query to optimize:":
+ print "Trace for the following query:"
+ state = 1
+ elif state == 1:
+ if len(line) == 0:
+ state = 2
+ else:
+ print line
+ elif state == 2:
+ if line == "Query to optimize:":
+ state = 20 # ignore the rest of the file
+ tempLine = line.lstrip()
+ if tempLine.startswith('Task: ') and tempLine.find('Create plan'):
+ print
+ print tempLine
+ print
+ state = 3
+ elif state == 3:
+ tempLine = line.lstrip()
+ if tempLine == "*** Qualified plan ***":
+ state = 4
+ elif state == 4:
+ tempLine = line.lstrip()
+ if tempLine.startswith('SQL node('):
+ saveSQLNode = line
+ state = 5
+ elif state == 5:
+ tempLine = line.lstrip()
+ if tempLine.startswith('**** Roll-up Cost ****'):
+ state = 6
+ elif state == 6:
+ tempLine = line.lstrip()
+ if tempLine.startswith('elapsed time = '):
+ planCost = 'Plan cost ' + tempLine
+ print planCost
+ print
+ print saveSQLNode
+ n = indentLevel(line)
+ rollupCost = line[0:n] + 'Rollup cost ' + tempLine
+ print rollupCost
+ state = 9
+ elif state == 7:
+ tempLine = line.lstrip()
+ if tempLine.startswith('**** Roll-up Cost ****'):
+ state = 8
+ elif state == 8:
+ tempLine = line.lstrip()
+ if tempLine.startswith('elapsed time = '):
+ n = indentLevel(line)
+ rollupCost = line[0:n] + 'Rollup cost ' + tempLine
+ print rollupCost
+ state = 9
+ elif state == 9:
+ tempLine = line.lstrip()
+ if tempLine.startswith('**** Operator Cost ****'):
+ state = 10
+ elif state == 10:
+ tempLine = line.lstrip()
+ if tempLine.startswith('elapsed time = '):
+ n = indentLevel(line)
+ operatorCost = line[0:n] + 'Operator cost ' + tempLine
+ print operatorCost
+ state = 11
+ elif state == 11:
+ tempLine = line.lstrip()
+ if tempLine.startswith('SQL node('):
+ print line
+ state = 7
+ elif tempLine.startswith('Task: ') and tempLine.find('Create plan'):
+ print
+ print tempLine
+ print
+ state = 3
+ elif tempLine.startswith('*** Chosen plan ***') or tempLine.startswith('*** Non-optimal plan ***'):
+ print
+ print line
+ print
+ state = 2
+ f.close()
+
+ except IOError as detail:
+ print "Could not open " + self.OptimizerTraceFileName
+ print detail
+
+
+
+
+
+# beginning of main
+
+
+# process command line arguments
+
+parser = argparse.ArgumentParser(
+ description='This script parses out interesting data from an optimizer debug trace.')
+parser.add_argument("OptimizerTraceFileName", help='The name of the trace file you wish to parse.')
+
+args = parser.parse_args() # exits and prints help if args are incorrect
+
+exitCode = 0
+
+Traceparser = ParseTrace(args.OptimizerTraceFileName)
+
+Traceparser.parseTrace()
+
+exit(exitCode)
+
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/20d0fac4/core/sql/optimizer/ScmCostMethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScmCostMethod.cpp b/core/sql/optimizer/ScmCostMethod.cpp
index c7a82c0..405829c 100644
--- a/core/sql/optimizer/ScmCostMethod.cpp
+++ b/core/sql/optimizer/ScmCostMethod.cpp
@@ -4162,9 +4162,9 @@ CostMethodHbaseUpdate::scmComputeOperatorCostInternal(RelExpr* op,
if ( CmpCommon::getDefault( OPTIMIZER_PRINT_COST ) == DF_ON )
{
pfp = stdout;
- fprintf(pfp, "HbaseDelete::scmComputeOperatorCostInternal()\n");
+ fprintf(pfp, "HbaseUpdate::scmComputeOperatorCostInternal()\n");
updateCost->getScmCplr().print(pfp);
- fprintf(pfp, "HBase Delete elapsed time: ");
+ fprintf(pfp, "HBase Update elapsed time: ");
fprintf(pfp,"%f", updateCost->
convertToElapsedTime(
myContext->getReqdPhysicalProperty()).