You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by hz...@apache.org on 2017/07/26 16:02:55 UTC
[2/5] incubator-trafodion git commit: [TRAFODION-2628] Updates for
elasticity
[TRAFODION-2628] Updates for elasticity
Replaced the list of CPU (node) numbers with an array, since it is
really a mapping array from logical to physical numbers.
Also changed some code in the adaptive segmentation code, but it is
likely that Adaptive Segmentation will not work (with or without my
change), this is not turned on by default and is not tested.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d17167cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d17167cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d17167cc
Branch: refs/heads/master
Commit: d17167ccf05176b1ddc945ae2ed04b54671bfd4a
Parents: 4bea445
Author: Hans Zeller <hz...@apache.org>
Authored: Tue Jul 11 00:00:56 2017 +0000
Committer: Hans Zeller <hz...@apache.org>
Committed: Tue Jul 11 00:00:56 2017 +0000
----------------------------------------------------------------------
core/sql/generator/Generator.cpp | 13 ++++++++-----
core/sql/optimizer/NAClusterInfo.cpp | 13 +++++++------
core/sql/optimizer/NAClusterInfo.h | 4 ++--
core/sql/optimizer/NodeMap.cpp | 8 ++++----
core/sql/optimizer/OptimizerSimulator.cpp | 16 ++++++++--------
5 files changed, 29 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d17167cc/core/sql/generator/Generator.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/Generator.cpp b/core/sql/generator/Generator.cpp
index 5e6f18b..0ca0e6f 100644
--- a/core/sql/generator/Generator.cpp
+++ b/core/sql/generator/Generator.cpp
@@ -935,6 +935,9 @@ NABoolean remapESPAllocationViaUserInputs(FragmentDir *fragDir,
// remapESPAllocationAS: Called by RelRoot::codeGen()
+// NOTE: This is likely not working at this time! Keep the value
+// of CQD AFFINITY_VALUE at '-2' for now.
+
// Re-assign each ESP to a CPU for adaptive segmentation based on the
// affinity value.
//
@@ -1073,8 +1076,8 @@ Generator::remapESPAllocationAS()
// A list of nodes in this cluster
//
- const NAList<CollIndex> &cpuList(gpClusterInfo->getCPUList());
- Int32 numCPUs = cpuList.entries();
+ const NAArray<CollIndex> &cpuArray(gpClusterInfo->getCPUArray());
+ Int32 numCPUs = cpuArray.entries();
Int32 numSegs = 1;
CollIndex espsPerNode =
@@ -1186,7 +1189,7 @@ Generator::remapESPAllocationAS()
for (i = 0; i < (CollIndex)numCPUs; i++) {
- cpus[i] = cpuList[cpu];
+ cpus[i] = cpu;
#ifdef _DEBUG
if ((CmpCommon::getDefault( NSK_DBG ) == DF_ON) &&
@@ -1202,7 +1205,7 @@ Generator::remapESPAllocationAS()
//
if((i % espsPerNode) == (espsPerNode-1))
cpu++;
- if (cpu >= (cpuList.entries() * espsPerNode)) {
+ if (cpu >= (cpuArray.entries() * espsPerNode)) {
cpu = 0;
}
}
@@ -1350,7 +1353,7 @@ Generator::remapESPAllocationAS()
// Set the cpu and segment for this node map entry.
//
- nodeMap->setNodeNumber(j, cpu);
+ nodeMap->setNodeNumber(j, cpuArray[cpu % numCPUs]);
}
// After remapping the node map (copy), make it the
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d17167cc/core/sql/optimizer/NAClusterInfo.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NAClusterInfo.cpp b/core/sql/optimizer/NAClusterInfo.cpp
index 714916a..4dc0985 100644
--- a/core/sql/optimizer/NAClusterInfo.cpp
+++ b/core/sql/optimizer/NAClusterInfo.cpp
@@ -187,7 +187,7 @@ static ULng32 intHashFunc(const Int32& Int)
//==============================================================================
NAClusterInfo::NAClusterInfo(CollHeap * heap)
: heap_(heap),
- cpuList_(heap),
+ cpuArray_(heap),
inTestMode_(FALSE)
{
OptimizerSimulator::osimMode mode = OptimizerSimulator::OFF;
@@ -244,7 +244,8 @@ NAClusterInfo::NAClusterInfo(CollHeap * heap)
if ((nodeInfo[i].type & MS_Mon_ZoneType_Aggregation) != 0 ||
((nodeInfo[i].type & MS_Mon_ZoneType_Storage) != 0 ))
{
- cpuList_.insert(nodeInfo[i].nid);
+ cpuArray_.insertAt(cpuArray_.entries(),
+ nodeInfo[i].nid);
// store nodeName-nodeId pairs
NAString *key_nodeName = new (heap_) NAString(nodeInfo[i].node_name, heap_);
@@ -279,7 +280,7 @@ NAClusterInfo::NAClusterInfo(CollHeap * heap)
case OptimizerSimulator::SIMULATE:
nodeIdToNodeNameMap_ = NULL;
- cpuList_.clear();
+ cpuArray_.clear();
//load NAClusterInfo from OSIM file
simulateNAClusterInfo();
break;
@@ -339,13 +340,13 @@ NABoolean NAClusterInfo::NODE_ID_TO_NAME(Int32 nodeId, char *nodeName, short max
Int32
NAClusterInfo::numOfPhysicalSMPs()
{
- return cpuList_.entries();
+ return cpuArray_.entries();
}
Int32
NAClusterInfo::numOfSMPs()
{
- Int32 result = cpuList_.entries();
+ Int32 result = cpuArray_.entries();
// This is temporary patch for PARALLEL_NUM_ESPS issue. This CQD should
// be used in many places for costing, NodeMap allocation, synthesizing
@@ -374,7 +375,7 @@ NAClusterInfo::numOfSMPs()
// Returns total number of CPUs (including down CPUs)
Lng32 NAClusterInfo::getTotalNumberOfCPUs()
{
- Lng32 cpuCount = cpuList_.entries();
+ Lng32 cpuCount = cpuArray_.entries();
#ifndef NDEBUG
// LCOV_EXCL_START
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d17167cc/core/sql/optimizer/NAClusterInfo.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NAClusterInfo.h b/core/sql/optimizer/NAClusterInfo.h
index c62f848..fbcfc66 100644
--- a/core/sql/optimizer/NAClusterInfo.h
+++ b/core/sql/optimizer/NAClusterInfo.h
@@ -108,7 +108,7 @@ public:
// return total number of CPUs (includes all, that is, even down CPUs)
Lng32 getTotalNumberOfCPUs();
- const NAList<CollIndex> &getCPUList() { return cpuList_; }
+ const NAArray<CollIndex> &getCPUArray() { return cpuArray_; }
Lng32 mapNodeNameToNodeNum(const NAString &node) const;
void cleanupPerStatement();
@@ -147,7 +147,7 @@ protected :
// the middle may be removed, so we end up with "holes" in the
// node ids.
// ------------------------------------------------------------------------
- NAList<CollIndex> cpuList_;
+ NAArray<CollIndex> cpuArray_;
//------------------------------------------------------------------------
// hashdictionary used to store the mapping of cluster name to cluster id
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d17167cc/core/sql/optimizer/NodeMap.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NodeMap.cpp b/core/sql/optimizer/NodeMap.cpp
index ddd556c..432200f 100644
--- a/core/sql/optimizer/NodeMap.cpp
+++ b/core/sql/optimizer/NodeMap.cpp
@@ -768,8 +768,8 @@ NodeMap::synthesizeLogicalMap(const CollIndex logicalNumEntries,
NodeMapEntry::NOT_ACTIVE, type());
//get a list of the nodes in the cluster
- const NAList<CollIndex> &cpuList(gpClusterInfo->getCPUList());
- Int32 cpuCount = cpuList.entries();
+ const NAArray<CollIndex> &cpuArray(gpClusterInfo->getCPUArray());
+ Int32 cpuCount = cpuArray.entries();
Lng32 affinityDef = ActiveSchemaDB()->getDefaults().getAsLong(AFFINITY_VALUE);
// "float" single ESP
@@ -812,9 +812,9 @@ NodeMap::synthesizeLogicalMap(const CollIndex logicalNumEntries,
// each ESP in an ESP exchange is active
logicalMap->setPartitionState(espX, NodeMapEntry::ACTIVE);
// assign ESP consumers to segments/cpus, round-robin
- logicalMap->setNodeNumber(espX, cpuList[cpuX]);
+ logicalMap->setNodeNumber(espX, cpuArray[cpuX]);
cpuX++; // advance to next cpu
- if (cpuX >= cpuList.entries()) {
+ if (cpuX >= cpuArray.entries()) {
cpuX = 0;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d17167cc/core/sql/optimizer/OptimizerSimulator.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptimizerSimulator.cpp b/core/sql/optimizer/OptimizerSimulator.cpp
index da2cd3f..39088cf 100644
--- a/core/sql/optimizer/OptimizerSimulator.cpp
+++ b/core/sql/optimizer/OptimizerSimulator.cpp
@@ -1200,10 +1200,10 @@ void NAClusterInfo::captureNAClusterInfo(ofstream & naclfile)
naclfile << " ";
naclfile.width(10); naclfile << LOCAL_CLUSTER << " ";
- naclfile << cpuList_.entries() << " : ";
- for (ci=0; ci<cpuList_.entries(); ci++)
+ naclfile << cpuArray_.entries() << " : ";
+ for (ci=0; ci<cpuArray_.entries(); ci++)
{
- naclfile.width(3); naclfile << cpuList_[ci] << " ";
+ naclfile.width(3); naclfile << cpuArray_[ci] << " ";
}
naclfile << endl;
@@ -1232,7 +1232,7 @@ void NAClusterInfo::simulateNAClusterInfo()
const char* filepath = CURRCONTEXT_OPTSIMULATOR->getLogFilePath(OptimizerSimulator::NACLUSTERINFO);
- cpuList_.clear();
+ cpuArray_.clear();
ifstream naclfile(filepath);
@@ -1257,7 +1257,7 @@ void NAClusterInfo::simulateNAClusterInfo()
}
else if (!strcmp(var, "clusterToCPUMap_"))
{
- Int32 numClusters, clusterNum, cpuList_entries, cpuNum;
+ Int32 numClusters, clusterNum, cpuArray_entries, cpuNum;
naclfile >> numClusters; naclfile.ignore(OSIM_LINEMAX, '\n');
// we don't support multiple clusters at this time
@@ -1269,11 +1269,11 @@ void NAClusterInfo::simulateNAClusterInfo()
for (i=0; i<numClusters; i++)
{
naclfile >> clusterNum;
- naclfile >> cpuList_entries; naclfile.ignore(OSIM_LINEMAX, ':');
- for (ci=0; ci<cpuList_entries; ci++)
+ naclfile >> cpuArray_entries; naclfile.ignore(OSIM_LINEMAX, ':');
+ for (ci=0; ci<cpuArray_entries; ci++)
{
naclfile >> cpuNum;
- cpuList_.insert(cpuNum);
+ cpuArray_.insertAt(ci, cpuNum);
}
naclfile.ignore(OSIM_LINEMAX, '\n');
}