You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2017/06/22 22:40:50 UTC
[1/4] incubator-trafodion git commit: [TRAFODION-2655] Fix two MDAM
optimizer bugs. Update optimizer simulator.
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 1555d5c16 -> 8eacf5b00
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/OptimizerSimulator.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptimizerSimulator.h b/core/sql/optimizer/OptimizerSimulator.h
index 11b6506..66f3a47 100644
--- a/core/sql/optimizer/OptimizerSimulator.h
+++ b/core/sql/optimizer/OptimizerSimulator.h
@@ -48,37 +48,12 @@ class HiveClient_JNI;
class ExeCliInterface;
class CmpSeabaseDDL;
class Queue;
-// Define PATH_MAX, FILENAME_MAX, LINE_MAX for both NT and NSK.
- // _MAX_PATH and _MAX_FNAME are defined in stdlib.h that is
- // included above.
- #define OSIM_PATHMAX PATH_MAX
- #define OSIM_FNAMEMAX FILENAME_MAX
- #define OSIM_LINEMAX 4096
-
+
class OsimAllHistograms;
class OsimHistogramEntry;
-
-#define TAG_ALL_HISTOGRAMS "all_histograms"
-#define TAG_HISTOGRAM_ENTRY "histogram_entry"
-#define TAG_FULL_PATH "fullpath"
-#define TAG_USER_NAME "username"
-#define TAG_PID "pid"
-#define TAG_CATALOG "catalog"
-#define TAG_SCHEMA "schema"
-#define TAG_TABLE "table"
-#define TAG_HISTOGRAM "histogram"
-//<TAG_ALL_HISTOGRAMS>
-// <TAG_HISTOGRAM_ENTRY>
-// <TAG_FULL_PATH>/opt/home/xx/xxx </TAG_FULL_PATH>
-// <TAG_USER_NAME>root</TAG_USER_NAME>
-// <TAG_PID>12345</TAG_PID>
-// <TAG_CATALOG>trafodion</TAG_CATALOG>
-// <TAG_SCHEMA>seabase</TAG_SCHEMA>
-// <TAG_TABLE>order042</TAG_TABLE>
-// <TAG_HISTOGRAM>sb_histogram_interval</TAG_HISTOGRAM>
-// </TAG_HISTOGRAM_ENTRY>
-// ...
-//</TAG_ALL_HISTOGRAMS>
+class HHDFSStatsBase;
+class HHDFSTableStats;
+struct hive_tbl_desc;
class OsimAllHistograms : public XMLElement
{
@@ -87,9 +62,7 @@ public:
OsimAllHistograms(NAMemory * heap = 0)
: XMLElement(NULL, heap)
, list_(heap)
- {
- setParent(this);
- }
+ { setParent(this); }
virtual const char *getElementName() const
{ return elemName; }
@@ -97,7 +70,7 @@ public:
virtual ElementType getElementType() const
{ return ElementType::ET_List; }
- NAList<OsimHistogramEntry*> & getEntries() { return list_; }
+ NAList<OsimHistogramEntry*> & getHistograms() { return list_; }
void addEntry( const char* fullpath,
const char* username,
@@ -110,15 +83,15 @@ public:
protected:
virtual void serializeBody(XMLString& xml);
virtual void startElement(void *parser, const char *elementName, const char **atts);
-
- NAList<OsimHistogramEntry*> list_;
-
private:
- // Copy construction/assignment not defined.
+ //disable copy constructor
OsimAllHistograms(const OsimAllHistograms&);
+ //disable assign operator
OsimAllHistograms& operator=(const OsimAllHistograms&);
+ NAList<OsimHistogramEntry*> list_;
};
+//represent one histogram file xxx.histograms or xxx.sb_histogram_interval
class OsimHistogramEntry : public XMLElement
{
friend class OsimAllHistograms;
@@ -144,7 +117,7 @@ public:
NAString & getFullPath() { return fullPath_; }
NAString & getUserName() { return userName_; }
NAString & getPID() { return pid_; }
- NAString & getCatalog() { return catalog_; }
+ NAString & getCatalog() { return catalog_; }
NAString & getSchema() { return schema_; }
NAString & getTable() { return table_; }
NAString & getHistogram() { return histogram_; }
@@ -162,17 +135,136 @@ private:
NAString schema_;
NAString table_;
NAString histogram_;
- // Copy construction/assignment not defined.
+ //disable copy constructor and assignment.
OsimHistogramEntry(const OsimHistogramEntry&);
OsimHistogramEntry& operator=(const OsimHistogramEntry&);
};
-
+//for reading histogram files path
class OsimElementMapper : public XMLElementMapper
{
public:
- virtual XMLElementPtr operator()(void *parser,
- char *elementName,
- AttributeList atts);
+ virtual XMLElementPtr operator()(void *parser, char *elementName, AttributeList atts);
+};
+//for reading hive table stats file path
+class OsimHHDFSStatsMapper : public XMLElementMapper
+{
+ public:
+ OsimHHDFSStatsMapper(NAMemory* h) : heap_(h){}
+ virtual XMLElementPtr operator()(void *parser, char *elementName, AttributeList atts);
+ NAMemory* heap_;
+};
+
+class OsimHHDFSStatsBase : public XMLElement
+{
+public:
+ OsimHHDFSStatsBase(XMLElement* parent, HHDFSStatsBase* mirror, NAMemory * heap = 0)
+ : XMLElement(parent, heap)
+ , statsList_(heap)
+ , mirror_(mirror)
+ , pos_(-1)
+ , heap_(heap)
+ {}
+
+ ~OsimHHDFSStatsBase(){
+ for ( CollIndex i = 0; i < statsList_.entries(); i++){
+ NADELETE(statsList_[i], OsimHHDFSStatsBase, heap_);
+ }
+ }
+
+ virtual ElementType getElementType() const
+ { return ElementType::ET_List; }
+
+ virtual void addEntry(OsimHHDFSStatsBase* statsBase, Int32 pos){ statsBase->setPosition(pos); statsList_.insert(statsBase); }
+ void setHHStats(HHDFSStatsBase* st) { mirror_ = st; }
+ HHDFSStatsBase * getHHStats() { return mirror_; }
+ void setPosition(Int32 p) { pos_ = p; }
+ Int32 getPosition() const { return pos_; }
+ virtual NABoolean restoreHHDFSStats(HHDFSStatsBase* hhstats, const char ** atts);
+ virtual NABoolean setValue(HHDFSStatsBase* hhstats, const char *attrName, const char *attrValue);
+
+protected:
+ virtual void serializeBody(XMLString& xml);
+ virtual void serializeAttrs(XMLString & xml);
+ virtual void startElement(void *parser, const char *elementName, const char **atts){}
+ virtual void endElement(void * parser, const char * elementName);
+
+ LIST(OsimHHDFSStatsBase*) statsList_;
+ HHDFSStatsBase* mirror_;
+ Int32 pos_;//position of this object in partition/bucket/file list
+ NAMemory* heap_;
+};
+
+class OsimHHDFSFileStats : public OsimHHDFSStatsBase
+{
+public:
+ static const char elemName[];
+ OsimHHDFSFileStats(XMLElement* parent, HHDFSStatsBase* mirror, NAMemory * heap = 0)
+ : OsimHHDFSStatsBase(parent, mirror, heap)
+ {}
+
+ virtual const char *getElementName() const
+ { return elemName; }
+
+ virtual NABoolean setValue(HHDFSStatsBase* stats, const char *attrName, const char *attrValue);
+
+protected:
+ virtual void serializeAttrs(XMLString & xml);
+};
+
+class OsimHHDFSBucketStats : public OsimHHDFSStatsBase
+{
+public:
+ static const char elemName[];
+ OsimHHDFSBucketStats(XMLElement* parent, HHDFSStatsBase* mirror, NAMemory * heap = 0)
+ : OsimHHDFSStatsBase(parent, mirror, heap)
+ {}
+
+ virtual const char *getElementName() const
+ { return elemName; }
+
+ virtual NABoolean setValue(HHDFSStatsBase* stats, const char *attrName, const char *attrValue);
+
+protected:
+ virtual void serializeAttrs(XMLString & xml);
+ virtual void startElement(void *parser, const char *elementName, const char **atts);
+};
+
+class OsimHHDFSListPartitionStats : public OsimHHDFSStatsBase
+{
+public:
+ static const char elemName[];
+ OsimHHDFSListPartitionStats(XMLElement* parent, HHDFSStatsBase* mirror, NAMemory * heap = 0)
+ : OsimHHDFSStatsBase(parent, mirror, heap)
+ {}
+
+ virtual const char *getElementName() const
+ { return elemName; }
+
+ virtual NABoolean setValue(HHDFSStatsBase* stats, const char *attrName, const char *attrValue);
+
+protected:
+ virtual void serializeAttrs(XMLString & xml);
+ virtual void startElement(void *parser, const char *elementName, const char **atts);
+
+};
+
+class OsimHHDFSTableStats : public OsimHHDFSStatsBase
+{
+public:
+ static const char elemName[];
+ OsimHHDFSTableStats(XMLElement* parent, HHDFSStatsBase * mirror, NAMemory * heap = 0)
+ : OsimHHDFSStatsBase(parent, mirror, heap)
+ {setParent(this);}
+
+ virtual const char *getElementName() const
+ { return elemName; }
+
+ virtual NABoolean setValue(HHDFSStatsBase* hhstats, const char *attrName, const char *attrValue);
+
+protected:
+ virtual void serializeAttrs(XMLString & xml);
+ virtual void startElement(void *parser, const char *elementName, const char **atts);
+
};
// This class initializes OSIM and implements capture and simulation
@@ -190,9 +282,9 @@ class OptimizerSimulator : public NABasicObject
UNLOAD
};
- enum sysCall {
- FIRST_SYSCALL,
- ESTIMATED_ROWS = FIRST_SYSCALL,
+ enum OsimLog {
+ FIRST_LOG,
+ ESTIMATED_ROWS = FIRST_LOG,
NODE_AND_CLUSTER_NUMBERS,
NACLUSTERINFO,
MYSYSTEMNUMBER,
@@ -208,7 +300,12 @@ class OptimizerSimulator : public NABasicObject
VERSIONSFILE,
CAPTURE_SYS_TYPE,
HISTOGRAM_PATHS,
- NUM_OF_SYSCALLS
+ HIVE_HISTOGRAM_PATHS,
+ HIVE_CREATE_TABLE,
+ HIVE_CREATE_EXTERNAL_TABLE,
+ HIVE_TABLE_LIST,
+ HHDFS_MASTER_HOST_LIST,
+ NUM_OF_LOGS
};
// sysType indicates the type of system that captured the queries
@@ -225,15 +322,11 @@ class OptimizerSimulator : public NABasicObject
osimMode getOsimMode() { return osimMode_; }
// Accessor methods to get and set osimLogHDFSDir_.
- void setOsimLogdir(const char *localdir) {
- if (localdir) {
- osimLogLocalDir_ = localdir;
- }
- }
+ void setOsimLogdir(const char *localdir);
const char* getOsimLogdir() const { return osimLogLocalDir_.isNull()? NULL : osimLogLocalDir_.data(); }
void initHashDictionaries();
- void setLogFilepath(sysCall sc);
- void openAndAddHeaderToLogfile(sysCall sc);
+ void createLogFilepath(OsimLog sc);
+ void openWriteLogStreams(OsimLog sc);
void initLogFilePaths();
void capture_CQDs();
@@ -252,13 +345,14 @@ class OptimizerSimulator : public NABasicObject
void debugMessage(const char* format, ...);
NABoolean setOsimModeAndLogDir(osimMode mode, const char * localdir);
-
+ NABoolean readHiveStmt(ifstream & DDLFile, NAString & stmt, NAString & comment);
NABoolean readStmt(ifstream & DDLFile, NAString & stmt, NAString & comment);
- NABoolean massageTableUID(OsimHistogramEntry* entry, NAHashDictionary<NAString, QualifiedName> * modifiedPathList);
+ NABoolean massageTableUID(OsimHistogramEntry* entry, NAHashDictionary<NAString, QualifiedName> * modifiedPathList, NABoolean isHive);
NABoolean isCallDisabled(ULng32 callBitPosition);
NABoolean runningSimulation();
NABoolean runningInCaptureMode();
+ NABoolean runningInLoadMode();
NAHashDictionary<const QualifiedName, Int64> *getHashDictTables()
{ return hashDict_Tables_; }
@@ -266,12 +360,15 @@ class OptimizerSimulator : public NABasicObject
NAHashDictionary<const QualifiedName, Int64> *getHashDictViews()
{ return hashDict_Views_; }
+ NAHashDictionary<const QualifiedName, Int64> *getHashDictHiveTables()
+ { return hashDict_HiveTables_; }
+
void readSysCallLogfiles();
void capture_getEstimatedRows(const char *tableName, double estRows);
void readLogfile_getEstimatedRows();
double simulate_getEstimatedRows(const char *tableName);
-
+ void restoreHHDFSMasterHostList();
void simulate_getNodeAndClusterNumbers(short& nodeNum, Int32& clusterNum);
void readLogFile_getNodeAndClusterNumbers();
void capture_getNodeAndClusterNumbers(short& nodeNum, Int32& clusterNum);
@@ -290,56 +387,55 @@ class OptimizerSimulator : public NABasicObject
void setClusterInfoInitialized(NABoolean b) { clusterInfoInitialized_ = b; }
// File pathnames for log files that contain system call data.
- const char * getLogFilePath (sysCall index) const
- { return sysCallLogFilePath_[index]; }
+ const char * getLogFilePath (OsimLog index) const
+ { return logFilePaths_[index]; }
void setForceLoad(NABoolean b) { forceLoad_ = b; }
NABoolean isForceLoad() const { return forceLoad_; }
-
+ HHDFSTableStats * restoreHiveTableStats(const QualifiedName & qualName, NAMemory* heap, hive_tbl_desc* hvt_desc);
+ void captureHiveTableStats(HHDFSTableStats* tablestats, const NATable* naTab);
private:
-
void readAndSetCQDs();
void dumpHistograms();
void dumpDDLs(const QualifiedName & qualifiedName);
+ void dumpHiveTableDDLs();
+ void dumpHiveHistograms();
void initializeCLI();
- void loadHistograms();
- short loadHistogramsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen);
- short loadHistogramIntervalsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen);
+ void loadHistograms(const char* histogramPath, NABoolean isHive);
+ short loadHistogramsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen, NABoolean isHive);
+ short loadHistogramIntervalsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen, NABoolean isHive);
Int64 getTableUID(const char * catName, const char * schName, const char * objName);
short fetchAllRowsFromMetaContext(Queue * &q, const char* query);
short executeFromMetaContext(const char* query);
void loadDDLs();
+ void loadHiveDDLs();
void histogramHDFSToLocal();
void removeHDFSCacheDirectory();
void createLogDir();
- void checkDuplicateNames();
void dropObjects();
void dumpVersions();
- void saveTablesBeforeStart();
- void saveViewsBeforeStart();
+ void dumpHHDFSMasterHostList();
void execHiveSQL(const char* hiveSQL);
// This is the directory OSIM uses to read/write log files.
NAString osimLogLocalDir_; //OSIM dir in local disk, used during capture and simu mode.
-
+ NAString hiveTableStatsDir_;
// This is the mode under which OSIM is running (default is OFF).
// It is set by an environment variable OSIM_MODE.
osimMode osimMode_;
// System call names.
- static const char *sysCallLogFileName_[NUM_OF_SYSCALLS];
+ static const char *logFileNames_[NUM_OF_LOGS];
- char *sysCallLogFilePath_[NUM_OF_SYSCALLS];
+ char *logFilePaths_[NUM_OF_LOGS];
- ofstream* writeSysCallStream_[NUM_OF_SYSCALLS];
+ ofstream* writeLogStreams_[NUM_OF_LOGS];
NAHashDictionary<NAString, double> *hashDict_getEstimatedRows_;
-
NAHashDictionary<const QualifiedName, Int64> *hashDict_Views_;
NAHashDictionary<const QualifiedName, Int64> *hashDict_Tables_;
NAHashDictionary<const QualifiedName, Int32> *hashDict_Synonyms_;
- NAHashDictionary<NAString, Int32> * hashDict_TablesBeforeAction_;
- NAHashDictionary<NAString, Int32> * hashDict_ViewsBeforeAction_;
+ NAHashDictionary<const QualifiedName, Int64> *hashDict_HiveTables_;
short nodeNum_;
Int32 clusterNum_;
@@ -367,21 +463,26 @@ class OptimizerSimulator : public NABasicObject
};
// System call wrappers.
-
+ void OSIM_restoreHHDFSMasterHostList();
short OSIM_MYSYSTEMNUMBER();
void OSIM_getNodeAndClusterNumbers(short& nodeNum, Int32& clusterNum);
void OSIM_captureTableOrView(NATable * naTab);
+ void OSIM_captureHiveTableStats(HHDFSTableStats* tablestats, const NATable * naTab);
void OSIM_capturePrologue();
- void OSIM_captureQueryText(const char * query);
void OSIM_captureQueryShape(const char * shape);
- void OSIM_captureEstimatedRows(const char *tableName, double estRows);
- double OSIM_simulateEstimatedRows(const char *tableName);
// errorMessage and warningMessage wrappers.
void OSIM_errorMessage(const char *msg);
void OSIM_warningMessage(const char *msg);
+ HHDFSTableStats * OSIM_restoreHiveTableStats(const QualifiedName & qualName, NAMemory* heap, hive_tbl_desc* hvt_desc);
+
+ // Check this, the parent and all ancestor CmpContext
+ // to find out if one of them is running in Load mode.
+ NABoolean OSIM_runningLoadEmbedded();
NABoolean OSIM_runningSimulation();
+ NABoolean OSIM_runningLoad();
NABoolean OSIM_runningInCaptureMode();
NABoolean OSIM_ustatIsDisabled();
NABoolean OSIM_ClusterInfoInitialized();
+ void raiseOsimException(const char* fmt, ...);
#endif
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/ScanOptimizer.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScanOptimizer.cpp b/core/sql/optimizer/ScanOptimizer.cpp
index c7b024b..8edc8b3 100644
--- a/core/sql/optimizer/ScanOptimizer.cpp
+++ b/core/sql/optimizer/ScanOptimizer.cpp
@@ -639,6 +639,9 @@ private:
// total number of rows in the table, it is akin to adding an additional
// table scan.
CostScalar cumulativePrefixSubsets_;
+ // Cached value of CQD MDAM_PROBE_TAX, to which cumulativePrefixSubsets_
+ // is multiplied when computing cost
+ double probeTax_;
// # of subset seeks of each effective probe
CostScalar prefixSubsetsAsSeeks_;
// # of rows of all probes at the current column level
@@ -8901,6 +8904,7 @@ MDAMOptimalDisjunctPrefixWA::MDAMOptimalDisjunctPrefixWA
,firstColOverlaps_(FALSE)
,prefixSubsets_(csOne) // MDAM subsets
,cumulativePrefixSubsets_(csZero)
+ ,probeTax_(ActiveSchemaDB()->getDefaults().getAsDouble(MDAM_PROBE_TAX))
,prefixSubsetsAsSeeks_(csOne) // MDAM subsets for all probes
,prefixRows_(0)
,prefixRqsts_(csOne)
@@ -9475,6 +9479,8 @@ void MDAMOptimalDisjunctPrefixWA::updatePositions()
// Note that there cannot be more positionings and more
// subsets than there are rows in the table
+ MDAM_DEBUG1(MTL2, "updatePositions() entered, prefixColumnPosition_: %d:", prefixColumnPosition_);
+
// The positionings include probing for the next subset
if (prefixColumnPosition_ == 0)
{
@@ -9483,6 +9489,9 @@ void MDAMOptimalDisjunctPrefixWA::updatePositions()
uecForPreviousCol_.getValue());
prefixSubsetsAsSeeks_ = prefixSubsets_;
prefixRqstsForSubsetBoundaries_ = csZero;
+
+ MDAM_DEBUG1(MTL2, "updatePositions(), prefixSubsets_ set: %f:", prefixSubsets_.value());
+ MDAM_DEBUG1(MTL2, "updatePositions(), uecForPreviousCol_: %f:", uecForPreviousCol_.value());
}
else
{
@@ -9500,11 +9509,11 @@ void MDAMOptimalDisjunctPrefixWA::updatePositions()
// case, don't multiply:
if (uecForPreviousCol_.isGreaterThanZero())
{
- MDAM_DEBUG1(MTL2, "updatePositions(), prefixSubsets_: %f:", prefixSubsets_.value());
+ MDAM_DEBUG1(MTL2, "updatePositions(), prefixSubsets_ before change: %f:", prefixSubsets_.value());
MDAM_DEBUG1(MTL2, "updatePositions(), uecForPreviousCol_: %f:", uecForPreviousCol_.value());
prefixSubsets_ *= uecForPreviousCol_;
- MDAM_DEBUG1(MTL2, "updatePositions(), updated prefixSubsets_: %f:", prefixSubsets_.value());
+ MDAM_DEBUG1(MTL2, "updatePositions(), updated prefixSubsets_ after change: %f:", prefixSubsets_.value());
prefixSubsets_ =
MINOF(innerRowsUpperBound_.getValue(),
@@ -9665,7 +9674,7 @@ void MDAMOptimalDisjunctPrefixWA::updateMinPrefix()
// adding cumulativePrefixSubsets_ represents the row handling costs of the probes of
// the MDAM algorithm as it traverses over key columns; the algorithm is recursive
// and thus has cumulative costs
- CostScalar scmPrefixRows = (prefixRows_ + cumulativePrefixSubsets_) * rowSizeFactor;
+ CostScalar scmPrefixRows = (prefixRows_ + (cumulativePrefixSubsets_ * probeTax_)) * rowSizeFactor;
CostScalar scmPrefixOutputRows = prefixRows_ * outputRowSizeFactor;
CostScalar rowSizeFactorSeqIO = optimizer_.scmRowSizeFactor(rowSize,
@@ -9695,6 +9704,13 @@ void MDAMOptimalDisjunctPrefixWA::updateMinPrefix()
scmCost =
optimizer_.scmCost(scmPrefixRows, scmPrefixOutputRows, csZero, scmPrefixIORand,
scmPrefixIOSeq, incomingProbes_, rowSize, csZero, outputRowSize, csZero);
+
+ MDAM_DEBUG1(MTL2, " Inputs to NCM cost for prefix: incomingProbes_: %f:", incomingProbes_.value());
+ MDAM_DEBUG1(MTL2, " scmPrefixRows: %f:", scmPrefixRows.value());
+ MDAM_DEBUG1(MTL2, " scmPrefixOutputRows: %f:", scmPrefixOutputRows.value());
+ MDAM_DEBUG1(MTL2, " scmPrefixIORand: %f:", scmPrefixIORand.value());
+ MDAM_DEBUG1(MTL2, " scmPrefixIOSeq: %f:", scmPrefixIOSeq.value());
+
MDAM_DEBUG1(MTL2, "NCM cost for prefix: %f:", scmCost->convertToElapsedTime(NULL).value());
}
else
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/ScmCostMethod.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ScmCostMethod.cpp b/core/sql/optimizer/ScmCostMethod.cpp
index 4fc6d88..d8ee81e 100644
--- a/core/sql/optimizer/ScmCostMethod.cpp
+++ b/core/sql/optimizer/ScmCostMethod.cpp
@@ -616,7 +616,7 @@ CostMethodDP2Scan::scmComputeOperatorCostInternal(RelExpr* op,
} else {
// Hive table case
- HHDFSStatsBase hdfsStats;
+ HHDFSStatsBase hdfsStats(/* HHDFSTableStats * tableStats */ NULL);
fs->getHiveSearchKey()->accumulateSelectedStats(hdfsStats);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index 7cf7a97..a7840ee 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -3760,6 +3760,9 @@ enum DefaultConstants
// A threshold of minitmal RC above which the above total UEC check will be applied.
MDAM_TOTAL_UEC_CHECK_MIN_RC_THRESHOLD,
+ // A multiplier of cumulative probe cost for MDAM
+ MDAM_PROBE_TAX,
+
// set to ON to aggressively allocate ESP per core
AGGRESSIVE_ESP_ALLOCATION_PER_CORE,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 08baca4..1cba380 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -2171,8 +2171,22 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591),
//
XDDflt__(MC_SKEW_SENSITIVITY_THRESHOLD, "0.1"),
-
- DDui___(MDAM_APPLY_RESTRICTION_CHECK, "2"),
+ // Applies additional heuristics to turn off MDAM. This code was added
+ // at a time when cumulative probe cost was not considered in the MDAM
+ // cost model (a MAJOR error). The meaning of the values:
+ // 0 - do nothing (no additional heuristics)
+ // 1 - require that more key columns have predicates than key columns
+ // without predicates
+ // 2 - require that the UECs of key columns lacking predicates is below
+ // a given threshold
+ // 3 - AND of 1 and 2
+ // These heuristics made sense when the cost model was broken and we
+ // traversed more deeply into the key than we should have. The cost model
+ // now takes cumulative probe cost into account so these should no longer
+ // be necessary (and indeed these heuristics can cause us to miss some
+ // very good plans, because they prevent us from considering MDAM plans
+ // that probe prefix columns before a column with high UEC, for example).
+ DDui___(MDAM_APPLY_RESTRICTION_CHECK, "0"),
DDflt0_(MDAM_CPUCOST_NET_OVH, "2000."),
@@ -2185,6 +2199,9 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591),
// allowed. Set it to 0 turns off the feature.
XDDui___(MDAM_NO_STATS_POSITIONS_THRESHOLD, "10"),
+ // a multiplier of probe cost used for MDAM
+ DDflt0_(MDAM_PROBE_TAX, "3"),
+
// MDAM_SCAN_METHOD ON means MDAM is enabled,
// OFF means MDAM is disabled. MDAM is enabled by default
// externalized 06/21/01 RV
@@ -4472,7 +4489,7 @@ void NADefaults::updateSystemParameters(NABoolean reInit)
case DEF_NUM_NODES_IN_ACTIVE_CLUSTERS:
- utoa_(((NAClusterInfoLinux*)gpClusterInfo)->numLinuxNodes(), valuestr);
+ utoa_(gpClusterInfo->numOfPhysicalSMPs(), valuestr);
strcpy(newValue, valuestr);
if(reInit)
[4/4] incubator-trafodion git commit: Merge [TRAFODION-2655] 2 MDAM
opt fixes + update to Optimizer Simulator
Posted by db...@apache.org.
Merge [TRAFODION-2655] 2 MDAM opt fixes + update to Optimizer Simulator
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/8eacf5b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/8eacf5b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/8eacf5b0
Branch: refs/heads/master
Commit: 8eacf5b0026808a694980968be4a40f175f60f09
Parents: 1555d5c 802029e
Author: Dave Birdsall <db...@apache.org>
Authored: Thu Jun 22 22:38:39 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Thu Jun 22 22:38:39 2017 +0000
----------------------------------------------------------------------
core/sql/arkcmp/CmpContext.cpp | 2 +
core/sql/arkcmp/CmpContext.h | 7 +
core/sql/cli/Context.h | 3 +
core/sql/export/NAStringDef.cpp | 14 +
core/sql/export/NAStringDef.h | 8 +-
core/sql/optimizer/HDFSHook.cpp | 58 +-
core/sql/optimizer/HDFSHook.h | 78 +-
core/sql/optimizer/NAClusterInfo.cpp | 265 +--
core/sql/optimizer/NAClusterInfo.h | 12 +-
core/sql/optimizer/NodeMap.cpp | 12 +-
core/sql/optimizer/ObjectNames.cpp | 17 +
core/sql/optimizer/ObjectNames.h | 3 +
core/sql/optimizer/OptPhysRelExpr.cpp | 2 +-
core/sql/optimizer/OptimizerSimulator.cpp | 2322 +++++++++++++++++-------
core/sql/optimizer/OptimizerSimulator.h | 257 ++-
core/sql/optimizer/ScanOptimizer.cpp | 22 +-
core/sql/optimizer/ScmCostMethod.cpp | 2 +-
core/sql/sqlcomp/DefaultConstants.h | 3 +
core/sql/sqlcomp/nadefaults.cpp | 23 +-
19 files changed, 2076 insertions(+), 1034 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8eacf5b0/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8eacf5b0/core/sql/optimizer/ObjectNames.h
----------------------------------------------------------------------
[3/4] incubator-trafodion git commit: [TRAFODION-2655] Fix two MDAM
optimizer bugs. Update optimizer simulator.
Posted by db...@apache.org.
[TRAFODION-2655] Fix two MDAM optimizer bugs. Update optimizer simulator.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/802029e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/802029e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/802029e5
Branch: refs/heads/master
Commit: 802029e583f790e7dd3ea0ecfbf3eb921dd8f073
Parents: 1083be3
Author: Dave Birdsall <db...@apache.org>
Authored: Wed Jun 21 22:58:41 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Wed Jun 21 22:58:41 2017 +0000
----------------------------------------------------------------------
core/sql/arkcmp/CmpContext.cpp | 2 +
core/sql/arkcmp/CmpContext.h | 7 +
core/sql/cli/Context.h | 3 +
core/sql/export/NAStringDef.cpp | 14 +
core/sql/export/NAStringDef.h | 8 +-
core/sql/optimizer/HDFSHook.cpp | 58 +-
core/sql/optimizer/HDFSHook.h | 78 +-
core/sql/optimizer/NAClusterInfo.cpp | 265 +--
core/sql/optimizer/NAClusterInfo.h | 12 +-
core/sql/optimizer/NodeMap.cpp | 12 +-
core/sql/optimizer/ObjectNames.cpp | 17 +
core/sql/optimizer/ObjectNames.h | 3 +
core/sql/optimizer/OptPhysRelExpr.cpp | 2 +-
core/sql/optimizer/OptimizerSimulator.cpp | 2322 +++++++++++++++++-------
core/sql/optimizer/OptimizerSimulator.h | 257 ++-
core/sql/optimizer/ScanOptimizer.cpp | 22 +-
core/sql/optimizer/ScmCostMethod.cpp | 2 +-
core/sql/sqlcomp/DefaultConstants.h | 3 +
core/sql/sqlcomp/nadefaults.cpp | 23 +-
19 files changed, 2076 insertions(+), 1034 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/arkcmp/CmpContext.cpp
----------------------------------------------------------------------
diff --git a/core/sql/arkcmp/CmpContext.cpp b/core/sql/arkcmp/CmpContext.cpp
index 7ca7d32..a2cdeb9 100644
--- a/core/sql/arkcmp/CmpContext.cpp
+++ b/core/sql/arkcmp/CmpContext.cpp
@@ -131,6 +131,8 @@ CmpContext::CmpContext(UInt32 f, CollHeap * h)
sqlTextBuf_(NULL),
uninitializedSeabaseErrNum_(0),
hbaseErrNum_(0),
+ numSQNodes_(0),
+ hasVirtualSQNodes_(FALSE),
trafMDDescsInfo_(NULL),
transMode_(TransMode::IL_NOT_SPECIFIED_, // init'd below
TransMode::READ_WRITE_,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/arkcmp/CmpContext.h
----------------------------------------------------------------------
diff --git a/core/sql/arkcmp/CmpContext.h b/core/sql/arkcmp/CmpContext.h
index 144d18b..dbe970d 100644
--- a/core/sql/arkcmp/CmpContext.h
+++ b/core/sql/arkcmp/CmpContext.h
@@ -395,7 +395,12 @@ public :
void setArkcmpEnvDirect(const char *name, const char *value,
NABoolean unset);
+ // variables managed by the HHDFSMasterHostList class
ARRAY(const char *) *getHosts() { return &hosts_; }
+ CollIndex getNumSQNodes() { return numSQNodes_; }
+ void setNumSQNodes(CollIndex n) { numSQNodes_ = n; }
+ NABoolean getHasVirtualSQNodes() { return hasVirtualSQNodes_; }
+ void setHasVirtualSQNodes(NABoolean v) { hasVirtualSQNodes_ = v; }
#endif // NA_CMPDLL
// used by sendAllControlsAndFlags() and restoreAllControlsAndFlags()
@@ -585,6 +590,8 @@ private:
Lng32 hbaseErrNum_;
NAString hbaseErrStr_;
ARRAY(const char *) hosts_;
+ CollIndex numSQNodes_;
+ NABoolean hasVirtualSQNodes_;
NAClusterInfo *clusterInfo_;
RuleSet *ruleSet_;
OptDebug *optDbg_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/cli/Context.h
----------------------------------------------------------------------
diff --git a/core/sql/cli/Context.h b/core/sql/cli/Context.h
index 0489acf..d1cce8d 100644
--- a/core/sql/cli/Context.h
+++ b/core/sql/cli/Context.h
@@ -207,6 +207,9 @@ public:
//expose cmpContextInfo_ to get HQC info of different contexts
const NAArray<CmpContextInfo *> & getCmpContextInfo() const { return cmpContextInfo_; }
+ //expose cmpContext stack to allow search
+ const LIST(CmpContext *)& getCmpContextsInUse() const { return cmpContextInUse_; }
+
CollIndex addTrustedRoutine(LmRoutine *r);
LmRoutine *findTrustedRoutine(CollIndex ix);
void putTrustedRoutine(CollIndex ix);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/export/NAStringDef.cpp
----------------------------------------------------------------------
diff --git a/core/sql/export/NAStringDef.cpp b/core/sql/export/NAStringDef.cpp
index 94ef1d6..b9d0960 100644
--- a/core/sql/export/NAStringDef.cpp
+++ b/core/sql/export/NAStringDef.cpp
@@ -239,6 +239,19 @@ NAString::copy() const
return temp;
}
+int
+NAString::extract(int begin, int end, NAString & target) const
+{
+ if(end >= length() || begin < 0 ||end < begin)
+ return -1;
+
+ for(int i = begin; i <= end; i++)
+ target.append((*this)[i]);
+
+ return end - begin;
+}
+
+
UInt32
NAString::hashFoldCase() const
{
@@ -424,6 +437,7 @@ char* NAString::buildBuffer(const char* formatTemplate, va_list args)
int bufferSize = 20049;
int msgSize = 0;
//buffer is managed by this static function
+ //the allocated memory is shared by all NAString objects.
static THREAD_P char *buffer = NULL;
va_list args2;
va_copy(args2, args);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/export/NAStringDef.h
----------------------------------------------------------------------
diff --git a/core/sql/export/NAStringDef.h b/core/sql/export/NAStringDef.h
index e46b6e1..c5e1124 100644
--- a/core/sql/export/NAStringDef.h
+++ b/core/sql/export/NAStringDef.h
@@ -379,6 +379,8 @@ public:
NABoolean contains(const NAString& pat, caseCompare cmp = exact) const;
NAString copy() const;
+ //Extract part of this string and append it to target
+ int extract(int begin, int end, NAString & target) const;
inline const char* toCharStar() const {return fbstring_.data();} ;
const char* data() const {return fbstring_.data();}
size_t first(char c) const { return fbstring_.find_first_of(c);}
@@ -393,6 +395,10 @@ public:
UInt32 hash() const;
UInt32 hashFoldCase() const;
void mash(UInt32& hash, UInt32 chars) const;
+
+ // index methods return:
+ // NA_NPOS if not found.
+ // 0 based index, if found.
size_t index(const char* pat, size_t i=0, caseCompare cmp = exact)
const;
@@ -477,7 +483,7 @@ protected:
void cow(size_t nc); // Do copy on write as needed
void initChar(char, NAMemory *h); // Initialize from char
-
+public:
static char* buildBuffer(const char* formatTemplate, va_list args);
private:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp
index 1a1cac4..9903be3 100644
--- a/core/sql/optimizer/HDFSHook.cpp
+++ b/core/sql/optimizer/HDFSHook.cpp
@@ -489,7 +489,7 @@ void HHDFSBucketStats::addFile(hdfsFS fs, hdfsFileInfo *fileInfo,
char recordTerminator,
CollIndex pos)
{
- HHDFSFileStats *fileStats = new(heap_) HHDFSFileStats(heap_);
+ HHDFSFileStats *fileStats = new(heap_) HHDFSFileStats(heap_, getTable());
if ( scount_ > 10 )
doEstimate = FALSE;
@@ -527,6 +527,19 @@ void HHDFSBucketStats::print(FILE *ofd)
HHDFSStatsBase::print(ofd, "bucket");
}
+OsimHHDFSStatsBase* HHDFSBucketStats::osimSnapShot(NAMemory * heap)
+{
+ OsimHHDFSBucketStats* stats = new(heap) OsimHHDFSBucketStats(NULL, this, heap);
+
+ for(Int32 i = 0; i < fileStatsList_.getUsedLength(); i++){
+ //"gaps" are not added, but record the position
+ if(fileStatsList_.getUsage(i) != UNUSED_COLL_ENTRY)
+ stats->addEntry(fileStatsList_[i]->osimSnapShot(heap), i);
+ }
+ return stats;
+}
+
+
HHDFSListPartitionStats::~HHDFSListPartitionStats()
{
for (CollIndex b=0; b<=defaultBucketIdx_; b++)
@@ -580,7 +593,7 @@ void HHDFSListPartitionStats::populate(hdfsFS fs,
if (! bucketStatsList_.used(bucketNum))
{
- bucketStats = new(heap_) HHDFSBucketStats(heap_);
+ bucketStats = new(heap_) HHDFSBucketStats(heap_, getTable());
bucketStatsList_.insertAt(bucketNum, bucketStats);
}
else
@@ -639,7 +652,7 @@ NABoolean HHDFSListPartitionStats::validateAndRefresh(hdfsFS fs, HHDFSDiags &dia
// first file for a new bucket got added
if (!refresh)
return FALSE;
- bucketStats = new(heap_) HHDFSBucketStats(heap_);
+ bucketStats = new(heap_) HHDFSBucketStats(heap_, getTable());
bucketStatsList_.insertAt(bucketNum, bucketStats);
}
else
@@ -798,6 +811,19 @@ void HHDFSListPartitionStats::print(FILE *ofd)
HHDFSStatsBase::print(ofd, "partition");
}
+OsimHHDFSStatsBase* HHDFSListPartitionStats::osimSnapShot(NAMemory * heap)
+{
+ OsimHHDFSListPartitionStats* stats = new(heap) OsimHHDFSListPartitionStats(NULL, this, heap);
+
+ for(Int32 i = 0; i < bucketStatsList_.getUsedLength(); i++)
+ {
+ //"gaps" are not added, but record the position
+ if(bucketStatsList_.getUsage(i) != UNUSED_COLL_ENTRY)
+ stats->addEntry(bucketStatsList_[i]->osimSnapShot(heap), i);
+ }
+ return stats;
+}
+
HHDFSTableStats::~HHDFSTableStats()
{
for (int p=0; p<totalNumPartitions_; p++)
@@ -1017,7 +1043,8 @@ NABoolean HHDFSTableStats::splitLocation(const char *tableLocation,
void HHDFSTableStats::processDirectory(const NAString &dir, Int32 numOfBuckets,
NABoolean doEstimate, char recordTerminator)
{
- HHDFSListPartitionStats *partStats = new(heap_) HHDFSListPartitionStats(heap_);
+ HHDFSListPartitionStats *partStats = new(heap_)
+ HHDFSListPartitionStats(heap_, this);
partStats->populate(fs_, dir, numOfBuckets, diags_, doEstimate, recordTerminator);
if (diags_.isSuccess())
@@ -1118,3 +1145,26 @@ void HHDFSTableStats::disconnectHDFS()
// is dropped or the thread exits.
}
+
+OsimHHDFSStatsBase* HHDFSTableStats::osimSnapShot(NAMemory * heap)
+{
+ OsimHHDFSTableStats* stats = new(heap) OsimHHDFSTableStats(NULL, this, heap);
+
+ for(Int32 i = 0; i < listPartitionStatsList_.getUsedLength(); i++)
+ {
+ //"gaps" are not added, but record the position
+ if(listPartitionStatsList_.getUsage(i) != UNUSED_COLL_ENTRY)
+ stats->addEntry(listPartitionStatsList_[i]->osimSnapShot(heap), i);
+ }
+ return stats;
+}
+
+OsimHHDFSStatsBase* HHDFSFileStats::osimSnapShot(NAMemory * heap)
+{
+ OsimHHDFSFileStats* stats = new(heap) OsimHHDFSFileStats(NULL, this, heap);
+
+ return stats;
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/HDFSHook.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/HDFSHook.h b/core/sql/optimizer/HDFSHook.h
index 7eef0bd..1c60d0f 100644
--- a/core/sql/optimizer/HDFSHook.h
+++ b/core/sql/optimizer/HDFSHook.h
@@ -49,6 +49,9 @@
// forward declarations
struct hive_tbl_desc;
class HivePartitionAndBucketKey;
+class HHDFSTableStats;
+class OsimHHDFSStatsBase;
+class OptimizerSimulator;
typedef CollIndex HostId;
typedef Int64 BucketNum;
@@ -57,6 +60,7 @@ typedef Int64 Offset;
class HHDFSMasterHostList : public NABasicObject
{
+friend class OptimizerSimulator;
public:
HHDFSMasterHostList(NAMemory *heap) {}
~HHDFSMasterHostList();
@@ -112,13 +116,18 @@ private:
class HHDFSStatsBase : public NABasicObject
{
+ friend class OsimHHDFSStatsBase;
public:
- HHDFSStatsBase() : numBlocks_(0),
- numFiles_(0),
- totalSize_(0),
- modificationTS_(0),
- sampledBytes_(0),
- sampledRows_(0) {}
+ HHDFSStatsBase(HHDFSTableStats *table) : numBlocks_(0),
+ numFiles_(0),
+ totalRows_(0),
+ totalStringLengths_(0),
+ totalSize_(0),
+ numStripes_(0),
+ modificationTS_(0),
+ sampledBytes_(0),
+ sampledRows_(0),
+ table_(table) {}
void add(const HHDFSStatsBase *o);
void subtract(const HHDFSStatsBase *o);
@@ -126,6 +135,9 @@ public:
Int64 getTotalSize() const { return totalSize_; }
Int64 getNumFiles() const { return numFiles_; }
Int64 getNumBlocks() const { return numBlocks_; }
+ Int64 getTotalRows() const { return totalRows_; }
+ Int64 getTotalStringLengths() { return totalStringLengths_; }
+ Int64 getNumStripes() const { return numStripes_; }
Int64 getSampledBytes() const { return sampledBytes_; }
Int64 getSampledRows() const { return sampledRows_; }
time_t getModificationTS() const { return modificationTS_; }
@@ -133,22 +145,34 @@ public:
Int64 getEstimatedRowCount() const;
Int64 getEstimatedRecordLength() const;
void print(FILE *ofd, const char *msg);
+ const HHDFSTableStats *getTable() const { return table_; }
+ HHDFSTableStats *getTable() { return table_; }
+
+ virtual OsimHHDFSStatsBase* osimSnapShot(NAMemory * heap){ return NULL; }
protected:
Int64 numBlocks_;
Int64 numFiles_;
+ Int64 totalRows_; // for ORC files
+ Int64 numStripes_; // for ORC files
+ Int64 totalStringLengths_; // for ORC files
Int64 totalSize_;
time_t modificationTS_; // last modification time of this object (file, partition/directory, bucket or table)
Int64 sampledBytes_;
Int64 sampledRows_;
+ HHDFSTableStats *table_;
};
class HHDFSFileStats : public HHDFSStatsBase
{
+ friend class OsimHHDFSFileStats;
public:
- HHDFSFileStats(NAMemory *heap) : heap_(heap),
- fileName_(heap),
- blockHosts_(NULL) {}
+ HHDFSFileStats(NAMemory *heap,
+ HHDFSTableStats *table) :
+ HHDFSStatsBase(table),
+ heap_(heap),
+ fileName_(heap),
+ blockHosts_(NULL) {}
~HHDFSFileStats();
void populate(hdfsFS fs,
hdfsFileInfo *fileInfo,
@@ -162,7 +186,9 @@ public:
Int64 getBlockSize() const { return blockSize_; }
HostId getHostId(Int32 replicate, Int64 blockNum) const
{ return blockHosts_[replicate*numBlocks_+blockNum]; }
- void print(FILE *ofd);
+ virtual void print(FILE *ofd);
+
+ virtual OsimHHDFSStatsBase* osimSnapShot(NAMemory * heap);
private:
@@ -178,8 +204,12 @@ private:
class HHDFSBucketStats : public HHDFSStatsBase
{
+ friend class OsimHHDFSBucketStats;
public:
- HHDFSBucketStats(NAMemory *heap) : heap_(heap), fileStatsList_(heap), scount_(0) {}
+ HHDFSBucketStats(NAMemory *heap,
+ HHDFSTableStats *table) :
+ HHDFSStatsBase(table),
+ heap_(heap), fileStatsList_(heap), scount_(0) {}
~HHDFSBucketStats();
const CollIndex entries() const { return fileStatsList_.entries(); }
@@ -196,6 +226,10 @@ public:
void removeAt(CollIndex i);
void print(FILE *ofd);
+ void insertAt(Int32 pos, HHDFSFileStats* st){ fileStatsList_.insertAt(pos, st); }
+
+ virtual OsimHHDFSStatsBase* osimSnapShot(NAMemory * heap);
+
private:
// list of files in this bucket
@@ -207,8 +241,12 @@ private:
class HHDFSListPartitionStats : public HHDFSStatsBase
{
+ friend class OsimHHDFSListPartitionStats;
public:
- HHDFSListPartitionStats(NAMemory *heap) : heap_(heap), partitionDir_(heap),
+ HHDFSListPartitionStats(NAMemory *heap,
+ HHDFSTableStats *table) :
+ HHDFSStatsBase(table),
+ heap_(heap), partitionDir_(heap),
bucketStatsList_(heap),
doEstimation_(FALSE),
recordTerminator_(0)
@@ -232,10 +270,16 @@ public:
Int32 determineBucketNum(const char *fileName);
void print(FILE *ofd);
+ void insertAt(Int32 pos, HHDFSBucketStats* st){ bucketStatsList_.insertAt(pos, st); }
+
+ virtual OsimHHDFSStatsBase* osimSnapShot(NAMemory * heap);
+
private:
// directory of the partition
NAString partitionDir_;
+ NAString partitionKeyValues_;
+ int partIndex_; // index in HDFSTableStats list
// number of buckets (from table DDL) or 0 if partition is not bucketed
// Note this value can never be 1. This value indicates the last
@@ -258,8 +302,11 @@ private:
class HHDFSTableStats : public HHDFSStatsBase
{
friend class HivePartitionAndBucketKey; // to be able to make a subarray of the partitions
+ friend class OsimHHDFSTableStats;
+ friend class OptimizerSimulator;
public:
- HHDFSTableStats(NAMemory *heap) : currHdfsPort_(-1),
+ HHDFSTableStats(NAMemory *heap) : HHDFSStatsBase(this),
+ currHdfsPort_(-1),
fs_(NULL),
hdfsPortOverride_(-1),
tableDir_(heap),
@@ -337,6 +384,11 @@ public:
const NAString &tableDir() const { return tableDir_; }
+ void insertAt(Int32 pos, HHDFSListPartitionStats * st) { listPartitionStatsList_.insertAt(pos, st); }
+ virtual OsimHHDFSStatsBase* osimSnapShot(NAMemory * heap);
+ void captureHiveTableStats(const NAString &tableName, Int64 lastModificationTs, hive_tbl_desc* hvt_desc );
+ static HHDFSTableStats * restoreHiveTableStats(const NAString & tableName, Int64 lastModificationTs, hive_tbl_desc* hvt_desc, NAMemory* heap);
+
const Lng32 numOfPartCols() const { return numOfPartCols_; }
const Lng32 totalNumPartitions() const { return totalNumPartitions_; }
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/NAClusterInfo.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NAClusterInfo.cpp b/core/sql/optimizer/NAClusterInfo.cpp
index 7682d08..17d3162 100644
--- a/core/sql/optimizer/NAClusterInfo.cpp
+++ b/core/sql/optimizer/NAClusterInfo.cpp
@@ -310,7 +310,7 @@ NAClusterInfo::NAClusterInfo(CollHeap * heap)
(&intHashFunc, 101,TRUE,heap);
activeClusters_= NULL;
- smpCount_ = -1;
+ physicalSMPCount_ = -1;
NADefaults::getNodeAndClusterNumbers(localSMP_ , localCluster_);
@@ -336,7 +336,7 @@ NAClusterInfo::NAClusterInfo(CollHeap * heap)
CMPASSERT(error == 0);
maps *cpuList=new(heap) maps(heap);
- smpCount_ = 0;
+ physicalSMPCount_ = 0;
NAList<CollIndex> storageList(heap, nodeCount);
@@ -357,7 +357,7 @@ NAClusterInfo::NAClusterInfo(CollHeap * heap)
cpuList->insertToAggregationNodeList(nodeInfo[i].nid);
if (!nodeInfo[i].spare_node)
- smpCount_++;
+ physicalSMPCount_++;
// store nodeName-nodeId pairs
NAString *key_nodeName = new (heap_) NAString(nodeInfo[i].node_name, heap_);
@@ -414,7 +414,7 @@ NAClusterInfo::NAClusterInfo(CollHeap * heap)
clusterToCPUMap_ = NULL;
nodeIdToNodeNameMap_ = NULL;
activeClusters_= NULL;
- smpCount_ = -1;
+ physicalSMPCount_ = -1;
//load NAClusterInfo from OSIM file
simulateNAClusterInfo();
break;
@@ -468,181 +468,6 @@ NAClusterInfo::~NAClusterInfo()
}
-//============================================================================
-// This method writes the information related to the NAClusterInfo class to a
-// logfile called "NAClusterInfo.txt".
-//============================================================================
-void NAClusterInfo::captureNAClusterInfo(ofstream & naclfile)
-{
- CollIndex i, ci;
- char filepath[OSIM_PATHMAX];
- char filename[OSIM_FNAMEMAX];
-
- // We don't capture data members that are computed during the compilation of
- // a query. These include:
- //
- // * smpCount_;
- // * tableToClusterMap_;
- // * activeClusters_;
- //
-
- naclfile << "localCluster_: " << localCluster_ << endl
- << "localSMP_: " << localSMP_ << endl;
-
- CollIndex *key_collindex;
- maps *val_maps;
- // Iterator for logging all the entries in clusterToCPUMap_ HashDictionary.
- NAHashDictionaryIterator<CollIndex, maps> C2CPUIter (*clusterToCPUMap_, NULL, NULL);
- naclfile << "clusterToCPUMap_: " << C2CPUIter.entries() << " :" << endl;
- if (C2CPUIter.entries() > 0)
- {
- // Write the header line for the table.
- naclfile << " ";
- naclfile.width(10);
- naclfile << "clusterNum" << " ";
- naclfile << "cpuList" << endl;
- for (i=0; i<C2CPUIter.entries(); i++)
- {
- C2CPUIter.getNext(key_collindex, val_maps);
- naclfile << " ";
- naclfile.width(10); naclfile << *key_collindex << " ";
- naclfile << val_maps->list->entries() << " : ";
- for (ci=0; ci<val_maps->list->entries(); ci++)
- {
- naclfile.width(3); naclfile << (*(val_maps->list))[ci] << " ";
- }
- naclfile << endl;
- }
- }
-
- Int32 * nodeID = NULL;
- NAString* nodeName = NULL;
- NAHashDictionaryIterator<Int32, NAString> nodeNameAndIDIter (*nodeIdToNodeNameMap_);
- naclfile << "nodeIdAndNodeNameMap: " << nodeNameAndIDIter.entries() << endl;
- for(nodeNameAndIDIter.getNext(nodeID, nodeName); nodeID && nodeName; nodeNameAndIDIter.getNext(nodeID, nodeName))
- {
- naclfile << *nodeID << " " << nodeName->data() << endl;
- }
-
- // Now save the OS-specific information to the NAClusterInfo.txt file
- captureOSInfo(naclfile);
-}
-
-//============================================================================
-// This method reads the information needed for NAClusterInfo class from
-// a logfile called "NAClusterInfo.txt" and then populates the variables
-// accordigly.
-//============================================================================
-void NAClusterInfo::simulateNAClusterInfo()
-{
- Int32 i, ci;
- char var[256];
-
- const char* filepath = CURRCONTEXT_OPTSIMULATOR->getLogFilePath(OptimizerSimulator::NACLUSTERINFO);
-
- activeClusters_= NULL;
- smpCount_ = -1;
-
- ifstream naclfile(filepath);
-
- if(!naclfile.good())
- {
- char errMsg[38+OSIM_PATHMAX+1]; // Error msg below + filename + '\0'
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.", filepath);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
- }
-
- while(naclfile.good())
- {
- // Read the variable name from the file.
- naclfile.getline(var, sizeof(var), ':');
- if(!strcmp(var, "localCluster_"))
- {
- naclfile >> localCluster_; naclfile.ignore(OSIM_LINEMAX, '\n');
- }
- else if (!strcmp(var, "localSMP_"))
- {
- naclfile >> localSMP_; naclfile.ignore(OSIM_LINEMAX, '\n');
- }
- else if (!strcmp(var, "clusterToCPUMap_"))
- {
- Int32 C2CPU_entries, clusterNum, cpuList_entries, cpuNum;
-
- clusterToCPUMap_ = new(heap_) NAHashDictionary<CollIndex,maps>(&clusterNumHashFunc,17,TRUE, heap_);
- naclfile >> C2CPU_entries; naclfile.ignore(OSIM_LINEMAX, '\n');
- if(C2CPU_entries > 0)
- {
- // Read and ignore the header line.
- naclfile.ignore(OSIM_LINEMAX, '\n');
- for (i=0; i<C2CPU_entries; i++)
- {
- naclfile >> clusterNum;
- naclfile >> cpuList_entries; naclfile.ignore(OSIM_LINEMAX, ':');
- CollIndex *key_clusterNum = new(heap_) CollIndex(clusterNum);
- maps *val_cpuList = new(heap_) maps(heap_);
- for (ci=0; ci<cpuList_entries; ci++)
- {
- naclfile >> cpuNum;
- val_cpuList->list->insert(cpuNum);
- }
- naclfile.ignore(OSIM_LINEMAX, '\n');
- CollIndex *checkClusterNum = clusterToCPUMap_->insert(key_clusterNum, val_cpuList);
- CMPASSERT(checkClusterNum);
- }
- }
- }
- else if(!strcmp(var, "nodeIdAndNodeNameMap"))
- {
- Int32 id_name_entries;
- Int32 nodeId;
- char nodeName[256];
- nodeIdToNodeNameMap_ = new(heap_) NAHashDictionary<Int32, NAString>
- (&intHashFunc, 101,TRUE,heap_);
-
- nodeNameToNodeIdMap_ = new(heap_) NAHashDictionary<NAString, Int32>
- (&NAString::hash, 101,TRUE,heap_);
- naclfile >> id_name_entries;
- naclfile.ignore(OSIM_LINEMAX, '\n');
- for(i = 0; i < id_name_entries; i++)
- {
- naclfile >> nodeId >> nodeName;
- naclfile.ignore(OSIM_LINEMAX, '\n');
-
- //populate clusterId<=>clusterName map from file
- Int32 * key_nodeId = new Int32(nodeId);
- NAString * val_nodeName = new (heap_) NAString(nodeName, heap_);
- Int32 * retId = nodeIdToNodeNameMap_->insert(key_nodeId, val_nodeName);
- //CMPASSERT(retId);
-
- NAString * key_nodeName = new (heap_) NAString(nodeName, heap_);
- Int32 * val_nodeId = new Int32(nodeId);
- NAString * retName = nodeNameToNodeIdMap_->insert(key_nodeName, val_nodeId);
- //some node names are like g4t3024:0, g4t3024:1
- //I don't know why we need to remove strings after ':' or '.' in node name,
- //but if string after ':' or '.' is removed, same node names correspond to different node ids,
- //this can cause problems here
- //CMPASSERT(retName);
- }
- }
- else
- {
- // This variable will either be read in simulateNAClusterInfoNSK()
- // method of NAClusterInfoNSK class or is not the one that we want
- // to read here in this method. So discard it and continue.
- naclfile.ignore(OSIM_LINEMAX, '\n');
- while (naclfile.peek() == ' ')
- {
- // The main variables are listed at the beginning of a line
- // with additional information indented. If one or more spaces
- // are seen at the beginning of the line upon the entry to this
- // while loop, it is because of that additional information.
- // So, ignore this line since the variable is being ignored.
- naclfile.ignore(OSIM_LINEMAX, '\n');
- }
- }
- }
-}
-
Lng32
NAClusterInfo::getNumActiveCluster()
{
@@ -804,10 +629,18 @@ void NAClusterInfo::setUseAggregationNodesOnly(NABoolean x)
}
Int32
+NAClusterInfo::numOfPhysicalSMPs()
+{
+ if (physicalSMPCount_ < 0)
+ physicalSMPCount_ = computeNumOfSMPs();
+
+ return physicalSMPCount_;
+}
+
+Int32
NAClusterInfo::numOfSMPs()
{
- if(smpCount_ <0)
- smpCount_ = computeNumOfSMPs();
+ CMPASSERT(physicalSMPCount_ > 0);
// This is temporary patch for PARALLEL_NUM_ESPS issue. This CQD should
// be used in many places for costing, NodeMap allocation, synthesizing
@@ -824,11 +657,11 @@ NAClusterInfo::numOfSMPs()
// A value for PARALLEL_NUM_ESPS exists. Use it for the count of cpus
// but don't exceed the number of cpus available in the cluster.
// -------------------------------------------------------------------
- smpCount_ = MINOF(smpCount_,
+ physicalSMPCount_ = MINOF(physicalSMPCount_,
(Int32)(ActiveSchemaDB()->getDefaults().getAsLong(PARALLEL_NUM_ESPS)));
}
- return smpCount_;
+ return physicalSMPCount_;
} // NAClusterInfo::numOfSMPs()
#pragma warn(1506) // warning elimination
@@ -987,7 +820,6 @@ void NAClusterInfo::cleanupPerStatement()
//After every statement activeClusters_ should be NULL
// because statement heap has been cleared already.
activeClusters_ = NULL;
- smpCount_ = -1;
// reset the mebers for versioning support
maxOSV_ = COM_VERS_UNKNOWN;
}
@@ -1175,71 +1007,6 @@ void NAClusterInfoLinux::captureOSInfo(ofstream & nacllinuxfile) const
<< "numCPUcoresPerNode_: " << numCPUcoresPerNode_ << endl;
}
-void NAClusterInfoLinux::simulateNAClusterInfoLinux()
-{
- char var[256];
-
- const char* filepath = CURRCONTEXT_OPTSIMULATOR->getLogFilePath(OptimizerSimulator::NACLUSTERINFO);
-
- ifstream nacllinuxfile(filepath);
-
- if(!nacllinuxfile.good())
- {
- char errMsg[38+OSIM_PATHMAX+1]; // Error msg below + filename + '\0'
- // LCOV_EXCL_START
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.", filepath);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
- // LCOV_EXCL_STOP
- }
-
- while(nacllinuxfile.good())
- {
- // Read the variable name from the file
- nacllinuxfile.getline(var, sizeof(var), ':');
- if(!strcmp(var, "frequency_"))
- {
- nacllinuxfile >> frequency_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- }
- else if (!strcmp(var, "iorate_"))
- {
- nacllinuxfile >> iorate_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- }
- else if (!strcmp(var, "seekTime_"))
- {
- nacllinuxfile >> seekTime_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- }
- else if (!strcmp(var, "pageSize_"))
- {
- nacllinuxfile >> pageSize_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- }
- else if (!strcmp(var, "totalMemoryAvailable_"))
- {
- nacllinuxfile >> totalMemoryAvailable_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- }
- else if (!strcmp(var, "numCPUcoresPerNode_"))
- {
- nacllinuxfile >> numCPUcoresPerNode_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- }
- else
- {
- // This variable either may have been read in simulateNAClusterInfo()
- // method of NAClusterInfo class or is not the one that we want to
- // read here in this method. So discard it.
- nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- while (nacllinuxfile.peek() == ' ')
- {
- // The main variables are listed at the beginning of a line
- // with additional information indented. If one or more spaces
- // are seen at the beginning of the line upon the entry to this
- // while loop, it is because of that additional information.
- // So, ignore this line since the variable is being ignored.
- nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
- }
- }
- }
-}
-
-
Int32 compareTSEs( const void* a, const void* b )
{
// compare function
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/NAClusterInfo.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NAClusterInfo.h b/core/sql/optimizer/NAClusterInfo.h
index 8292841..52ede89 100644
--- a/core/sql/optimizer/NAClusterInfo.h
+++ b/core/sql/optimizer/NAClusterInfo.h
@@ -75,7 +75,7 @@ extern void setUpClusterInfo(CollHeap* heap);
#define MAX_NUM_SMPS_NSK 16 // number of SMPs in the cluster for NSK
#define MAX_NUM_SMPS_SQ 512 // number of CPUs in the cluster for SQ
//<pb>
-
+ULng32 clusterNumHashFunc(const CollIndex& num);
//used to encapsulate dp2 names
class DP2name : public NABasicObject
@@ -179,6 +179,9 @@ public:
virtual size_t totalMemoryAvailable() const = 0;
virtual size_t virtualMemoryAvailable() = 0;
+ // number of physical nodes (from Trafodion monitor or OSIM)
+ Int32 numOfPhysicalSMPs();
+ // this is an adjusted number, based on CQDs
Int32 numOfSMPs();
// This is called by captureNAClusterInfo() to capture the OSIM
@@ -247,10 +250,9 @@ protected :
short localSMP_;
//------------------------------------------------------------------------
- // Earlier smpCount_ was the number of CPUs on a segment. On Linux,
- // smpCount_ is the number of Linux nodes in the cluster.
+ // physical number of SMPs/Linux nodes (real configuration or OSIM config)
//------------------------------------------------------------------------
- Int32 smpCount_;
+ Int32 physicalSMPCount_;
//------------------------------------------------------------------------
// heap_ is where this NAClusterInfo was allocated. This should be the
@@ -309,8 +311,6 @@ public:
float seekTime() const;
Int32 cpuArchitecture() const;
- size_t numLinuxNodes() const { return smpCount_; }
-
//-------------------------------------------------------------------------
// On Linux, numberOfCpusPerSMP() returns the number of Linux nodes in the
// cluster.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/NodeMap.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NodeMap.cpp b/core/sql/optimizer/NodeMap.cpp
index 1c4d5ee..a901948 100644
--- a/core/sql/optimizer/NodeMap.cpp
+++ b/core/sql/optimizer/NodeMap.cpp
@@ -2068,7 +2068,15 @@ short NodeMap::codeGen(const PartitioningFunction *partFunc,
NABoolean
NodeMap::hasRemotePartitions() const
{
- short sysNum = OSIM_MYSYSTEMNUMBER();
+ short sysNum;
+ try {
+ sysNum = OSIM_MYSYSTEMNUMBER();
+ }
+ catch(OsimLogException & e)
+ {
+ OSIM_errorMessage(e.getErrMessage());
+ return FALSE;
+ }
for (ULng32 i = 0; i < getNumEntries(); i++) {
const NodeMapEntry *ne = getNodeMapEntry(i);
@@ -2106,7 +2114,7 @@ void NodeMap::assignScanInfos(HivePartitionAndBucketKey *hiveSearchKey)
NABoolean useLocality = useLocalityForHiveScanInfo();
// distribute <n> files associated the hive scan among numESPs.
HiveFileIterator i;
- HHDFSStatsBase selectedStats;
+ HHDFSStatsBase selectedStats(/* HHDFSTableStats *table */ NULL); // TODO: fix this later
CMPASSERT(type_ = HIVE);
hiveSearchKey->accumulateSelectedStats(selectedStats);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/ObjectNames.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ObjectNames.cpp b/core/sql/optimizer/ObjectNames.cpp
index bff93d4..d309ca6 100644
--- a/core/sql/optimizer/ObjectNames.cpp
+++ b/core/sql/optimizer/ObjectNames.cpp
@@ -356,6 +356,23 @@ const NAString SchemaName::getSchemaNameAsAnsiString() const
return result;
}
+void SchemaName::getHiveSchemaName(
+ NAString &hiveSchemaName // OUT: Hive schema name
+ ) const
+{
+ NAString hiveDefSchema =
+ ActiveSchemaDB()->getDefaults().getValue(HIVE_DEFAULT_SCHEMA);
+ hiveDefSchema.toUpper();
+
+ if (schemaName_ == hiveDefSchema)
+ hiveSchemaName = HiveMetaData::getDefaultSchemaName();
+ else
+ {
+ hiveSchemaName = schemaName_;
+ hiveSchemaName.toLower();
+ }
+}
+
// ODBC SHORTANSI -- the actual MPLoc is encoded in the schName
// using underscore delimiters, i.e. "systemName_volumeName_subvolName".
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/ObjectNames.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ObjectNames.h b/core/sql/optimizer/ObjectNames.h
index dadffe5..ee3b467 100644
--- a/core/sql/optimizer/ObjectNames.h
+++ b/core/sql/optimizer/ObjectNames.h
@@ -188,6 +188,9 @@ public:
const NAString getUnqualifiedSchemaNameAsAnsiString() const
{ return ToAnsiIdentifier(schemaName_); }
+ // Translate Trafodion to Hive schema
+ void getHiveSchemaName(NAString &hiveSchemaName) const;
+
// mutator
void setSchemaName(const NAString &schName) { schemaName_ = schName; }
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/OptPhysRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp
index 392ee8d..2eb90f2 100644
--- a/core/sql/optimizer/OptPhysRelExpr.cpp
+++ b/core/sql/optimizer/OptPhysRelExpr.cpp
@@ -14506,7 +14506,7 @@ PhysicalProperty * FileScan::synthHiveScanPhysicalProperty(
const HHDFSTableStats *tableStats = hiveSearchKey_->getHDFSTableStats();
// stats for partitions/buckets selected by predicates
- HHDFSStatsBase selectedStats;
+ HHDFSStatsBase selectedStats((HHDFSTableStats *)tableStats);
hiveSearchKey_->accumulateSelectedStats(selectedStats);
[2/4] incubator-trafodion git commit: [TRAFODION-2655] Fix two MDAM
optimizer bugs. Update optimizer simulator.
Posted by db...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/OptimizerSimulator.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptimizerSimulator.cpp b/core/sql/optimizer/OptimizerSimulator.cpp
index 6767f84..9709db8 100644
--- a/core/sql/optimizer/OptimizerSimulator.cpp
+++ b/core/sql/optimizer/OptimizerSimulator.cpp
@@ -59,20 +59,91 @@
#include "CmpSeabaseDDL.h"
#include "ExExeUtilCli.h"
#include "ComUser.h"
+#include "HDFSHook.h"
-extern THREAD_P NAClusterInfo *gpClusterInfo;
+#include "Globals.h"
+#include "CmpContext.h"
+#include "Context.h"
+extern THREAD_P NAClusterInfo *gpClusterInfo;
extern const WordAsBits SingleBitArray[];
-//the dir path should start from /bulkload
+// Define PATH_MAX, FILENAME_MAX, LINE_MAX for both NT and NSK.
+// _MAX_PATH and _MAX_FNAME are defined in stdlib.h that is
+// included above.
+#define OSIM_PATHMAX PATH_MAX
+#define OSIM_FNAMEMAX FILENAME_MAX
+#define OSIM_LINEMAX 4096
+#define OSIM_HIVE_TABLE_STATS_DIR "/hive_table_stats"
+//#define HIVE_CRAETE_TABLE_SQL "/hive_create_table.sql"
+//#define HIVE_TABLE_LIST "/hive_table_list.txt"
+//the hdfs dir path should start from /bulkload
#define UNLOAD_HDFS_DIR "/user/trafodion/bulkload/osim_capture"
+//tags for histograms file path
+#define TAG_ALL_HISTOGRAMS "all_histograms"
+#define TAG_HISTOGRAM_ENTRY "histogram_entry"
+#define TAG_FULL_PATH "fullpath"
+#define TAG_USER_NAME "username"
+#define TAG_PID "pid"
+#define TAG_CATALOG "catalog"
+#define TAG_SCHEMA "schema"
+#define TAG_TABLE "table"
+#define TAG_HISTOGRAM "histogram"
+//tags for hive table stats
+#define TAG_HHDFSFILESTATS "hhdfs_file_stats"
+#define TAG_HHDFSBUCKETSTATS "hhdfs_bucket_stats"
+#define TAG_HHDFSLISTPARTSTATS "hhdfs_list_part_stats"
+#define TAG_HHDFSTABLESTATS "hhdfs_table_stats"
+
+//<TAG_ALL_HISTOGRAMS>
+// <TAG_HISTOGRAM_ENTRY>
+// <TAG_FULL_PATH>/opt/home/xx/xxx </TAG_FULL_PATH>
+// <TAG_USER_NAME>root</TAG_USER_NAME>
+// <TAG_PID>12345</TAG_PID>
+// <TAG_CATALOG>trafodion</TAG_CATALOG>
+// <TAG_SCHEMA>seabase</TAG_SCHEMA>
+// <TAG_TABLE>order042</TAG_TABLE>
+// <TAG_HISTOGRAM>sb_histogram_interval</TAG_HISTOGRAM>
+// </TAG_HISTOGRAM_ENTRY>
+// ...
+//</TAG_ALL_HISTOGRAMS>
+const char OsimAllHistograms::elemName[] = TAG_ALL_HISTOGRAMS;
+const char OsimHistogramEntry::elemName[] = TAG_HISTOGRAM_ENTRY;
+const char OsimHHDFSFileStats::elemName[] = TAG_HHDFSFILESTATS;
+const char OsimHHDFSBucketStats::elemName[] = TAG_HHDFSBUCKETSTATS;
+const char OsimHHDFSListPartitionStats::elemName[] = TAG_HHDFSLISTPARTSTATS;
+const char OsimHHDFSTableStats::elemName[] = TAG_HHDFSTABLESTATS;
+
+const char* OptimizerSimulator::logFileNames_[NUM_OF_LOGS]= {
+ "ESTIMATED_ROWS.txt" ,
+ "NODE_AND_CLUSTER_NUMBERS.txt",
+ "NAClusterInfo.txt",
+ "MYSYSTEMNUMBER.txt",
+ "VIEWS.txt" ,
+ "VIEWDDLS.txt",
+ "TABLES.txt",
+ "CREATE_SCHEMA_DDLS.txt",
+ "CREATE_TABLE_DDLS.txt" ,
+ "SYNONYMS.txt",
+ "SYNONYMDDLS.txt",
+ "CQDS.txt" ,
+ "QUERIES.txt",
+ "VERSIONS.txt",
+ "CaptureSysType.txt",
+ "HISTOGRAM_PATHS.xml",
+ "HIVE_HISTOGRAM_PATHS.xml",
+ "HIVE_CREATE_TABLE.sql",
+ "HIVE_CREATE_EXTERNAL_TABLE.sql",
+ "HIVE_TABLE_LIST.txt",
+ "HHDFS_MASTER_HOST_LIST.txt"
+};
-static ULng32 hashFunc_int(const Int32& Int)
+static ULng32 intHashFunc(const Int32& Int)
{
return (ULng32)Int;
}
-static NABoolean fileExists(const char *filename, NABoolean & isDir)
+static NABoolean isFileExists(const char *filename, NABoolean & isDir)
{
struct stat sb;
Int32 rVal = stat(filename, &sb);
@@ -82,10 +153,6 @@ static NABoolean fileExists(const char *filename, NABoolean & isDir)
return rVal != -1;
}
-
-const char OsimAllHistograms::elemName[] = TAG_ALL_HISTOGRAMS;
-const char OsimHistogramEntry::elemName[] = TAG_HISTOGRAM_ENTRY;
-
void OsimAllHistograms::startElement(void * parser, const char * elementName, const char * * atts)
{
OsimHistogramEntry* entry = NULL;
@@ -95,7 +162,7 @@ void OsimAllHistograms::startElement(void * parser, const char * elementName, co
list_.insert(entry);
}
else
- OsimLogException("Errors Parsing hitograms file.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Parsing hitograms file.");
}
void OsimAllHistograms::serializeBody(XMLString& xml)
@@ -161,7 +228,6 @@ void OsimHistogramEntry::endElement(void * parser, const char * elementName)
currentTag_="";
}
-
void OsimHistogramEntry::serializeBody(XMLString& xml)
{
xml.append(" ");
@@ -208,7 +274,6 @@ void OsimHistogramEntry::serializeBody(XMLString& xml)
}
-
XMLElementPtr OsimElementMapper::operator()(void *parser,
char *elementName,
AttributeList atts)
@@ -217,32 +282,10 @@ XMLElementPtr OsimElementMapper::operator()(void *parser,
//atts is not used here
if (!strcmp( elementName, "all_histograms"))
elemPtr = new (XMLPARSEHEAP) OsimAllHistograms(XMLPARSEHEAP);
-
return elemPtr;
}
/////////////////////////////////////////////////////////////////////////
-
-
-const char* OptimizerSimulator::sysCallLogFileName_[NUM_OF_SYSCALLS]= {
- "ESTIMATED_ROWS.txt" ,
- "NODE_AND_CLUSTER_NUMBERS.txt",
- "NAClusterInfo.txt",
- "MYSYSTEMNUMBER.txt",
- "VIEWS.txt" ,
- "VIEWDDLS.txt",
- "TABLES.txt",
- "CREATE_SCHEMA_DDLS.txt",
- "CREATE_TABLE_DDLS.txt" ,
- "SYNONYMS.txt",
- "SYNONYMDDLS.txt",
- "CQDS.txt" ,
- "QUERIES.txt",
- "VERSIONS.txt",
- "captureSysType.txt",
- "HISTOGRAM_PATHS.xml"
-};
-
OptimizerSimulator::OptimizerSimulator(CollHeap *heap)
:osimLogLocalDir_(heap),
osimMode_(OptimizerSimulator::OFF),
@@ -250,8 +293,7 @@ OptimizerSimulator::OptimizerSimulator(CollHeap *heap)
hashDict_Views_(NULL),
hashDict_Tables_(NULL),
hashDict_Synonyms_(NULL),
- hashDict_TablesBeforeAction_(NULL),
- hashDict_ViewsBeforeAction_(NULL),
+ hashDict_HiveTables_(NULL),
nodeNum_(-1),
clusterNum_(-1),
captureSysType_(OSIM_LINUX),
@@ -270,10 +312,10 @@ OptimizerSimulator::OptimizerSimulator(CollHeap *heap)
forceLoad_(FALSE),
heap_(heap)
{
- for (sysCall sc=FIRST_SYSCALL; sc<NUM_OF_SYSCALLS; sc = sysCall(sc+1))
+ for (OsimLog sc=FIRST_LOG; sc<NUM_OF_LOGS; sc = OsimLog(sc+1))
{
- sysCallLogFilePath_[sc]=NULL;
- writeSysCallStream_[sc]=NULL;
+ logFilePaths_[sc]=NULL;
+ writeLogStreams_[sc]=NULL;
}
}
@@ -300,7 +342,6 @@ void OSIM_warningMessage(const char *errMsg)
void OptimizerSimulator::warningMessage(const char *errMsg)
{
- // WARNING message
*CmpCommon::diags() << DgSqlCode(OSIM_ERRORORWARNING)
<< DgString0(errMsg);
}
@@ -322,23 +363,60 @@ void OptimizerSimulator::dumpVersions()
{
//dump version info
NAString cmd = "sqvers -u > ";
- cmd += sysCallLogFilePath_[VERSIONSFILE];
+ cmd += logFilePaths_[VERSIONSFILE];
system(cmd.data()); //dump versions
}
+void OptimizerSimulator::dumpHHDFSMasterHostList()
+{
+ (*writeLogStreams_[HHDFS_MASTER_HOST_LIST])
+ << "HasVirtualSQNodes"
+ << " "
+ << HHDFSMasterHostList::hasVirtualSQNodes() << endl;
+ (*writeLogStreams_[HHDFS_MASTER_HOST_LIST])
+ << "NumSQNodes"
+ << " "
+ << HHDFSMasterHostList::getNumSQNodes() << endl;
+ NAString hostNameList;
+ for(Int32 i = 0; i < HHDFSMasterHostList::entries(); i++)
+ {
+ hostNameList += HHDFSMasterHostList::getHostName(i);
+ hostNameList += '|'; //delimiter
+ }
+ if(hostNameList.length() > 0)
+ {
+ //if there's any hostname,
+ //hostNameList would be like "node0|node1|node2|",
+ //this is to replace '|'.
+ hostNameList[hostNameList.length()-1] = '\n';
+ (*writeLogStreams_[HHDFS_MASTER_HOST_LIST]) << hostNameList.data();
+ }
+}
+
+void raiseOsimException(const char* fmt, ...)
+{
+ char * buffer;
+ va_list args ;
+ va_start(args, fmt);
+ buffer = NAString::buildBuffer(fmt, args);
+ va_end(args);
+ //throw anyway null buffer will be handled inside constructor of
+ //OsimLogException, empty string will be issued.
+ OsimLogException(buffer, __FILE__, __LINE__).throwException();
+}
+
NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const char * localDir)
{
try{
-
if(targetMode == UNLOAD)
{
- setOsimMode(targetMode);
- setOsimLogdir(localDir);
- initLogFilePaths();
- setOsimMode(OFF);
- dropObjects();
- cleanup();
- return TRUE;
+ setOsimMode(targetMode);
+ setOsimLogdir(localDir);
+ initLogFilePaths();
+ setOsimMode(OFF);
+ dropObjects();
+ cleanup();
+ return TRUE;
}
switch(osimMode_)
@@ -353,10 +431,6 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
createLogDir();
initHashDictionaries();
initLogFilePaths();
- //record all qualified table names before running query,
- //except meta tables and histogram tables in any schema
- saveTablesBeforeStart();
- saveViewsBeforeStart();
setClusterInfoInitialized(TRUE);
break;
case LOAD: //OFF --> LOAD
@@ -364,10 +438,10 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
setOsimLogdir(localDir);
initHashDictionaries();
initLogFilePaths();
- saveTablesBeforeStart();
- saveViewsBeforeStart();
loadDDLs();
- loadHistograms();
+ loadHistograms(logFilePaths_[HISTOGRAM_PATHS], FALSE);
+ loadHiveDDLs();
+ loadHistograms(logFilePaths_[HIVE_HISTOGRAM_PATHS], TRUE);
break;
case SIMULATE: //OFF-->SIMU
setOsimMode(targetMode);
@@ -385,14 +459,21 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
break;
case CAPTURE:
if(targetMode == OFF) //CAPURE --> OFF only
- {
+ {
+ //call dumpHHDFSMasterHostList() first
+ //otherwise context-switch will reset
+ //HHDFSMasterHostList::hasVirtualSQNodes_
+ //HHDFSMasterHostList::numSQNodes_
+ dumpHHDFSMasterHostList();
+ dumpHiveTableDDLs();
+ dumpHiveHistograms();
dumpHistograms();
dumpVersions();
setOsimMode(targetMode);
cleanup();//NOTE: osimMode_ is set OFF in cleanup()
}
else
- errorMessage("Mode transition is not allowed.");
+ warningMessage("Mode transition is not allowed.");
break;
case LOAD:
if(targetMode == SIMULATE)//LOAD --> SIMU only
@@ -405,16 +486,17 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
setClusterInfoInitialized(TRUE);
}
else
- errorMessage("Mode transition rather than LOAD to SIMULATE is not allowed.");
+ warningMessage("Mode transition other than LOAD to SIMULATE is not allowed.");
break;
default :
- errorMessage("Mode transition is not allowed.");
+ warningMessage("Mode transition is not allowed.");
break;
}
}
catch(OsimLogException & e)
{
cleanup();
+ //move err string from exception object to diagnostic area
errorMessage(e.getErrMessage());
return FALSE;
}
@@ -439,39 +521,53 @@ void OptimizerSimulator::dumpDDLs(const QualifiedName & qualifiedName)
retcode = fetchAllRowsFromMetaContext(outQueue, query.data());
if (retcode < 0 || retcode == 100/*rows not found*/) {
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- OsimLogException("Errors Dumping Table DDL.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Dumping Table DDL.");
}
- outQueue->position();
-
- ofstream * createSchema = writeSysCallStream_[CREATE_SCHEMA_DDLS];
- ofstream * createTable = writeSysCallStream_[CREATE_TABLE_DDLS];
+ if(outQueue)
+ {
+ ofstream * createSchema = writeLogStreams_[CREATE_SCHEMA_DDLS];
+
+ ofstream * createTable = writeLogStreams_[CREATE_TABLE_DDLS];
+
+ //Dump a "create schema ..." to schema ddl file for every table.
- //Dump a "create schema ..." to schema ddl file for every table.
+ //This comment line will be printed during loading, ';' must be omitted
+ (*createSchema) << "--" <<"CREATE SCHEMA IF NOT EXISTS "
+ << qualifiedName.getCatalogName()
+ << "." << qualifiedName.getSchemaName() <<endl;
- //This comment line will be printed during loading, ';' must be omitted
- (*createSchema) << "--" <<"CREATE SCHEMA IF NOT EXISTS "
- << qualifiedName.getCatalogName()
- << "." << qualifiedName.getSchemaName() <<endl;
-
- (*createSchema) << "CREATE SCHEMA IF NOT EXISTS "
- << qualifiedName.getCatalogName()
- <<"."<< qualifiedName.getSchemaName()
- << ";" << endl;
-
- for (int i = 0; i < outQueue->numEntries(); i++) {
- OutputInfo * vi = (OutputInfo*)outQueue->getNext();
- char * ptr = vi->get(0);
- //skip heading newline, and add a comment line
- Int32 ix = 0;
- for(; ptr[ix]=='\n'; ix++);
- if( strstr(ptr, "CREATE TABLE") ||
- strstr(ptr, "CREATE INDEX") ||
- strstr(ptr, "CREATE UNIQUE INDEX") ||
- strstr(ptr, "ALTER TABLE") )
- (*createTable) << "--" << ptr+ix << endl;
- //output ddl
- (*createTable) << ptr << endl;
+ (*createSchema) << "CREATE SCHEMA IF NOT EXISTS "
+ << qualifiedName.getCatalogName()
+ <<"."<< qualifiedName.getSchemaName()
+ << ";" << endl;
+
+ outQueue->position();//rewind
+ for (int i = 0; i < outQueue->numEntries(); i++) {
+ OutputInfo * vi = (OutputInfo*)outQueue->getNext();
+ char * ptr = vi->get(0);
+ // skip heading newline, and add a comment line
+ // for the DDL text upto the first trailing '\n'
+ Int32 ix = 0;
+ for(; ptr[ix]=='\n'; ix++);
+ if( strstr(ptr, "CREATE TABLE") ||
+ strstr(ptr, "CREATE INDEX") ||
+ strstr(ptr, "CREATE UNIQUE INDEX") ||
+ strstr(ptr, "ALTER TABLE") )
+
+ {
+ (*createTable) << "--";
+ char* x = ptr+ix;
+ while ( (*x) && *x != '\n' ) {
+ (*createTable) << *x;
+ x++;
+ }
+ (*createTable) << endl;
+ }
+
+ //output ddl
+ (*createTable) << ptr << endl;
+ }
}
}
@@ -489,9 +585,18 @@ void OptimizerSimulator::dumpHistograms()
//enumerate captured table names and tableUIDs in hash table
for(iterator.getNext(name, tableUID); name && tableUID; iterator.getNext(name, tableUID))
{
-
+ //check if this table_uid is in TRAFODION."_HIVESTATS_".SB_HISTOGRAMS,
+ //if not, we consider this table has no histogram data.
+ Queue * outQueue = NULL;
+ query = "SELECT TABLE_UID FROM TRAFODION.";
+ query += name->getSchemaName();
+ query += ".SB_HISTOGRAMS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+ retcode = fetchAllRowsFromMetaContext(outQueue, query.data());
+ if(retcode < 0 || outQueue && outQueue->entries() == 0)
+ continue;
+
debugMessage("Dumping histograms for %s\n", name->getQualifiedNameAsAnsiString().data());
-
//dump histograms data to hdfs
query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
query += "'"UNLOAD_HDFS_DIR"/";
@@ -523,7 +628,14 @@ void OptimizerSimulator::dumpHistograms()
" FROM ";
query += name->getCatalogName();
query += ".";
- query += name->getSchemaName();
+ if(name->getSchemaName()[0]=='_')
+ {
+ query += "\"";
+ query += name->getSchemaName();
+ query += "\"";
+ }
+ else
+ query += name->getSchemaName();
query += ".SB_HISTOGRAMS WHERE TABLE_UID = ";
query += std::to_string((long long)(*tableUID)).c_str();
@@ -553,10 +665,7 @@ void OptimizerSimulator::dumpHistograms()
else if(retcode < 0 && -4082 != retcode)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg;
- errMsg = "Unload histogram data error: ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Unload histogram data error: %d", retcode);
}
query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
@@ -580,7 +689,14 @@ void OptimizerSimulator::dumpHistograms()
" FROM ";
query += name->getCatalogName();
query += ".";
- query += name->getSchemaName();
+ if(name->getSchemaName()[0]=='_')
+ {
+ query += "\"";
+ query += name->getSchemaName();
+ query += "\"";
+ }
+ else
+ query += name->getSchemaName();
query += ".SB_HISTOGRAM_INTERVALS WHERE TABLE_UID = ";
query += std::to_string((long long)(*tableUID)).c_str();
@@ -610,18 +726,161 @@ void OptimizerSimulator::dumpHistograms()
else if(retcode < 0 && -4082 != retcode)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg;
- errMsg = "Unload histogram data error: ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Unload histogram data error: %d", retcode);
+ }
+
+ }
+
+ //Do not use XMLFormatString as we do format ourself
+ XMLString* xmltext = new (STMTHEAP) XMLString(STMTHEAP);
+ histoInfoList->toXML(*xmltext);
+ (*writeLogStreams_[HISTOGRAM_PATHS]) << xmltext->data() << endl;
+ NADELETE(xmltext, XMLString, STMTHEAP);
+ //copy histograms data from hdfs to osim directory.
+ histogramHDFSToLocal();
+}
+
+void OptimizerSimulator::dumpHiveHistograms()
+{
+ short retcode;
+ const QualifiedName* name = NULL;
+ Int64* tableUID = NULL;
+ NAString query(STMTHEAP);
+
+ NAHashDictionaryIterator<const QualifiedName, Int64> iterator(*hashDict_HiveTables_);
+
+ OsimAllHistograms* histoInfoList = new (STMTHEAP) OsimAllHistograms(STMTHEAP);
+ NAString fullPath(STMTHEAP);
+ //enumerate captured table names and tableUIDs in hash table
+ for(iterator.getNext(name, tableUID); name && tableUID; iterator.getNext(name, tableUID))
+ {
+ //check if this table_uid is in TRAFODION."_HIVESTATS_".SB_HISTOGRAMS,
+ //if not, we consider this table has no histogram data.
+ Queue * outQueue = NULL;
+ query = "SELECT TABLE_UID FROM TRAFODION.\"_HIVESTATS_\".SB_HISTOGRAMS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+ retcode = fetchAllRowsFromMetaContext(outQueue, query.data());
+ if(retcode < 0 || outQueue && outQueue->entries() == 0)
+ continue;
+
+ debugMessage("Dumping histograms for %s\n", name->getQualifiedNameAsAnsiString().data());
+
+ //dump histograms data to hdfs
+ query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
+ query += "'"UNLOAD_HDFS_DIR"/";
+ query += ComUser::getCurrentUsername();
+ query += "/";
+ query += std::to_string((long long unsigned int)(getpid())).c_str();
+ query += "/";
+ query += name->getQualifiedNameAsAnsiString();
+ query += ".SB_HISTOGRAMS'";
+ query += " SELECT TABLE_UID"
+ ", HISTOGRAM_ID"
+ ", COL_POSITION"
+ ", COLUMN_NUMBER"
+ ", COLCOUNT"
+ ", INTERVAL_COUNT"
+ ", ROWCOUNT"
+ ", TOTAL_UEC"
+ ", STATS_TIME"
+ ", TRANSLATE(LOW_VALUE USING UCS2TOUTF8)"
+ ", TRANSLATE(HIGH_VALUE USING UCS2TOUTF8)"
+ ", READ_TIME"
+ ", READ_COUNT"
+ ", SAMPLE_SECS"
+ ", COL_SECS"
+ ", SAMPLE_PERCENT"
+ ", CV,REASON, V1, V2, V3, V4"
+ ", TRANSLATE(V5 USING UCS2TOUTF8)"
+ ", TRANSLATE(V6 USING UCS2TOUTF8)"
+ " FROM TRAFODION.\"_HIVESTATS_\".SB_HISTOGRAMS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+
+ retcode = executeFromMetaContext(query.data());
+ //succeed
+ if(retcode >= 0)
+ {
+ fullPath = osimLogLocalDir_;
+ fullPath += "/";
+ fullPath += ComUser::getCurrentUsername();
+ fullPath += "/";
+ fullPath += std::to_string((long long unsigned int)(getpid())).c_str();
+ fullPath += "/";
+ fullPath += name->getQualifiedNameAsAnsiString();
+ fullPath += ".SB_HISTOGRAMS";
+ histoInfoList->addEntry( fullPath.data(),
+ ComUser::getCurrentUsername(),
+ std::to_string((long long unsigned int)(getpid())).c_str(),
+ name->getCatalogName().data(),
+ name->getSchemaName().data(),
+ name->getObjectName().data(),
+ "SB_HISTOGRAMS");
+ }
+ //ignore -4082,
+ //which means histogram tables are not exist,
+ //i.e. update stats hasn't been done for any table.
+ else if(retcode < 0 && -4082 != retcode)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("Unload histogram data error: %d", retcode);
+ }
+
+ query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
+ query += "'"UNLOAD_HDFS_DIR"/";
+ query += ComUser::getCurrentUsername();
+ query += "/";
+ query += std::to_string((long long unsigned int)(getpid())).c_str();
+ query += "/";
+ query += name->getQualifiedNameAsAnsiString();
+ query += ".SB_HISTOGRAM_INTERVALS'";
+ query += " SELECT TABLE_UID"
+ ", HISTOGRAM_ID"
+ ", INTERVAL_NUMBER"
+ ", INTERVAL_ROWCOUNT"
+ ", INTERVAL_UEC"
+ ", TRANSLATE(INTERVAL_BOUNDARY USING UCS2TOUTF8)"
+ ", STD_DEV_OF_FREQ"
+ ", V1, V2, V3, V4"
+ ", TRANSLATE(V5 USING UCS2TOUTF8)"
+ ", TRANSLATE(V6 USING UCS2TOUTF8)"
+ " FROM TRAFODION.\"_HIVESTATS_\".SB_HISTOGRAM_INTERVALS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+
+ retcode = executeFromMetaContext(query.data());
+
+ if(retcode >= 0)
+ {
+ fullPath = osimLogLocalDir_;
+ fullPath += "/";
+ fullPath += ComUser::getCurrentUsername();
+ fullPath += "/";
+ fullPath += std::to_string((long long unsigned int)(getpid())).c_str();
+ fullPath += "/";
+ fullPath += name->getQualifiedNameAsAnsiString();
+ fullPath += ".SB_HISTOGRAM_INTERVALS";
+ histoInfoList->addEntry( fullPath.data(),
+ ComUser::getCurrentUsername(),
+ std::to_string((long long unsigned int)(getpid())).c_str(),
+ name->getCatalogName().data(),
+ name->getSchemaName().data(),
+ name->getObjectName().data(),
+ "SB_HISTOGRAM_INTERVALS");
+ }
+ //ignore -4082,
+ //which means histogram tables are not exist,
+ //i.e. update stats hasn't been done for any table.
+ else if(retcode < 0 && -4082 != retcode)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("Unload histogram data error: %d", retcode);
}
}
- //Do not user XMLFormatString as we do format ourself
+ //Do not use XMLFormatString as we do format ourself
XMLString* xmltext = new (STMTHEAP) XMLString(STMTHEAP);
histoInfoList->toXML(*xmltext);
- (*writeSysCallStream_[HISTOGRAM_PATHS]) << xmltext->data() << endl;
+ (*writeLogStreams_[HIVE_HISTOGRAM_PATHS]) << xmltext->data() << endl;
NADELETE(xmltext, XMLString, STMTHEAP);
//copy histograms data from hdfs to osim directory.
histogramHDFSToLocal();
@@ -630,16 +889,10 @@ void OptimizerSimulator::dumpHistograms()
void OptimizerSimulator::dropObjects()
{
short retcode;
- ifstream tables(sysCallLogFilePath_[TABLESFILE]);
+ ifstream tables(logFilePaths_[TABLESFILE]);
if(!tables.good())
- {
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_TABLE_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- //ignore first 2 lines
- tables.ignore(OSIM_LINEMAX, '\n');
- tables.ignore(OSIM_LINEMAX, '\n');
+ raiseOsimException("Error open %s", logFilePaths_[TABLESFILE]);
+
std::string stdQualTblNm;//get qualified table name from file
NAString query(STMTHEAP);
while(tables.good())
@@ -654,200 +907,56 @@ void OptimizerSimulator::dropObjects()
query = "DROP TABLE IF EXISTS ";
query += stdQualTblNm.c_str();
query += " CASCADE;";
- debugMessage("DELETING %s ...\n", stdQualTblNm.c_str());
+ debugMessage("%s\n", query.data());
retcode = executeFromMetaContext(query.data());
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Drop Table " ;
- errMsg += stdQualTblNm.c_str();
- errMsg += " Error: ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- }
-}
-
-void OptimizerSimulator::checkDuplicateNames()
-{
- //Get captured qualied table names
- //and compare each with the names in hash dictionary.
- ifstream tables(sysCallLogFilePath_[TABLESFILE]);
- if(!tables.good())
- {
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_TABLE_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- //ignore first 2 lines
- tables.ignore(OSIM_LINEMAX, '\n');
- tables.ignore(OSIM_LINEMAX, '\n');
- std::string stdQualTblNm;
- NAString naQualTblNm(STMTHEAP);
- while(tables.good())
- {
- //get one qualified table name from file
- std::getline(tables, stdQualTblNm);
-
- // Exit the loop if there was no data to read.
- // eofbit is not set until an attempt is made to read beyond EOF.
- if(!tables.good())
- break;
-
- //Check if table name is in existance
- naQualTblNm = stdQualTblNm.c_str();
- if(hashDict_TablesBeforeAction_->contains(&naQualTblNm))
- {
- NAString errMsg = "Object " + naQualTblNm + " already exists.";
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Drop Table %s error: %d", stdQualTblNm.c_str(), retcode);
}
}
-}
-
-void OptimizerSimulator::saveViewsBeforeStart()
-{
- if(viewsBeforeActionInitilized_)
- return;
- //Ask _MD_.OBJECTS for a list of all qualified view names.
- initializeCLI();
-
- short retcode;
- if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
- }
-
- char * ptr = NULL;
- Lng32 len = 0;
-
- retcode = cliInterface_->fetchRowsPrologue("select catalog_name, schema_name, object_name "
- " from TRAFODION.\""SEABASE_MD_SCHEMA"\"."SEABASE_OBJECTS
- " where object_type = 'VI' "
- "and schema_name <> '_MD_' "
- "and schema_name <> '_REPOS_' "
- "and schema_name <> '_PRIVMGR_MD_' "
- "and object_name <> 'SB_HISTOGRAMS' "
- "and object_name <> 'SB_HISTOGRAM_INTERVALS'; ");
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing views, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
-
- while(1)
- {
- retcode = cliInterface_->fetch();
-
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing views, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- if (retcode == 100) //no more data
- break;
-
- NAString * qualifiedName = new (heap_) NAString(heap_);
- Int32 * dummy = new Int32(0);
- //append catalog name
- cliInterface_->getPtrAndLen(1,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append schema name
- cliInterface_->getPtrAndLen(2,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append table name
- cliInterface_->getPtrAndLen(3,ptr,len);
- qualifiedName->append(ptr, len);
-
- hashDict_ViewsBeforeAction_->insert(qualifiedName, dummy);
-
- }
- //end fetch
- retcode = cliInterface_->fetchRowsEpilogue(NULL);
-
- cmpSBD_->switchBackCompiler();
-
- viewsBeforeActionInitilized_ = TRUE;
-}
-
-void OptimizerSimulator::saveTablesBeforeStart()
-{
- if(tablesBeforeActionInitilized_)
- return;
- //Ask _MD_.OBJECTS for a list of all qualified table names.
- initializeCLI();
-
- short retcode;
- if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
- }
-
- char * ptr = NULL;
- Lng32 len = 0;
-
- retcode = cliInterface_->fetchRowsPrologue("select catalog_name, schema_name, object_name "
- " from TRAFODION.\""SEABASE_MD_SCHEMA"\"."SEABASE_OBJECTS
- " where object_type = 'BT' "
- "and schema_name <> '_MD_' "
- "and schema_name <> '_REPOS_' "
- "and schema_name <> '_PRIVMGR_MD_' "
- "and object_name <> 'SB_HISTOGRAMS' "
- "and object_name <> 'SB_HISTOGRAM_INTERVALS'; ");
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing tables, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
-
- while(1)
+ std::string str;
+ Int64 uid;
+ const QualifiedName * qualName = NULL;
+ Int64 * tableUID;
+ NAString trafName;
+ std::ifstream hiveTableListFile(logFilePaths_[HIVE_TABLE_LIST]);
+ //we only need one loop, no need to populate hashDict_HiveTables_
+ while(hiveTableListFile.good())
{
- retcode = cliInterface_->fetch();
-
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing tables, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- if (retcode == 100) //no more data
+ // read tableName and uid from the file
+ hiveTableListFile >> str >> uid;
+ // eofbit is not set until an attempt is made to read beyond EOF.
+ // Exit the loop if there was no data to read above.
+ if(!hiveTableListFile.good())
break;
-
- NAString * qualifiedName = new (heap_) NAString(heap_);
- Int32 * dummy = new Int32(0);
- //append catalog name
- cliInterface_->getPtrAndLen(1,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append schema name
- cliInterface_->getPtrAndLen(2,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append table name
- cliInterface_->getPtrAndLen(3,ptr,len);
- qualifiedName->append(ptr, len);
-
- hashDict_TablesBeforeAction_->insert(qualifiedName, dummy);
-
+ NAString name = str.c_str();
+ qualName = new (heap_) QualifiedName(name,3);
+ trafName = ComConvertNativeNameToTrafName(qualName->getCatalogName(),
+ qualName->getSchemaName(),
+ qualName->getObjectName());
+ QualifiedName qualTrafName(trafName,3);
+ //drop external table
+ NAString dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += trafName;
+ debugMessage("%s\n", dropStmt.data());
+ retcode = executeFromMetaContext(dropStmt.data());
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("drop external table: %d", retcode);
+ }
+ //drop hive table
+ NAString hiveSchemaName;
+ qualName->getHiveSchemaName(hiveSchemaName);
+ dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += hiveSchemaName;
+ dropStmt += '.';
+ dropStmt += qualName->getObjectName();
+ debugMessage("%s\n", dropStmt.data());
+ execHiveSQL(dropStmt.data());//drop hive table
}
-
- //end fetch
- retcode = cliInterface_->fetchRowsEpilogue(NULL);
-
- cmpSBD_->switchBackCompiler();
-
- tablesBeforeActionInitilized_ = TRUE;
}
void OptimizerSimulator::loadDDLs()
@@ -857,29 +966,25 @@ void OptimizerSimulator::loadDDLs()
//If force option is present,
//drop tables with same names, otherwise rollback
- if(isForceLoad())
- dropObjects();
- else
- checkDuplicateNames();
-
+ //if(isForceLoad())
+ // dropObjects();
+ //else
+ // checkDuplicateNames();
+ dropObjects();
+
NAString statement(STMTHEAP);
NAString comment(STMTHEAP);
statement.capacity(4096);
comment.capacity(4096);
- //Step 1 :
+ //Step 1:
//Fetch and execute "create schema ..." from schema ddl file.
debugMessage("Step 1 Create Schemas:\n");
- ifstream createSchemas(sysCallLogFilePath_[CREATE_SCHEMA_DDLS]);
+ ifstream createSchemas(logFilePaths_[CREATE_SCHEMA_DDLS]);
if(!createSchemas.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_SCHEMA_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[CREATE_SCHEMA_DDLS]);
}
- //ignore first 2 lines
- createSchemas.ignore(OSIM_LINEMAX, '\n');
- createSchemas.ignore(OSIM_LINEMAX, '\n');
while(readStmt(createSchemas, statement, comment))
{
if(comment.length() > 0)
@@ -892,16 +997,11 @@ void OptimizerSimulator::loadDDLs()
//Step 2:
//Fetch and execute "create table ... " from table ddl file.
debugMessage("Step 2 Create Tables:\n");
- ifstream createTables(sysCallLogFilePath_[CREATE_TABLE_DDLS]);
+ ifstream createTables(logFilePaths_[CREATE_TABLE_DDLS]);
if(!createTables.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_TABLE_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[CREATE_TABLE_DDLS]);
}
- //ignore first 2 lines
- createTables.ignore(OSIM_LINEMAX, '\n');
- createTables.ignore(OSIM_LINEMAX, '\n');
while(readStmt(createTables, statement, comment))
{
if(comment.length() > 0)
@@ -911,26 +1011,20 @@ void OptimizerSimulator::loadDDLs()
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Create Table Error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Create Table Error: %d", retcode);
}
}
}
//Step 3:
//Fetch and execute "create view ..." from view ddl file.
- debugMessage("Step 3 Create Views:");
- ifstream createViews(sysCallLogFilePath_[VIEWDDLS]);
+ debugMessage("Step 3 Create Views:\n");
+ ifstream createViews(logFilePaths_[VIEWDDLS]);
if(!createViews.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[VIEWDDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[VIEWDDLS]);
}
- //ignore first 2 lines
- createViews.ignore(OSIM_LINEMAX, '\n');
- createViews.ignore(OSIM_LINEMAX, '\n');
+
while(readStmt(createViews, statement, comment))
{
if(comment.length() > 0)
@@ -940,16 +1034,377 @@ void OptimizerSimulator::loadDDLs()
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Create View Error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- errMsg += statement;
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Create View Error: %d %s", retcode, statement.data());
}
}
}
}
-NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashDictionary<NAString, QualifiedName> * modifiedPathList)
+static const char* extractAsComment(const char* header, const NAString & stmt)
+{
+ NAString tmp;
+ int begin = stmt.index(header);
+ if(begin > -1)
+ {
+ int end = stmt.index('\n', begin);
+ if(end > begin)
+ {
+ stmt.extract(begin, end-1, tmp);
+ return tmp.data();
+ }
+ }
+ return NULL;
+}
+
+void OptimizerSimulator::loadHiveDDLs()
+{
+ debugMessage("creating hive tables ...\n");
+ short retcode;
+ NAString statement(STMTHEAP);
+ NAString comment(STMTHEAP);
+ statement.capacity(4096);
+ comment.capacity(4096);
+ std::ifstream hiveCreateTableSql(logFilePaths_[HIVE_CREATE_TABLE]);
+ std::ifstream hiveTableListFile(logFilePaths_[HIVE_TABLE_LIST]);
+ std::ifstream hiveCreateExternalTableSql(logFilePaths_[HIVE_CREATE_EXTERNAL_TABLE]);
+ if(!hiveTableListFile.good() || !hiveCreateTableSql.good())
+ return;
+
+ //read hive sql file and create hive table
+ std::string str;
+ Int64 uid;
+ const QualifiedName * qualName = NULL;
+ Int64 * tableUID;
+ int counter = 0;
+ NAString trafName;
+
+ while(hiveTableListFile.good())
+ {
+ // read tableName and uid from the file
+ hiveTableListFile >> str >> uid;
+ // eofbit is not set until an attempt is made to read beyond EOF.
+ // Exit the loop if there was no data to read above.
+ if(!hiveTableListFile.good())
+ break;
+ NAString name = str.c_str();
+ qualName = new (heap_) QualifiedName(name,3);
+ tableUID = new Int64(uid);
+ hashDict_HiveTables_->insert(qualName, tableUID);
+ }
+
+ NAHashDictionaryIterator<const QualifiedName, Int64> iterator(*hashDict_HiveTables_);
+ //create hive schema and trafodion external schema and
+ //drop external tables and hive tables with same names
+ for(iterator.getNext(qualName, tableUID); qualName && tableUID; iterator.getNext(qualName, tableUID))
+ {
+ trafName = ComConvertNativeNameToTrafName(
+ qualName->getCatalogName(),
+ qualName->getSchemaName(),
+ qualName->getObjectName());
+
+ QualifiedName qualTrafName(trafName,3);
+ //create external table schema
+ NAString create_ext_schema = "CREATE SCHEMA IF NOT EXISTS ";
+ create_ext_schema += qualTrafName.getCatalogName();
+ create_ext_schema += ".\"";
+ create_ext_schema += qualTrafName.getSchemaName();
+ create_ext_schema += "\" AUTHORIZATION DB__ROOT";
+ retcode = executeFromMetaContext(create_ext_schema);
+ if(retcode < 0) {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("create hive external schema: %d %s", retcode, statement.data());
+ }
+ //drop external table
+ NAString dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += trafName;
+ retcode = executeFromMetaContext(dropStmt.data());
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("drop external table: %d", retcode);
+ }
+
+ //create hive schema
+ NAString hiveSchemaName;
+ qualName->getHiveSchemaName(hiveSchemaName);
+ NAString create_hive_schema = "CREATE SCHEMA IF NOT EXISTS ";
+ create_hive_schema += hiveSchemaName;
+ execHiveSQL(create_hive_schema.data());
+ //drop hive table
+ dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += hiveSchemaName;
+ dropStmt += '.';
+ dropStmt += qualName->getObjectName();
+ execHiveSQL(dropStmt.data());//drop hive table
+ }
+ //create hive table
+ debugMessage("Begin creating hive tables\n");
+
+ while(readHiveStmt(hiveCreateTableSql, statement, comment))
+ {
+ if(statement.length() > 0)
+ {
+ debugMessage("%s\n", extractAsComment("CREATE TABLE", statement));
+ execHiveSQL(statement.data());//create hive table
+ debugMessage("done\n");
+ }
+ }
+
+ debugMessage("Begin creating hive external tables\n");
+
+ //create external table
+ while(readHiveStmt(hiveCreateExternalTableSql, statement, comment))
+ {
+ if(statement.length() > 0) {
+ debugMessage("%s\n", extractAsComment("CREATE EXTERNAL TABLE", statement));
+ retcode = executeFromMetaContext(statement.data()); //create hive external table
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("Create hive external table error: %d", retcode);
+ }
+ debugMessage("done\n");
+ }
+ }
+}
+
+//============================================================================
+// This method writes the information related to the NAClusterInfo class to a
+// logfile called "NAClusterInfo.txt".
+//============================================================================
+void NAClusterInfo::captureNAClusterInfo(ofstream & naclfile)
+{
+ CollIndex i, ci;
+ char filepath[OSIM_PATHMAX];
+ char filename[OSIM_FNAMEMAX];
+
+ // We don't capture data members that are computed during the compilation of
+ // a query. These include:
+ //
+ // * smpCount_;
+ // * tableToClusterMap_;
+ // * activeClusters_;
+ //
+
+ naclfile << "localCluster_: " << localCluster_ << endl
+ << "localSMP_: " << localSMP_ << endl;
+
+ CollIndex *key_collindex;
+ maps *val_maps;
+ // Iterator for logging all the entries in clusterToCPUMap_ HashDictionary.
+ NAHashDictionaryIterator<CollIndex, maps> C2CPUIter (*clusterToCPUMap_, NULL, NULL);
+ naclfile << "clusterToCPUMap_: " << C2CPUIter.entries() << " :" << endl;
+ if (C2CPUIter.entries() > 0)
+ {
+ // Write the header line for the table.
+ naclfile << " ";
+ naclfile.width(10);
+ naclfile << "clusterNum" << " ";
+ naclfile << "cpuList" << endl;
+ for (i=0; i<C2CPUIter.entries(); i++)
+ {
+ C2CPUIter.getNext(key_collindex, val_maps);
+ naclfile << " ";
+ naclfile.width(10); naclfile << *key_collindex << " ";
+ naclfile << val_maps->list->entries() << " : ";
+ for (ci=0; ci<val_maps->list->entries(); ci++)
+ {
+ naclfile.width(3); naclfile << (*(val_maps->list))[ci] << " ";
+ }
+ naclfile << endl;
+ }
+ }
+
+ Int32 * nodeID = NULL;
+ NAString* nodeName = NULL;
+ NAHashDictionaryIterator<Int32, NAString> nodeNameAndIDIter (*nodeIdToNodeNameMap_);
+ naclfile << "nodeIdAndNodeNameMap: " << nodeNameAndIDIter.entries() << endl;
+ for(nodeNameAndIDIter.getNext(nodeID, nodeName); nodeID && nodeName; nodeNameAndIDIter.getNext(nodeID, nodeName))
+ {
+ naclfile << *nodeID << " " << nodeName->data() << endl;
+ }
+
+ // Now save the OS-specific information to the NAClusterInfo.txt file
+ captureOSInfo(naclfile);
+}
+
+//============================================================================
+// This method reads the information needed for NAClusterInfo class from
+// a logfile called "NAClusterInfo.txt" and then populates the variables
+// accordigly.
+//============================================================================
+void NAClusterInfo::simulateNAClusterInfo()
+{
+ Int32 i, ci;
+ char var[256];
+
+ const char* filepath = CURRCONTEXT_OPTSIMULATOR->getLogFilePath(OptimizerSimulator::NACLUSTERINFO);
+
+ activeClusters_= NULL;
+ physicalSMPCount_ = -1;
+
+ ifstream naclfile(filepath);
+
+ if(!naclfile.good())
+ {
+ raiseOsimException("Unable to open %s file for reading data.", filepath);
+ }
+
+ while(naclfile.good())
+ {
+ // Read the variable name from the file.
+ naclfile.getline(var, sizeof(var), ':');
+ if(!strcmp(var, "localCluster_"))
+ {
+ naclfile >> localCluster_; naclfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "localSMP_"))
+ {
+ naclfile >> localSMP_; naclfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "clusterToCPUMap_"))
+ {
+ Int32 C2CPU_entries, clusterNum, cpuList_entries, cpuNum;
+
+ clusterToCPUMap_ = new(heap_) NAHashDictionary<CollIndex,maps>(&clusterNumHashFunc,17,TRUE, heap_);
+ naclfile >> C2CPU_entries; naclfile.ignore(OSIM_LINEMAX, '\n');
+ if(C2CPU_entries > 0)
+ {
+ // Read and ignore the header line.
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ for (i=0; i<C2CPU_entries; i++)
+ {
+ naclfile >> clusterNum;
+ naclfile >> cpuList_entries; naclfile.ignore(OSIM_LINEMAX, ':');
+ CollIndex *key_clusterNum = new(heap_) CollIndex(clusterNum);
+ maps *val_cpuList = new(heap_) maps(heap_);
+ for (ci=0; ci<cpuList_entries; ci++)
+ {
+ naclfile >> cpuNum;
+ val_cpuList->list->insert(cpuNum);
+ }
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ CollIndex *checkClusterNum = clusterToCPUMap_->insert(key_clusterNum, val_cpuList);
+ CMPASSERT(checkClusterNum);
+ }
+ }
+ }
+ else if(!strcmp(var, "nodeIdAndNodeNameMap"))
+ {
+ Int32 id_name_entries;
+ Int32 nodeId;
+ char nodeName[256];
+ nodeIdToNodeNameMap_ = new(heap_) NAHashDictionary<Int32, NAString>
+ (&intHashFunc, 101,TRUE,heap_);
+
+ nodeNameToNodeIdMap_ = new(heap_) NAHashDictionary<NAString, Int32>
+ (&NAString::hash, 101,TRUE,heap_);
+ naclfile >> id_name_entries;
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ physicalSMPCount_ = id_name_entries;
+ for(i = 0; i < id_name_entries; i++)
+ {
+ naclfile >> nodeId >> nodeName;
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+
+ //populate clusterId<=>clusterName map from file
+ Int32 * key_nodeId = new Int32(nodeId);
+ NAString * val_nodeName = new (heap_) NAString(nodeName, heap_);
+ Int32 * retId = nodeIdToNodeNameMap_->insert(key_nodeId, val_nodeName);
+ //CMPASSERT(retId);
+
+ NAString * key_nodeName = new (heap_) NAString(nodeName, heap_);
+ Int32 * val_nodeId = new Int32(nodeId);
+ NAString * retName = nodeNameToNodeIdMap_->insert(key_nodeName, val_nodeId);
+ //some node names are like g4t3024:0, g4t3024:1
+ //I don't know why we need to remove strings after ':' or '.' in node name,
+ //but if string after ':' or '.' is removed, same node names correspond to different node ids,
+ //this can cause problems here
+ //CMPASSERT(retName);
+ }
+ }
+ else
+ {
+ // This variable will either be read in simulateNAClusterInfoNSK()
+ // method of NAClusterInfoNSK class or is not the one that we want
+ // to read here in this method. So discard it and continue.
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ while (naclfile.peek() == ' ')
+ {
+ // The main variables are listed at the beginning of a line
+ // with additional information indented. If one or more spaces
+ // are seen at the beginning of the line upon the entry to this
+ // while loop, it is because of that additional information.
+ // So, ignore this line since the variable is being ignored.
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ }
+ }
+}
+
+void NAClusterInfoLinux::simulateNAClusterInfoLinux()
+{
+ char var[256];
+
+ const char* filepath = CURRCONTEXT_OPTSIMULATOR->getLogFilePath(OptimizerSimulator::NACLUSTERINFO);
+
+ ifstream nacllinuxfile(filepath);
+
+ if(!nacllinuxfile.good())
+ {
+ raiseOsimException("Unable to open %s file for reading data.", filepath);
+ }
+
+ while(nacllinuxfile.good())
+ {
+ // Read the variable name from the file
+ nacllinuxfile.getline(var, sizeof(var), ':');
+ if(!strcmp(var, "frequency_"))
+ {
+ nacllinuxfile >> frequency_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "iorate_"))
+ {
+ nacllinuxfile >> iorate_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "seekTime_"))
+ {
+ nacllinuxfile >> seekTime_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "pageSize_"))
+ {
+ nacllinuxfile >> pageSize_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "totalMemoryAvailable_"))
+ {
+ nacllinuxfile >> totalMemoryAvailable_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "numCPUcoresPerNode_"))
+ {
+ nacllinuxfile >> numCPUcoresPerNode_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else
+ {
+ // This variable either may have been read in simulateNAClusterInfo()
+ // method of NAClusterInfo class or is not the one that we want to
+ // read here in this method. So discard it.
+ nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ while (nacllinuxfile.peek() == ' ')
+ {
+ // The main variables are listed at the beginning of a line
+ // with additional information indented. If one or more spaces
+ // are seen at the beginning of the line upon the entry to this
+ // while loop, it is because of that additional information.
+ // So, ignore this line since the variable is being ignored.
+ nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ }
+ }
+}
+
+//use local table UID to replace UID in captured histogram file,
+//then read from files to histogram.
+NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashDictionary<NAString, QualifiedName> * modifiedPathList, NABoolean isHive)
{
int retcode;
NAString tmp = osimLogLocalDir_ + '/';
@@ -967,36 +1422,48 @@ NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashD
const char* histogramTableName = entry->getHistogram();
NAString * UIDModifiedPath = new (STMTHEAP) NAString(STMTHEAP);
+ //qualifiedName is used to create histogram tables
+ QualifiedName* qualifiedName;
+
+ qualifiedName = new (STMTHEAP) QualifiedName(histogramTableName, schema, catalog, STMTHEAP);
+
+ Int64 tableUID;
+ //in _MD_.OBJECTS, schema of hive table is _HV_HIVE_, catalog is TRAFODION
+ if(isHive)
+ {
+ NAString trafName;
+ trafName = ComConvertNativeNameToTrafName(
+ entry->getCatalog(),
+ entry->getSchema(),
+ entry->getTable());
+ QualifiedName qname (trafName, 3);
+ tableUID = getTableUID(qname.getCatalogName(), qname.getSchemaName(), qname.getObjectName());
+ }
+ else
+ tableUID = getTableUID(catalog, schema, table);
- QualifiedName* qualifiedName = new (STMTHEAP) QualifiedName(histogramTableName, schema, catalog, STMTHEAP);
-
- Int64 tableUID = getTableUID(catalog, schema, table);
if(tableUID < 0)
{
- NAString errMsg = "Get Table UID Error: " ;
- errMsg += std::to_string((long long)(tableUID)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Get Table UID Error: %d", tableUID);
}
NAString dataPath(STMTHEAP);
//get text file path within the dir
DIR * histogramDir = opendir(fullPath);
if(!histogramDir)
{
- NAString errMsg = "Error open ";
- errMsg += fullPath;
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", fullPath);
}
struct dirent * dataPathInfo = readdir(histogramDir);
while(dataPathInfo != NULL)
{
- if(dataPathInfo->d_name[0] != '.')
+ if(dataPathInfo->d_name[0] != '.' && dataPathInfo->d_type != DT_DIR)
{//there should be only one
dataPath = fullPath;
dataPath += '/';
dataPath += dataPathInfo->d_name;
break;
}
- dataPathInfo = readdir(histogramDir);
+ dataPathInfo = readdir(histogramDir);
}
closedir(histogramDir);
@@ -1020,9 +1487,7 @@ NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashD
std::ifstream infile (dataPath.data(), std::ifstream::binary);
if(!infile.good())
{
- NAString errMsg = "Error open ";
- errMsg += dataPath;
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", dataPath.data());
}
std::ofstream outfile (UIDModifiedPath->data(), std::ofstream::binary|std::ofstream::app);
//update table UID between files
@@ -1059,48 +1524,9 @@ NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashD
outfile << V6 << endl;
}
else
- OsimLogException("Invalid format of histogram data.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Invalid format of histogram data.");
}
-#if 0
- enum workState { WRITEUID, READUID, RESTOFLINE };
- workState state = READUID;
- char a = ' ';
- char uidstr[256];
- snprintf(uidstr, 256, "%ld", tableUID);
- while(1)
- {
- switch(state)
- {
- case READUID :
- infile.get (a);
- if(infile.eof())
- return TRUE;
- else if('|' == a)
- state = WRITEUID;
- break;
- case WRITEUID :
- outfile.write(uidstr, strlen(uidstr));
- outfile.put(a);
- state = RESTOFLINE;
- break;
-
- case RESTOFLINE :
- infile.get (a);
- if(infile.eof())
- return TRUE;
- else if('\n' == a){
- outfile.put(a);
- state = READUID;
- }
- else if('\0' == a)
- continue;
- else
- outfile.put(a);
- break;
- }//switch
- }//while
-#endif
return TRUE;
}
@@ -1109,34 +1535,24 @@ void OptimizerSimulator::execHiveSQL(const char* hiveSQL)
HiveClient_JNI *hiveClient = CmpCommon::context()->getHiveClient();
if (hiveClient == NULL)
- {
- NAString errMsg;
- errMsg = "Error initialize hive client.";
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
+ raiseOsimException("Error initialize hive client.");
else
- {
- if (!CmpCommon::context()->execHiveSQL(hiveSQL))
- {
- NAString errMsg;
- errMsg = "Error running hive SQL.";
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- }
+ if (!CmpCommon::context()->execHiveSQL(hiveSQL))
+ raiseOsimException("Error running hive SQL.");
}
-short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen)
+short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen, NABoolean isHive)
{
debugMessage("loading %s\n", qualifiedName->getQualifiedNameAsString().data());
short retcode;
NAString cmd(STMTHEAP);
-
+ //drop hive sb_histogram table
cmd = "drop table "+qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
execHiveSQL(cmd.data());
-
+ //create hive sb_histogram table
cmd = "create table " + qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
- cmd += " ( table_uid bigint"
+ cmd += " (table_uid bigint"
",histogram_id int"
",col_position int"
",column_number int"
@@ -1163,13 +1579,17 @@ short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedN
" ) row format delimited fields terminated by '|' "
"tblproperties ('serialization.null.format' = '\\N')";
execHiveSQL(cmd.data());
-
+ //populate hive table with table UID modified file
cmd = "load data local inpath '" + *modifiedPath + "' into table ";
cmd += qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
execHiveSQL(cmd.data());
//create sb_histograms
- cmd = "CREATE TABLE IF NOT EXISTS " + qualifiedName->getQualifiedNameAsString();
+ cmd = "CREATE TABLE IF NOT EXISTS ";
+ if(isHive)
+ cmd += "TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd += qualifiedName->getQualifiedNameAsString();
cmd += " ( TABLE_UID LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , HISTOGRAM_ID INT UNSIGNED NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , COL_POSITION INT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
@@ -1201,36 +1621,43 @@ short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedN
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
- cmd = "upsert using load into " + qualifiedName->getQualifiedNameAsString() + " select * from hive.hive.";
- cmd += qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
+ cmd = "upsert using load into ";
+ if(isHive)
+ cmd += "TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd += qualifiedName->getQualifiedNameAsString();
+ //from hive table to trafodion table
+ cmd +=" select TABLE_UID, HISTOGRAM_ID, COL_POSITION, COLUMN_NUMBER, "
+ "COLCOUNT, INTERVAL_COUNT, ROWCOUNT, TOTAL_UEC, STATS_TIME, "
+ "CAST(LOW_VALUE AS VARCHAR(250) CHARACTER SET UCS2 NOT NULL), "
+ "CAST(HIGH_VALUE AS VARCHAR(250) CHARACTER SET UCS2 NOT NULL), "
+ "READ_TIME, READ_COUNT, SAMPLE_SECS, COL_SECS, SAMPLE_PERCENT, "
+ "CV, REASON, V1, V2, V3, V4, V5, V6 from hive.hive.";
+ cmd +=qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
retcode = executeFromMetaContext(cmd.data());
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
return retcode;
}
-short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen)
+short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen, NABoolean isHive)
{
debugMessage("loading %s\n", qualifiedName->getQualifiedNameAsString().data());
short retcode;
NAString cmd(STMTHEAP);
-
+ //drop hive histogram table
cmd = "drop table "+qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
execHiveSQL(cmd.data());
-
+ //create hive histogram table
cmd = "create table " + qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
cmd += " ( table_uid bigint"
",histogram_id int"
@@ -1254,7 +1681,11 @@ short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, Qu
execHiveSQL(cmd.data());
//create sb_histogram_intervals
- cmd = "CREATE TABLE IF NOT EXISTS " + qualifiedName->getQualifiedNameAsString();
+ cmd = "CREATE TABLE IF NOT EXISTS ";
+ if(isHive)
+ cmd +="TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd += qualifiedName->getQualifiedNameAsString();
cmd += " ( TABLE_UID LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , HISTOGRAM_ID INT UNSIGNED NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , INTERVAL_NUMBER SMALLINT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
@@ -1276,42 +1707,39 @@ short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, Qu
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
-
- cmd = "upsert using load into " + qualifiedName->getQualifiedNameAsString() + " select * from hive.hive.";
+ //from hive table to trafodion table.
+ cmd = "upsert using load into ";
+ if(isHive)
+ cmd +="TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd +=qualifiedName->getQualifiedNameAsString();
+ //from hive table to trafodion table
+ cmd += " select * from hive.hive.";
cmd += qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
retcode = executeFromMetaContext(cmd.data());
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
-
return retcode;
}
-void OptimizerSimulator::loadHistograms()
+void OptimizerSimulator::loadHistograms(const char* histogramPath, NABoolean isHive)
{
debugMessage("loading histograms ...\n");
OsimElementMapper om;
OsimAllHistograms * allHistograms = NULL;
XMLDocument doc(STMTHEAP, om);
- std::ifstream s (sysCallLogFilePath_[HISTOGRAM_PATHS], std::ifstream::binary);
+ std::ifstream s (histogramPath, std::ifstream::binary);
if(!s.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[HISTOGRAM_PATHS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", histogramPath);
}
char * txt = new (STMTHEAP) char[1024];
- s.ignore(OSIM_LINEMAX, '\n');
- s.ignore(OSIM_LINEMAX, '\n');
s.read(txt, 1024);
while(s.gcount() > 0)
{
@@ -1326,41 +1754,52 @@ void OptimizerSimulator::loadHistograms()
}
if(!allHistograms)
{
- NAString errMsg = "Error parsing ";
- errMsg += sysCallLogFilePath_[HISTOGRAM_PATHS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error parsing %s", histogramPath);
}
NAHashDictionary<NAString, QualifiedName> * modifiedPathDict =
new(STMTHEAP) NAHashDictionary<NAString, QualifiedName>
(&NAString::hash, 101, TRUE, STMTHEAP);
- for(CollIndex i = 0; i < allHistograms->getEntries().entries(); i++)
+ for(CollIndex i = 0; i < allHistograms->getHistograms().entries(); i++)
{
- OsimHistogramEntry * en = (allHistograms->getEntries())[i];
- massageTableUID(en, modifiedPathDict);
+ OsimHistogramEntry * en = (allHistograms->getHistograms())[i];
+ massageTableUID(en, modifiedPathDict, isHive);
}
-
+
//do load
NAHashDictionaryIterator<NAString, QualifiedName> iterator(*modifiedPathDict);
+ //create _HIVESTATS_ schema
+ if(isHive && iterator.entries() > 0)
+ {
+ //create hive stats schema
+ const char * create_stats_schema = "CREATE SCHEMA IF NOT EXISTS "
+ "TRAFODION.\"_HIVESTATS_\" AUTHORIZATION DB__ROOT";
+ short retcode = executeFromMetaContext(create_stats_schema);
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("create hive stats schema: return code %d", retcode);
+ }
+ }
NAString* modifiedPath = NULL;
QualifiedName* qualifiedName = NULL;
Queue * dummyQueue = NULL;
- iterator.getNext(modifiedPath, qualifiedName) ;
- while ( modifiedPath && qualifiedName )
+ for (iterator.getNext(modifiedPath, qualifiedName); modifiedPath && qualifiedName; iterator.getNext(modifiedPath, qualifiedName))
{
if(qualifiedName->getObjectName().compareTo("SB_HISTOGRAMS", NAString::ignoreCase) == 0)
{
- loadHistogramsTable(modifiedPath, qualifiedName, OSIM_LINEMAX);
+ loadHistogramsTable(modifiedPath, qualifiedName, OSIM_LINEMAX, isHive);
}
else if(qualifiedName->getObjectName().compareTo("SB_HISTOGRAM_INTERVALS", NAString::ignoreCase) == 0)
{
- loadHistogramIntervalsTable(modifiedPath, qualifiedName, OSIM_LINEMAX);
+ loadHistogramIntervalsTable(modifiedPath, qualifiedName, OSIM_LINEMAX, isHive);
}
unlink(modifiedPath->data());
- iterator.getNext(modifiedPath, qualifiedName);
}
}
+
+
void OptimizerSimulator::initializeCLI()
{
if(!CLIInitialized_)
@@ -1376,24 +1815,17 @@ void OptimizerSimulator::readAndSetCQDs()
{
initializeCLI();
NABoolean isDir;
- if(!fileExists(sysCallLogFilePath_[CQD_DEFAULTSFILE], isDir))
+ if(!isFileExists(logFilePaths_[CQD_DEFAULTSFILE], isDir))
{
- char errMsg[38+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.",
- sysCallLogFilePath_[CQD_DEFAULTSFILE]);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[CQD_DEFAULTSFILE]);
}
- ifstream inLogfile(sysCallLogFilePath_[CQD_DEFAULTSFILE]);
+ ifstream inLogfile(logFilePaths_[CQD_DEFAULTSFILE]);
if(!inLogfile.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CQD_DEFAULTSFILE];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[CQD_DEFAULTSFILE]);
}
- // Read and ignore the top 2 header lines.
- inLogfile.ignore(OSIM_LINEMAX, '\n');
- inLogfile.ignore(OSIM_LINEMAX, '\n');
+
Lng32 retcode;
std::string cqd;
while(inLogfile.good())
@@ -1408,9 +1840,7 @@ void OptimizerSimulator::readAndSetCQDs()
if(retcode < 0)
{
cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Error Setting CQD: ";
- errMsg += cqd.c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error Setting CQD: %s", cqd.c_str());
}
}
}
@@ -1421,7 +1851,7 @@ Int64 OptimizerSimulator::getTableUID(const char * catName, const char * schName
Int64 retcode;
if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Switch Context.");
}
retcode = cmpSBD_->getObjectUID(cliInterface_, catName, schName, objName, "BT");
@@ -1437,7 +1867,7 @@ short OptimizerSimulator::fetchAllRowsFromMetaContext(Queue * &q, const char* qu
short retcode;
if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Switch Context.");
}
retcode = cliInterface_->fetchAllRows(queue_, query, 0, FALSE, FALSE, TRUE);
@@ -1524,6 +1954,77 @@ NABoolean OptimizerSimulator::readStmt(ifstream & DDLFile, NAString & stmt, NASt
}
}
+//Get a complete SQL statement and a line of comment in front of the SQL statement
+NABoolean OptimizerSimulator::readHiveStmt(ifstream & DDLFile, NAString & stmt, NAString & comment)
+{
+ char a = ' ';
+ long index = 0;
+ stmt = "";
+ comment = "";
+ enum readState
+ {
+ PROBABLY_COMMENT, CONSUME, EAT_CHAR, EOSTMT, EOFILE
+ };
+ readState state = EAT_CHAR;
+ while(1)
+ {
+ switch(state)
+ {
+ case EAT_CHAR:
+ DDLFile.get(a);
+ if(DDLFile.eof())
+ state = EOFILE;
+ else if(a == '/')
+ state = PROBABLY_COMMENT;
+ else if(a == ';') //end of statement
+ state = EOSTMT;
+ else
+ stmt += a;
+ break;
+ case PROBABLY_COMMENT :
+ {
+ char b = ' ';
+ DDLFile.get(b);
+ if( b == '*' )
+ state = CONSUME;
+ else //not comment
+ {
+ stmt += a;
+ stmt += b;
+ state = EAT_CHAR;
+ }
+ break;
+ }
+ case CONSUME:
+ //comment line, eat up rest of the line
+ while(DDLFile.get(a))
+ {
+ if(a == '*'){
+ char b = ' ';
+ DDLFile.get(b);
+ if(b == '/') //end of comment
+ state = EAT_CHAR;
+ else
+ comment += b;
+ break;
+ }
+ else if(DDLFile.eof()){
+ state = EOFILE;
+ break;
+ }
+ else
+ comment += a;
+ }
+ break;
+ case EOSTMT:
+ return TRUE;
+ case EOFILE:
+ return FALSE;
+ }
+ }
+}
+
+
void OptimizerSimulator::histogramHDFSToLocal()
{
Int32 status;
@@ -1566,19 +2067,14 @@ void OptimizerSimulator::histogramHDFSToLocal()
status = hdfsCopy(hdfs, src.data(), locfs, dst.data());
if(status != 0)
{
- NAString errMsg;
- errMsg = "Error getting histogram data from ";
- errMsg += src + " to " + dst;
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Error getting histogram data from %s to %s", src.data(), dst.data());
}
}
if( hdfsDisconnect(locfs) != 0 ||
hdfsDisconnect(hdfs) != 0)
{
- NAString errMsg;
- errMsg = "Error getting histogram data, disconneting";
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Error getting histogram data, disconneting");
}
}
@@ -1609,51 +2105,31 @@ void OptimizerSimulator::createLogDir()
{
case EACCES:
{
- char errMsg[37+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, permission denied.",
- osimLogLocalDir_.data());
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, permission denied.", osimLogLocalDir_.data());
}
break;
case ENOENT:
{
- char errMsg[58+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, a component of the path does not exist.",
- osimLogLocalDir_.data());
-
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, a component of the path does not exist.", osimLogLocalDir_.data());
}
break;
case EROFS:
{
- char errMsg[40+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, read-only filesystem.",
- osimLogLocalDir_.data());
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, read-only filesystem.", osimLogLocalDir_.data());
}
break;
case ENOTDIR:
{
- char errMsg[62+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, a component of the path is not a directory.",
- osimLogLocalDir_.data());
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, a component of the path is not a directory.", osimLogLocalDir_.data());
}
break;
default:
{
- char errMsg[58+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create %s, errno is %d",
- osimLogLocalDir_.data(), error);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create %s, errno is %d", osimLogLocalDir_.data(), error);
}
break;
}
+ rval = mkdir(hiveTableStatsDir_.data(), S_IRWXU | S_IRWXG );
}
void OptimizerSimulator::readSysCallLogfiles()
@@ -1664,6 +2140,13 @@ void OptimizerSimulator::readSysCallLogfiles()
readLogFile_captureSysType();
}
+void OptimizerSimulator::setOsimLogdir(const char *localdir) {
+ if (localdir) {
+ osimLogLocalDir_ = localdir;
+ hiveTableStatsDir_ = osimLogLocalDir_ + OSIM_HIVE_TABLE_STATS_DIR;
+ }
+}
+
void OptimizerSimulator::initHashDictionaries()
{
// Initialize hash dictionary variables for all the system calls.
@@ -1682,48 +2165,37 @@ void OptimizerSimulator::initHashDictionaries()
hashDict_Synonyms_ = new(heap_) NAHashDictionary<const QualifiedName, Int32>
(&QualifiedName::hash, 101, TRUE, heap_);
- hashDict_TablesBeforeAction_ = new (heap_) NAHashDictionary<NAString, Int32>
- (&NAString::hash, 101, TRUE, heap_);
-
- hashDict_ViewsBeforeAction_= new (heap_) NAHashDictionary<NAString, Int32>
- (&NAString::hash, 101, TRUE, heap_);
+ hashDict_HiveTables_ = new(heap_) NAHashDictionary<const QualifiedName, Int64>
+ (&QualifiedName::hash, 101, TRUE, heap_);
hashDictionariesInitialized_ = TRUE;
}
}
-void OptimizerSimulator::setLogFilepath(sysCall sc)
+void OptimizerSimulator::createLogFilepath(OsimLog sc)
{
// Allocate memory for file pathname:
// dirname + '/' + syscallname + ".txt" + '\0'
- size_t pathLen = osimLogLocalDir_.length()+1+strlen(sysCallLogFileName_[sc])+4+1;
- sysCallLogFilePath_[sc] = new (heap_) char[pathLen];
+ size_t pathLen = osimLogLocalDir_.length()+1+strlen(logFileNames_[sc])+4+1;
+ logFilePaths_[sc] = new (heap_) char[pathLen];
// Construct an absolute pathname for the file.
- strcpy(sysCallLogFilePath_[sc], osimLogLocalDir_.data());
- strcat(sysCallLogFilePath_[sc], "/");
- strcat(sysCallLogFilePath_[sc], sysCallLogFileName_[sc]);
+ strcpy(logFilePaths_[sc], osimLogLocalDir_.data());
+ strcat(logFilePaths_[sc], "/");
+ strcat(logFilePaths_[sc], logFileNames_[sc]);
}
-void OptimizerSimulator::openAndAddHeaderToLogfile(sysCall sc)
+void OptimizerSimulator::openWriteLogStreams(OsimLog sc)
{
- NABoolean isDir;
- if(fileExists(sysCallLogFilePath_[sc],isDir))
- {
- char errMsg1[118+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- sprintf(errMsg1, "The target log file %s already exists. "
- "Delete this and other existing log files before "
- "running the OSIM in CAPTURE mode.", sysCallLogFilePath_[sc]);
- OsimLogException(errMsg1, __FILE__, __LINE__).throwException();
- }
-
- // Create the file and write header lines to it.
- writeSysCallStream_[sc] = new (heap_) ofstream(sysCallLogFilePath_[sc],ios::app);
-
- *writeSysCallStream_[sc] << "--" << sysCallLogFileName_[sc] << ":" << endl;
-
- // Indent the output.
- *writeSysCallStream_[sc] << " " << endl;
+ NABoolean isDir;
+ if(isFileExists(logFilePaths_[sc],isDir))
+ {
+ raiseOsimException("The target log file %s already exists. "
+ "Delete this and other existing log files before "
+ "running the OSIM in CAPTURE mode.", logFilePaths_[sc]);
+ }
+ // Create the file and write header lines to it.
+ writeLogStreams_[sc] = new (heap_) ofstream(logFilePaths_[sc],ios::app);
}
// Initialize the log files if OSIM is running under either CAPTURE
@@ -1732,26 +2204,23 @@ void OptimizerSimulator::openAndAddHeaderToLogfile(sysCall sc)
// to NULL if OSIM is not running(OFF).
void OptimizerSimulator::initLogFilePaths()
{
- for (sysCall sc=FIRST_SYSCALL; sc<NUM_OF_SYSCALLS; sc = sysCall(sc+1))
+ for (OsimLog sc=FIRST_LOG; sc<NUM_OF_LOGS; sc = OsimLog(sc+1))
{
switch (osimMode_)
{
case OFF:
- // OFF mode indicates no log files needed.
- sysCallLogFilePath_[sc] = NULL;
- break;
+ // OFF mode indicates no log files needed.
+ logFilePaths_[sc] = NULL;
+ break;
case CAPTURE:
- // Set log file path.
- setLogFilepath(sc);
- // Add header to the log file.
- openAndAddHeaderToLogfile(sc);
- break;
+ createLogFilepath(sc);
+ openWriteLogStreams(sc);
+ break;
case LOAD:
case UNLOAD:
case SIMULATE:
- // Set log file path.
- setLogFilepath(sc);
- break;;
+ createLogFilepath(sc);
+ break;;
}
}
}
@@ -1763,7 +2232,7 @@ void OptimizerSimulator::capture_MYSYSTEMNUMBER(short sysNum)
if (mySystemNumber_ == -1)
{
// Open file in append mode.
- ofstream * outLogfile = writeSysCallStream_[MYSYSTEMNUMBER];
+ ofstream * outLogfile = writeLogStreams_[MYSYSTEMNUMBER];
// Write data at the end of the file.
Int32 origWidth = (*outLogfile).width();
@@ -1779,19 +2248,10 @@ void OptimizerSimulator::readLogfile_MYSYSTEMNUMBER()
short sysNum;
NABoolean isDir;
- if(!fileExists(sysCallLogFilePath_[MYSYSTEMNUMBER],isDir))
- {
- char errMsg[38+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.",
- sysCallLogFilePath_[MYSYSTEMNUMBER]);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
- }
-
- ifstream inLogfile(sysCallLogFilePath_[MYSYSTEMNUMBER]);
+ if(!isFileExists(logFilePaths_[MYSYSTEMNUMBER],isDir))
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[MYSYSTEMNUMBER]);
- // Read and ignore the top 2 header lines.
- inLogfile.ignore(OSIM_LINEMAX, '\n');
- inLogfile.ignore(OSIM_LINEMAX, '\n');
+ ifstream inLogfile(logFilePaths_[MYSYSTEMNUMBER]);
if(inLogfile.good())
{
@@ -1808,6 +2268,8 @@ void OptimizerSimulator::readLogfile_MYSYSTEMNUMBER()
mySystemNumber_ = sysNum;
}
}
+ else
+ raiseOsimException("Unable to open %s, bad handle.", logFilePaths_[MYSYSTEMNUMBER]);
}
short OptimizerSimulator::simulate_MYSYSTEMNUMBER()
@@ -1839,29 +2301,13 @@ short OSIM_MYSYSTEMNUMBER()
break;
default:
// The OSIM must run under OFF (normal), CAPTURE or SIMULATE mode.
- OSIM_errorMessage("Invalid OSIM mode - It must be OFF or CAPTURE or SIMULATE.");
+ raiseOsimException("An invalid OSIM mode is encountered - The valid mode is OFF, CAPTURE or SIMULATE");
break;
}
return sysNum;
}
// END ************* System Call: MYSYSTEMNUMBER() *************
-
-// BEGIN *********** System Call: getEstimatedRows() ****************
-//
-void OSIM_captureEstimatedRows(const char *tableName, double estRows)
-{
- if(CURRCONTEXT_OPTSIMULATOR)
- CURRCONTEXT_OPTSIMULATOR->capture_getEstimatedRows(tableName, estRows);
-}
-
-double OSIM_simulateEstimatedRows(const char *tableName)
-{
- return CURRCONTEXT_OPTSIMULATOR ?
- CURRCONTEXT_OPTSIMULATOR->simulate_getEstimatedRows(tableName) :
- -1;
-}
-
void OptimizerSimulator::capture_getEstimatedRows(const char *tableName, double estRows)
{
NAString *key_tableName = new (heap_) NAString(tableName, heap_);
@@ -1879,7 +2325,7 @@ void OptimizerSimulator::capture_getEstimatedRows(const char *tableName, double
NAString *check = hashDict_getEstimatedRows_->insert(key_tableName,
val_estRows);
// Open file in append mode.
- ofstream * outLogfile = writeSysCallStream_[ESTIMATED_ROWS];
+ ofstream * outLogfile = writeLogStreams_[ESTIMATED_ROWS];
Int32 origWidth = (*outLogfile).width();
// Write data at the end of the file.
(*outLogfile) << " ";
@@ -1889,6 +2335,43 @@ void OptimizerSimulator::capture_getEstimatedRows(const char *tableName, double
}
}
+void OSIM_restoreHHDFSMasterHostList()
+{
+ if(CURRCONTEXT_OPTSIMULATOR &&
+ CURRCONTEXT_OPTSIMULATOR->getOsimMode() == OptimizerSimulator::SIMULATE)
+ CURRCONTEXT_OPTSIMULATOR->restoreHHDFSMasterHostList();
+}
+
+void OptimizerSimulator::restoreHHDFSMasterHostList()
+{
+ NABoolean isDir;
+ if(!isFileExists(logFilePaths_[HHDFS_MASTER_HOST_LIST],isDir))
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[HHDFS_MASTER_HOST_LIST]);
+
+ ifstream inLogfile(logFilePaths_[HHDFS_MASTER_HOST_LIST]);
+
+ if(inLogfile.good())
+ {
+ std::string name;
+ std::string value;
+ //read HHDFSMasterHostList::hasVirtualSQNodes_;
+ inLogfile >> name >> value;
+ CmpCommon::context()->setHasVirtualSQNodes(std::atoi(value.c_str()));
+ //read HHDFSMasterHostList::numSQNodes_;
+ inLogfile >> name >> value;
+ CmpCommon::context()->setNumSQNodes(std::atoi(value.c_str()));
+ inLogfile >> value;
+ if(value.length() > 0){
+ LIST(NAString) hosts(STMTHEAP);
+ NAString line = value.c_str();
+ line.split('|', hosts);
+ for(Int32 i = 0; i < hosts.entries(); i++)
+ HHDFSMasterHostList::getHostNumInternal(hosts[i].data());
+ }
+ }
+ else
+ raiseOsimException("Unable to open %s, bad handle.", logFilePaths_[HHDFS_MASTER_HOST_LIST]);
+}
void OptimizerSimulator::readLogfile_getEstimatedRows()
{
@@ -1896,19 +2379,10 @@ void OptimizerSimulator::readLogfile_getEstimatedRows()
double estRows;
NABoolean isDir;
- if(!fileExists(sysCallLogFilePath_[ESTIMATED_ROWS],isDir))
- {
- char errMsg[38+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.",
- sysCallLogFilePath_[ESTIMATED_ROWS]);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
- }
-
- ifstream inLogfile(sysCallLogFilePath_[ESTIMATED_ROWS]);
+ if(!isFileExists(logFilePaths_[ESTIMATED_ROWS],isDir))
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[ESTIMATED_ROWS]);
- // Read and ignore the top 2 header lines.
- inLogf
<TRUNCATED>