You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2017/06/22 22:40:51 UTC
[2/4] incubator-trafodion git commit: [TRAFODION-2655] Fix two MDAM
optimizer bugs. Update optimizer simulator.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/802029e5/core/sql/optimizer/OptimizerSimulator.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptimizerSimulator.cpp b/core/sql/optimizer/OptimizerSimulator.cpp
index 6767f84..9709db8 100644
--- a/core/sql/optimizer/OptimizerSimulator.cpp
+++ b/core/sql/optimizer/OptimizerSimulator.cpp
@@ -59,20 +59,91 @@
#include "CmpSeabaseDDL.h"
#include "ExExeUtilCli.h"
#include "ComUser.h"
+#include "HDFSHook.h"
-extern THREAD_P NAClusterInfo *gpClusterInfo;
+#include "Globals.h"
+#include "CmpContext.h"
+#include "Context.h"
+extern THREAD_P NAClusterInfo *gpClusterInfo;
extern const WordAsBits SingleBitArray[];
-//the dir path should start from /bulkload
+// Define PATH_MAX, FILENAME_MAX, LINE_MAX for both NT and NSK.
+// _MAX_PATH and _MAX_FNAME are defined in stdlib.h that is
+// included above.
+#define OSIM_PATHMAX PATH_MAX
+#define OSIM_FNAMEMAX FILENAME_MAX
+#define OSIM_LINEMAX 4096
+#define OSIM_HIVE_TABLE_STATS_DIR "/hive_table_stats"
+//#define HIVE_CRAETE_TABLE_SQL "/hive_create_table.sql"
+//#define HIVE_TABLE_LIST "/hive_table_list.txt"
+//the hdfs dir path should start from /bulkload
#define UNLOAD_HDFS_DIR "/user/trafodion/bulkload/osim_capture"
+//tags for histograms file path
+#define TAG_ALL_HISTOGRAMS "all_histograms"
+#define TAG_HISTOGRAM_ENTRY "histogram_entry"
+#define TAG_FULL_PATH "fullpath"
+#define TAG_USER_NAME "username"
+#define TAG_PID "pid"
+#define TAG_CATALOG "catalog"
+#define TAG_SCHEMA "schema"
+#define TAG_TABLE "table"
+#define TAG_HISTOGRAM "histogram"
+//tags for hive table stats
+#define TAG_HHDFSFILESTATS "hhdfs_file_stats"
+#define TAG_HHDFSBUCKETSTATS "hhdfs_bucket_stats"
+#define TAG_HHDFSLISTPARTSTATS "hhdfs_list_part_stats"
+#define TAG_HHDFSTABLESTATS "hhdfs_table_stats"
+
+//<TAG_ALL_HISTOGRAMS>
+// <TAG_HISTOGRAM_ENTRY>
+// <TAG_FULL_PATH>/opt/home/xx/xxx </TAG_FULL_PATH>
+// <TAG_USER_NAME>root</TAG_USER_NAME>
+// <TAG_PID>12345</TAG_PID>
+// <TAG_CATALOG>trafodion</TAG_CATALOG>
+// <TAG_SCHEMA>seabase</TAG_SCHEMA>
+// <TAG_TABLE>order042</TAG_TABLE>
+// <TAG_HISTOGRAM>sb_histogram_interval</TAG_HISTOGRAM>
+// </TAG_HISTOGRAM_ENTRY>
+// ...
+//</TAG_ALL_HISTOGRAMS>
+const char OsimAllHistograms::elemName[] = TAG_ALL_HISTOGRAMS;
+const char OsimHistogramEntry::elemName[] = TAG_HISTOGRAM_ENTRY;
+const char OsimHHDFSFileStats::elemName[] = TAG_HHDFSFILESTATS;
+const char OsimHHDFSBucketStats::elemName[] = TAG_HHDFSBUCKETSTATS;
+const char OsimHHDFSListPartitionStats::elemName[] = TAG_HHDFSLISTPARTSTATS;
+const char OsimHHDFSTableStats::elemName[] = TAG_HHDFSTABLESTATS;
+
+const char* OptimizerSimulator::logFileNames_[NUM_OF_LOGS]= {
+ "ESTIMATED_ROWS.txt" ,
+ "NODE_AND_CLUSTER_NUMBERS.txt",
+ "NAClusterInfo.txt",
+ "MYSYSTEMNUMBER.txt",
+ "VIEWS.txt" ,
+ "VIEWDDLS.txt",
+ "TABLES.txt",
+ "CREATE_SCHEMA_DDLS.txt",
+ "CREATE_TABLE_DDLS.txt" ,
+ "SYNONYMS.txt",
+ "SYNONYMDDLS.txt",
+ "CQDS.txt" ,
+ "QUERIES.txt",
+ "VERSIONS.txt",
+ "CaptureSysType.txt",
+ "HISTOGRAM_PATHS.xml",
+ "HIVE_HISTOGRAM_PATHS.xml",
+ "HIVE_CREATE_TABLE.sql",
+ "HIVE_CREATE_EXTERNAL_TABLE.sql",
+ "HIVE_TABLE_LIST.txt",
+ "HHDFS_MASTER_HOST_LIST.txt"
+};
-static ULng32 hashFunc_int(const Int32& Int)
+static ULng32 intHashFunc(const Int32& Int)
{
return (ULng32)Int;
}
-static NABoolean fileExists(const char *filename, NABoolean & isDir)
+static NABoolean isFileExists(const char *filename, NABoolean & isDir)
{
struct stat sb;
Int32 rVal = stat(filename, &sb);
@@ -82,10 +153,6 @@ static NABoolean fileExists(const char *filename, NABoolean & isDir)
return rVal != -1;
}
-
-const char OsimAllHistograms::elemName[] = TAG_ALL_HISTOGRAMS;
-const char OsimHistogramEntry::elemName[] = TAG_HISTOGRAM_ENTRY;
-
void OsimAllHistograms::startElement(void * parser, const char * elementName, const char * * atts)
{
OsimHistogramEntry* entry = NULL;
@@ -95,7 +162,7 @@ void OsimAllHistograms::startElement(void * parser, const char * elementName, co
list_.insert(entry);
}
else
- OsimLogException("Errors Parsing hitograms file.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Parsing hitograms file.");
}
void OsimAllHistograms::serializeBody(XMLString& xml)
@@ -161,7 +228,6 @@ void OsimHistogramEntry::endElement(void * parser, const char * elementName)
currentTag_="";
}
-
void OsimHistogramEntry::serializeBody(XMLString& xml)
{
xml.append(" ");
@@ -208,7 +274,6 @@ void OsimHistogramEntry::serializeBody(XMLString& xml)
}
-
XMLElementPtr OsimElementMapper::operator()(void *parser,
char *elementName,
AttributeList atts)
@@ -217,32 +282,10 @@ XMLElementPtr OsimElementMapper::operator()(void *parser,
//atts is not used here
if (!strcmp( elementName, "all_histograms"))
elemPtr = new (XMLPARSEHEAP) OsimAllHistograms(XMLPARSEHEAP);
-
return elemPtr;
}
/////////////////////////////////////////////////////////////////////////
-
-
-const char* OptimizerSimulator::sysCallLogFileName_[NUM_OF_SYSCALLS]= {
- "ESTIMATED_ROWS.txt" ,
- "NODE_AND_CLUSTER_NUMBERS.txt",
- "NAClusterInfo.txt",
- "MYSYSTEMNUMBER.txt",
- "VIEWS.txt" ,
- "VIEWDDLS.txt",
- "TABLES.txt",
- "CREATE_SCHEMA_DDLS.txt",
- "CREATE_TABLE_DDLS.txt" ,
- "SYNONYMS.txt",
- "SYNONYMDDLS.txt",
- "CQDS.txt" ,
- "QUERIES.txt",
- "VERSIONS.txt",
- "captureSysType.txt",
- "HISTOGRAM_PATHS.xml"
-};
-
OptimizerSimulator::OptimizerSimulator(CollHeap *heap)
:osimLogLocalDir_(heap),
osimMode_(OptimizerSimulator::OFF),
@@ -250,8 +293,7 @@ OptimizerSimulator::OptimizerSimulator(CollHeap *heap)
hashDict_Views_(NULL),
hashDict_Tables_(NULL),
hashDict_Synonyms_(NULL),
- hashDict_TablesBeforeAction_(NULL),
- hashDict_ViewsBeforeAction_(NULL),
+ hashDict_HiveTables_(NULL),
nodeNum_(-1),
clusterNum_(-1),
captureSysType_(OSIM_LINUX),
@@ -270,10 +312,10 @@ OptimizerSimulator::OptimizerSimulator(CollHeap *heap)
forceLoad_(FALSE),
heap_(heap)
{
- for (sysCall sc=FIRST_SYSCALL; sc<NUM_OF_SYSCALLS; sc = sysCall(sc+1))
+ for (OsimLog sc=FIRST_LOG; sc<NUM_OF_LOGS; sc = OsimLog(sc+1))
{
- sysCallLogFilePath_[sc]=NULL;
- writeSysCallStream_[sc]=NULL;
+ logFilePaths_[sc]=NULL;
+ writeLogStreams_[sc]=NULL;
}
}
@@ -300,7 +342,6 @@ void OSIM_warningMessage(const char *errMsg)
void OptimizerSimulator::warningMessage(const char *errMsg)
{
- // WARNING message
*CmpCommon::diags() << DgSqlCode(OSIM_ERRORORWARNING)
<< DgString0(errMsg);
}
@@ -322,23 +363,60 @@ void OptimizerSimulator::dumpVersions()
{
//dump version info
NAString cmd = "sqvers -u > ";
- cmd += sysCallLogFilePath_[VERSIONSFILE];
+ cmd += logFilePaths_[VERSIONSFILE];
system(cmd.data()); //dump versions
}
+void OptimizerSimulator::dumpHHDFSMasterHostList()
+{
+ (*writeLogStreams_[HHDFS_MASTER_HOST_LIST])
+ << "HasVirtualSQNodes"
+ << " "
+ << HHDFSMasterHostList::hasVirtualSQNodes() << endl;
+ (*writeLogStreams_[HHDFS_MASTER_HOST_LIST])
+ << "NumSQNodes"
+ << " "
+ << HHDFSMasterHostList::getNumSQNodes() << endl;
+ NAString hostNameList;
+ for(Int32 i = 0; i < HHDFSMasterHostList::entries(); i++)
+ {
+ hostNameList += HHDFSMasterHostList::getHostName(i);
+ hostNameList += '|'; //delimiter
+ }
+ if(hostNameList.length() > 0)
+ {
+ //if there's any hostname,
+ //hostNameList would be like "node0|node1|node2|",
+ //this is to replace '|'.
+ hostNameList[hostNameList.length()-1] = '\n';
+ (*writeLogStreams_[HHDFS_MASTER_HOST_LIST]) << hostNameList.data();
+ }
+}
+
+void raiseOsimException(const char* fmt, ...)
+{
+ char * buffer;
+ va_list args ;
+ va_start(args, fmt);
+ buffer = NAString::buildBuffer(fmt, args);
+ va_end(args);
+ //throw anyway null buffer will be handled inside constructor of
+ //OsimLogException, empty string will be issued.
+ OsimLogException(buffer, __FILE__, __LINE__).throwException();
+}
+
NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const char * localDir)
{
try{
-
if(targetMode == UNLOAD)
{
- setOsimMode(targetMode);
- setOsimLogdir(localDir);
- initLogFilePaths();
- setOsimMode(OFF);
- dropObjects();
- cleanup();
- return TRUE;
+ setOsimMode(targetMode);
+ setOsimLogdir(localDir);
+ initLogFilePaths();
+ setOsimMode(OFF);
+ dropObjects();
+ cleanup();
+ return TRUE;
}
switch(osimMode_)
@@ -353,10 +431,6 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
createLogDir();
initHashDictionaries();
initLogFilePaths();
- //record all qualified table names before running query,
- //except meta tables and histogram tables in any schema
- saveTablesBeforeStart();
- saveViewsBeforeStart();
setClusterInfoInitialized(TRUE);
break;
case LOAD: //OFF --> LOAD
@@ -364,10 +438,10 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
setOsimLogdir(localDir);
initHashDictionaries();
initLogFilePaths();
- saveTablesBeforeStart();
- saveViewsBeforeStart();
loadDDLs();
- loadHistograms();
+ loadHistograms(logFilePaths_[HISTOGRAM_PATHS], FALSE);
+ loadHiveDDLs();
+ loadHistograms(logFilePaths_[HIVE_HISTOGRAM_PATHS], TRUE);
break;
case SIMULATE: //OFF-->SIMU
setOsimMode(targetMode);
@@ -385,14 +459,21 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
break;
case CAPTURE:
if(targetMode == OFF) //CAPURE --> OFF only
- {
+ {
+ //call dumpHHDFSMasterHostList() first
+ //otherwise context-switch will reset
+ //HHDFSMasterHostList::hasVirtualSQNodes_
+ //HHDFSMasterHostList::numSQNodes_
+ dumpHHDFSMasterHostList();
+ dumpHiveTableDDLs();
+ dumpHiveHistograms();
dumpHistograms();
dumpVersions();
setOsimMode(targetMode);
cleanup();//NOTE: osimMode_ is set OFF in cleanup()
}
else
- errorMessage("Mode transition is not allowed.");
+ warningMessage("Mode transition is not allowed.");
break;
case LOAD:
if(targetMode == SIMULATE)//LOAD --> SIMU only
@@ -405,16 +486,17 @@ NABoolean OptimizerSimulator::setOsimModeAndLogDir(osimMode targetMode, const ch
setClusterInfoInitialized(TRUE);
}
else
- errorMessage("Mode transition rather than LOAD to SIMULATE is not allowed.");
+ warningMessage("Mode transition other than LOAD to SIMULATE is not allowed.");
break;
default :
- errorMessage("Mode transition is not allowed.");
+ warningMessage("Mode transition is not allowed.");
break;
}
}
catch(OsimLogException & e)
{
cleanup();
+ //move err string from exception object to diagnostic area
errorMessage(e.getErrMessage());
return FALSE;
}
@@ -439,39 +521,53 @@ void OptimizerSimulator::dumpDDLs(const QualifiedName & qualifiedName)
retcode = fetchAllRowsFromMetaContext(outQueue, query.data());
if (retcode < 0 || retcode == 100/*rows not found*/) {
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- OsimLogException("Errors Dumping Table DDL.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Dumping Table DDL.");
}
- outQueue->position();
-
- ofstream * createSchema = writeSysCallStream_[CREATE_SCHEMA_DDLS];
- ofstream * createTable = writeSysCallStream_[CREATE_TABLE_DDLS];
+ if(outQueue)
+ {
+ ofstream * createSchema = writeLogStreams_[CREATE_SCHEMA_DDLS];
+
+ ofstream * createTable = writeLogStreams_[CREATE_TABLE_DDLS];
+
+ //Dump a "create schema ..." to schema ddl file for every table.
- //Dump a "create schema ..." to schema ddl file for every table.
+ //This comment line will be printed during loading, ';' must be omitted
+ (*createSchema) << "--" <<"CREATE SCHEMA IF NOT EXISTS "
+ << qualifiedName.getCatalogName()
+ << "." << qualifiedName.getSchemaName() <<endl;
- //This comment line will be printed during loading, ';' must be omitted
- (*createSchema) << "--" <<"CREATE SCHEMA IF NOT EXISTS "
- << qualifiedName.getCatalogName()
- << "." << qualifiedName.getSchemaName() <<endl;
-
- (*createSchema) << "CREATE SCHEMA IF NOT EXISTS "
- << qualifiedName.getCatalogName()
- <<"."<< qualifiedName.getSchemaName()
- << ";" << endl;
-
- for (int i = 0; i < outQueue->numEntries(); i++) {
- OutputInfo * vi = (OutputInfo*)outQueue->getNext();
- char * ptr = vi->get(0);
- //skip heading newline, and add a comment line
- Int32 ix = 0;
- for(; ptr[ix]=='\n'; ix++);
- if( strstr(ptr, "CREATE TABLE") ||
- strstr(ptr, "CREATE INDEX") ||
- strstr(ptr, "CREATE UNIQUE INDEX") ||
- strstr(ptr, "ALTER TABLE") )
- (*createTable) << "--" << ptr+ix << endl;
- //output ddl
- (*createTable) << ptr << endl;
+ (*createSchema) << "CREATE SCHEMA IF NOT EXISTS "
+ << qualifiedName.getCatalogName()
+ <<"."<< qualifiedName.getSchemaName()
+ << ";" << endl;
+
+ outQueue->position();//rewind
+ for (int i = 0; i < outQueue->numEntries(); i++) {
+ OutputInfo * vi = (OutputInfo*)outQueue->getNext();
+ char * ptr = vi->get(0);
+ // skip heading newline, and add a comment line
+ // for the DDL text upto the first trailing '\n'
+ Int32 ix = 0;
+ for(; ptr[ix]=='\n'; ix++);
+ if( strstr(ptr, "CREATE TABLE") ||
+ strstr(ptr, "CREATE INDEX") ||
+ strstr(ptr, "CREATE UNIQUE INDEX") ||
+ strstr(ptr, "ALTER TABLE") )
+
+ {
+ (*createTable) << "--";
+ char* x = ptr+ix;
+ while ( (*x) && *x != '\n' ) {
+ (*createTable) << *x;
+ x++;
+ }
+ (*createTable) << endl;
+ }
+
+ //output ddl
+ (*createTable) << ptr << endl;
+ }
}
}
@@ -489,9 +585,18 @@ void OptimizerSimulator::dumpHistograms()
//enumerate captured table names and tableUIDs in hash table
for(iterator.getNext(name, tableUID); name && tableUID; iterator.getNext(name, tableUID))
{
-
+ //check if this table_uid is in TRAFODION."_HIVESTATS_".SB_HISTOGRAMS,
+ //if not, we consider this table has no histogram data.
+ Queue * outQueue = NULL;
+ query = "SELECT TABLE_UID FROM TRAFODION.";
+ query += name->getSchemaName();
+ query += ".SB_HISTOGRAMS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+ retcode = fetchAllRowsFromMetaContext(outQueue, query.data());
+ if(retcode < 0 || outQueue && outQueue->entries() == 0)
+ continue;
+
debugMessage("Dumping histograms for %s\n", name->getQualifiedNameAsAnsiString().data());
-
//dump histograms data to hdfs
query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
query += "'"UNLOAD_HDFS_DIR"/";
@@ -523,7 +628,14 @@ void OptimizerSimulator::dumpHistograms()
" FROM ";
query += name->getCatalogName();
query += ".";
- query += name->getSchemaName();
+ if(name->getSchemaName()[0]=='_')
+ {
+ query += "\"";
+ query += name->getSchemaName();
+ query += "\"";
+ }
+ else
+ query += name->getSchemaName();
query += ".SB_HISTOGRAMS WHERE TABLE_UID = ";
query += std::to_string((long long)(*tableUID)).c_str();
@@ -553,10 +665,7 @@ void OptimizerSimulator::dumpHistograms()
else if(retcode < 0 && -4082 != retcode)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg;
- errMsg = "Unload histogram data error: ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Unload histogram data error: %d", retcode);
}
query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
@@ -580,7 +689,14 @@ void OptimizerSimulator::dumpHistograms()
" FROM ";
query += name->getCatalogName();
query += ".";
- query += name->getSchemaName();
+ if(name->getSchemaName()[0]=='_')
+ {
+ query += "\"";
+ query += name->getSchemaName();
+ query += "\"";
+ }
+ else
+ query += name->getSchemaName();
query += ".SB_HISTOGRAM_INTERVALS WHERE TABLE_UID = ";
query += std::to_string((long long)(*tableUID)).c_str();
@@ -610,18 +726,161 @@ void OptimizerSimulator::dumpHistograms()
else if(retcode < 0 && -4082 != retcode)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg;
- errMsg = "Unload histogram data error: ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Unload histogram data error: %d", retcode);
+ }
+
+ }
+
+ //Do not use XMLFormatString as we do format ourself
+ XMLString* xmltext = new (STMTHEAP) XMLString(STMTHEAP);
+ histoInfoList->toXML(*xmltext);
+ (*writeLogStreams_[HISTOGRAM_PATHS]) << xmltext->data() << endl;
+ NADELETE(xmltext, XMLString, STMTHEAP);
+ //copy histograms data from hdfs to osim directory.
+ histogramHDFSToLocal();
+}
+
+void OptimizerSimulator::dumpHiveHistograms()
+{
+ short retcode;
+ const QualifiedName* name = NULL;
+ Int64* tableUID = NULL;
+ NAString query(STMTHEAP);
+
+ NAHashDictionaryIterator<const QualifiedName, Int64> iterator(*hashDict_HiveTables_);
+
+ OsimAllHistograms* histoInfoList = new (STMTHEAP) OsimAllHistograms(STMTHEAP);
+ NAString fullPath(STMTHEAP);
+ //enumerate captured table names and tableUIDs in hash table
+ for(iterator.getNext(name, tableUID); name && tableUID; iterator.getNext(name, tableUID))
+ {
+ //check if this table_uid is in TRAFODION."_HIVESTATS_".SB_HISTOGRAMS,
+ //if not, we consider this table has no histogram data.
+ Queue * outQueue = NULL;
+ query = "SELECT TABLE_UID FROM TRAFODION.\"_HIVESTATS_\".SB_HISTOGRAMS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+ retcode = fetchAllRowsFromMetaContext(outQueue, query.data());
+ if(retcode < 0 || outQueue && outQueue->entries() == 0)
+ continue;
+
+ debugMessage("Dumping histograms for %s\n", name->getQualifiedNameAsAnsiString().data());
+
+ //dump histograms data to hdfs
+ query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
+ query += "'"UNLOAD_HDFS_DIR"/";
+ query += ComUser::getCurrentUsername();
+ query += "/";
+ query += std::to_string((long long unsigned int)(getpid())).c_str();
+ query += "/";
+ query += name->getQualifiedNameAsAnsiString();
+ query += ".SB_HISTOGRAMS'";
+ query += " SELECT TABLE_UID"
+ ", HISTOGRAM_ID"
+ ", COL_POSITION"
+ ", COLUMN_NUMBER"
+ ", COLCOUNT"
+ ", INTERVAL_COUNT"
+ ", ROWCOUNT"
+ ", TOTAL_UEC"
+ ", STATS_TIME"
+ ", TRANSLATE(LOW_VALUE USING UCS2TOUTF8)"
+ ", TRANSLATE(HIGH_VALUE USING UCS2TOUTF8)"
+ ", READ_TIME"
+ ", READ_COUNT"
+ ", SAMPLE_SECS"
+ ", COL_SECS"
+ ", SAMPLE_PERCENT"
+ ", CV,REASON, V1, V2, V3, V4"
+ ", TRANSLATE(V5 USING UCS2TOUTF8)"
+ ", TRANSLATE(V6 USING UCS2TOUTF8)"
+ " FROM TRAFODION.\"_HIVESTATS_\".SB_HISTOGRAMS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+
+ retcode = executeFromMetaContext(query.data());
+ //succeed
+ if(retcode >= 0)
+ {
+ fullPath = osimLogLocalDir_;
+ fullPath += "/";
+ fullPath += ComUser::getCurrentUsername();
+ fullPath += "/";
+ fullPath += std::to_string((long long unsigned int)(getpid())).c_str();
+ fullPath += "/";
+ fullPath += name->getQualifiedNameAsAnsiString();
+ fullPath += ".SB_HISTOGRAMS";
+ histoInfoList->addEntry( fullPath.data(),
+ ComUser::getCurrentUsername(),
+ std::to_string((long long unsigned int)(getpid())).c_str(),
+ name->getCatalogName().data(),
+ name->getSchemaName().data(),
+ name->getObjectName().data(),
+ "SB_HISTOGRAMS");
+ }
+ //ignore -4082,
+ //which means histogram tables are not exist,
+ //i.e. update stats hasn't been done for any table.
+ else if(retcode < 0 && -4082 != retcode)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("Unload histogram data error: %d", retcode);
+ }
+
+ query = "UNLOAD WITH NULL_STRING '\\N' INTO ";
+ query += "'"UNLOAD_HDFS_DIR"/";
+ query += ComUser::getCurrentUsername();
+ query += "/";
+ query += std::to_string((long long unsigned int)(getpid())).c_str();
+ query += "/";
+ query += name->getQualifiedNameAsAnsiString();
+ query += ".SB_HISTOGRAM_INTERVALS'";
+ query += " SELECT TABLE_UID"
+ ", HISTOGRAM_ID"
+ ", INTERVAL_NUMBER"
+ ", INTERVAL_ROWCOUNT"
+ ", INTERVAL_UEC"
+ ", TRANSLATE(INTERVAL_BOUNDARY USING UCS2TOUTF8)"
+ ", STD_DEV_OF_FREQ"
+ ", V1, V2, V3, V4"
+ ", TRANSLATE(V5 USING UCS2TOUTF8)"
+ ", TRANSLATE(V6 USING UCS2TOUTF8)"
+ " FROM TRAFODION.\"_HIVESTATS_\".SB_HISTOGRAM_INTERVALS WHERE TABLE_UID = ";
+ query += std::to_string((long long)(*tableUID)).c_str();
+
+ retcode = executeFromMetaContext(query.data());
+
+ if(retcode >= 0)
+ {
+ fullPath = osimLogLocalDir_;
+ fullPath += "/";
+ fullPath += ComUser::getCurrentUsername();
+ fullPath += "/";
+ fullPath += std::to_string((long long unsigned int)(getpid())).c_str();
+ fullPath += "/";
+ fullPath += name->getQualifiedNameAsAnsiString();
+ fullPath += ".SB_HISTOGRAM_INTERVALS";
+ histoInfoList->addEntry( fullPath.data(),
+ ComUser::getCurrentUsername(),
+ std::to_string((long long unsigned int)(getpid())).c_str(),
+ name->getCatalogName().data(),
+ name->getSchemaName().data(),
+ name->getObjectName().data(),
+ "SB_HISTOGRAM_INTERVALS");
+ }
+ //ignore -4082,
+ //which means histogram tables are not exist,
+ //i.e. update stats hasn't been done for any table.
+ else if(retcode < 0 && -4082 != retcode)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("Unload histogram data error: %d", retcode);
}
}
- //Do not user XMLFormatString as we do format ourself
+ //Do not use XMLFormatString as we do format ourself
XMLString* xmltext = new (STMTHEAP) XMLString(STMTHEAP);
histoInfoList->toXML(*xmltext);
- (*writeSysCallStream_[HISTOGRAM_PATHS]) << xmltext->data() << endl;
+ (*writeLogStreams_[HIVE_HISTOGRAM_PATHS]) << xmltext->data() << endl;
NADELETE(xmltext, XMLString, STMTHEAP);
//copy histograms data from hdfs to osim directory.
histogramHDFSToLocal();
@@ -630,16 +889,10 @@ void OptimizerSimulator::dumpHistograms()
void OptimizerSimulator::dropObjects()
{
short retcode;
- ifstream tables(sysCallLogFilePath_[TABLESFILE]);
+ ifstream tables(logFilePaths_[TABLESFILE]);
if(!tables.good())
- {
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_TABLE_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- //ignore first 2 lines
- tables.ignore(OSIM_LINEMAX, '\n');
- tables.ignore(OSIM_LINEMAX, '\n');
+ raiseOsimException("Error open %s", logFilePaths_[TABLESFILE]);
+
std::string stdQualTblNm;//get qualified table name from file
NAString query(STMTHEAP);
while(tables.good())
@@ -654,200 +907,56 @@ void OptimizerSimulator::dropObjects()
query = "DROP TABLE IF EXISTS ";
query += stdQualTblNm.c_str();
query += " CASCADE;";
- debugMessage("DELETING %s ...\n", stdQualTblNm.c_str());
+ debugMessage("%s\n", query.data());
retcode = executeFromMetaContext(query.data());
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Drop Table " ;
- errMsg += stdQualTblNm.c_str();
- errMsg += " Error: ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- }
-}
-
-void OptimizerSimulator::checkDuplicateNames()
-{
- //Get captured qualied table names
- //and compare each with the names in hash dictionary.
- ifstream tables(sysCallLogFilePath_[TABLESFILE]);
- if(!tables.good())
- {
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_TABLE_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- //ignore first 2 lines
- tables.ignore(OSIM_LINEMAX, '\n');
- tables.ignore(OSIM_LINEMAX, '\n');
- std::string stdQualTblNm;
- NAString naQualTblNm(STMTHEAP);
- while(tables.good())
- {
- //get one qualified table name from file
- std::getline(tables, stdQualTblNm);
-
- // Exit the loop if there was no data to read.
- // eofbit is not set until an attempt is made to read beyond EOF.
- if(!tables.good())
- break;
-
- //Check if table name is in existance
- naQualTblNm = stdQualTblNm.c_str();
- if(hashDict_TablesBeforeAction_->contains(&naQualTblNm))
- {
- NAString errMsg = "Object " + naQualTblNm + " already exists.";
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Drop Table %s error: %d", stdQualTblNm.c_str(), retcode);
}
}
-}
-
-void OptimizerSimulator::saveViewsBeforeStart()
-{
- if(viewsBeforeActionInitilized_)
- return;
- //Ask _MD_.OBJECTS for a list of all qualified view names.
- initializeCLI();
-
- short retcode;
- if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
- }
-
- char * ptr = NULL;
- Lng32 len = 0;
-
- retcode = cliInterface_->fetchRowsPrologue("select catalog_name, schema_name, object_name "
- " from TRAFODION.\""SEABASE_MD_SCHEMA"\"."SEABASE_OBJECTS
- " where object_type = 'VI' "
- "and schema_name <> '_MD_' "
- "and schema_name <> '_REPOS_' "
- "and schema_name <> '_PRIVMGR_MD_' "
- "and object_name <> 'SB_HISTOGRAMS' "
- "and object_name <> 'SB_HISTOGRAM_INTERVALS'; ");
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing views, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
-
- while(1)
- {
- retcode = cliInterface_->fetch();
-
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing views, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- if (retcode == 100) //no more data
- break;
-
- NAString * qualifiedName = new (heap_) NAString(heap_);
- Int32 * dummy = new Int32(0);
- //append catalog name
- cliInterface_->getPtrAndLen(1,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append schema name
- cliInterface_->getPtrAndLen(2,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append table name
- cliInterface_->getPtrAndLen(3,ptr,len);
- qualifiedName->append(ptr, len);
-
- hashDict_ViewsBeforeAction_->insert(qualifiedName, dummy);
-
- }
- //end fetch
- retcode = cliInterface_->fetchRowsEpilogue(NULL);
-
- cmpSBD_->switchBackCompiler();
-
- viewsBeforeActionInitilized_ = TRUE;
-}
-
-void OptimizerSimulator::saveTablesBeforeStart()
-{
- if(tablesBeforeActionInitilized_)
- return;
- //Ask _MD_.OBJECTS for a list of all qualified table names.
- initializeCLI();
-
- short retcode;
- if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
- }
-
- char * ptr = NULL;
- Lng32 len = 0;
-
- retcode = cliInterface_->fetchRowsPrologue("select catalog_name, schema_name, object_name "
- " from TRAFODION.\""SEABASE_MD_SCHEMA"\"."SEABASE_OBJECTS
- " where object_type = 'BT' "
- "and schema_name <> '_MD_' "
- "and schema_name <> '_REPOS_' "
- "and schema_name <> '_PRIVMGR_MD_' "
- "and object_name <> 'SB_HISTOGRAMS' "
- "and object_name <> 'SB_HISTOGRAM_INTERVALS'; ");
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing tables, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
-
- while(1)
+ std::string str;
+ Int64 uid;
+ const QualifiedName * qualName = NULL;
+ Int64 * tableUID;
+ NAString trafName;
+ std::ifstream hiveTableListFile(logFilePaths_[HIVE_TABLE_LIST]);
+ //we only need one loop, no need to populate hashDict_HiveTables_
+ while(hiveTableListFile.good())
{
- retcode = cliInterface_->fetch();
-
- if (retcode < 0)
- {
- cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Get existing tables, error ";
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- if (retcode == 100) //no more data
+ // read tableName and uid from the file
+ hiveTableListFile >> str >> uid;
+ // eofbit is not set until an attempt is made to read beyond EOF.
+ // Exit the loop if there was no data to read above.
+ if(!hiveTableListFile.good())
break;
-
- NAString * qualifiedName = new (heap_) NAString(heap_);
- Int32 * dummy = new Int32(0);
- //append catalog name
- cliInterface_->getPtrAndLen(1,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append schema name
- cliInterface_->getPtrAndLen(2,ptr,len);
- qualifiedName->append(ptr, len);
- qualifiedName->append('.');
-
- //append table name
- cliInterface_->getPtrAndLen(3,ptr,len);
- qualifiedName->append(ptr, len);
-
- hashDict_TablesBeforeAction_->insert(qualifiedName, dummy);
-
+ NAString name = str.c_str();
+ qualName = new (heap_) QualifiedName(name,3);
+ trafName = ComConvertNativeNameToTrafName(qualName->getCatalogName(),
+ qualName->getSchemaName(),
+ qualName->getObjectName());
+ QualifiedName qualTrafName(trafName,3);
+ //drop external table
+ NAString dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += trafName;
+ debugMessage("%s\n", dropStmt.data());
+ retcode = executeFromMetaContext(dropStmt.data());
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("drop external table: %d", retcode);
+ }
+ //drop hive table
+ NAString hiveSchemaName;
+ qualName->getHiveSchemaName(hiveSchemaName);
+ dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += hiveSchemaName;
+ dropStmt += '.';
+ dropStmt += qualName->getObjectName();
+ debugMessage("%s\n", dropStmt.data());
+ execHiveSQL(dropStmt.data());//drop hive table
}
-
- //end fetch
- retcode = cliInterface_->fetchRowsEpilogue(NULL);
-
- cmpSBD_->switchBackCompiler();
-
- tablesBeforeActionInitilized_ = TRUE;
}
void OptimizerSimulator::loadDDLs()
@@ -857,29 +966,25 @@ void OptimizerSimulator::loadDDLs()
//If force option is present,
//drop tables with same names, otherwise rollback
- if(isForceLoad())
- dropObjects();
- else
- checkDuplicateNames();
-
+ //if(isForceLoad())
+ // dropObjects();
+ //else
+ // checkDuplicateNames();
+ dropObjects();
+
NAString statement(STMTHEAP);
NAString comment(STMTHEAP);
statement.capacity(4096);
comment.capacity(4096);
- //Step 1 :
+ //Step 1:
//Fetch and execute "create schema ..." from schema ddl file.
debugMessage("Step 1 Create Schemas:\n");
- ifstream createSchemas(sysCallLogFilePath_[CREATE_SCHEMA_DDLS]);
+ ifstream createSchemas(logFilePaths_[CREATE_SCHEMA_DDLS]);
if(!createSchemas.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_SCHEMA_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[CREATE_SCHEMA_DDLS]);
}
- //ignore first 2 lines
- createSchemas.ignore(OSIM_LINEMAX, '\n');
- createSchemas.ignore(OSIM_LINEMAX, '\n');
while(readStmt(createSchemas, statement, comment))
{
if(comment.length() > 0)
@@ -892,16 +997,11 @@ void OptimizerSimulator::loadDDLs()
//Step 2:
//Fetch and execute "create table ... " from table ddl file.
debugMessage("Step 2 Create Tables:\n");
- ifstream createTables(sysCallLogFilePath_[CREATE_TABLE_DDLS]);
+ ifstream createTables(logFilePaths_[CREATE_TABLE_DDLS]);
if(!createTables.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CREATE_TABLE_DDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[CREATE_TABLE_DDLS]);
}
- //ignore first 2 lines
- createTables.ignore(OSIM_LINEMAX, '\n');
- createTables.ignore(OSIM_LINEMAX, '\n');
while(readStmt(createTables, statement, comment))
{
if(comment.length() > 0)
@@ -911,26 +1011,20 @@ void OptimizerSimulator::loadDDLs()
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Create Table Error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Create Table Error: %d", retcode);
}
}
}
//Step 3:
//Fetch and execute "create view ..." from view ddl file.
- debugMessage("Step 3 Create Views:");
- ifstream createViews(sysCallLogFilePath_[VIEWDDLS]);
+ debugMessage("Step 3 Create Views:\n");
+ ifstream createViews(logFilePaths_[VIEWDDLS]);
if(!createViews.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[VIEWDDLS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[VIEWDDLS]);
}
- //ignore first 2 lines
- createViews.ignore(OSIM_LINEMAX, '\n');
- createViews.ignore(OSIM_LINEMAX, '\n');
+
while(readStmt(createViews, statement, comment))
{
if(comment.length() > 0)
@@ -940,16 +1034,377 @@ void OptimizerSimulator::loadDDLs()
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Create View Error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- errMsg += statement;
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Create View Error: %d %s", retcode, statement.data());
}
}
}
}
-NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashDictionary<NAString, QualifiedName> * modifiedPathList)
+static const char* extractAsComment(const char* header, const NAString & stmt)
+{
+ NAString tmp;
+ int begin = stmt.index(header);
+ if(begin > -1)
+ {
+ int end = stmt.index('\n', begin);
+ if(end > begin)
+ {
+ stmt.extract(begin, end-1, tmp);
+ return tmp.data();
+ }
+ }
+ return NULL;
+}
+
+void OptimizerSimulator::loadHiveDDLs()
+{
+ debugMessage("creating hive tables ...\n");
+ short retcode;
+ NAString statement(STMTHEAP);
+ NAString comment(STMTHEAP);
+ statement.capacity(4096);
+ comment.capacity(4096);
+ std::ifstream hiveCreateTableSql(logFilePaths_[HIVE_CREATE_TABLE]);
+ std::ifstream hiveTableListFile(logFilePaths_[HIVE_TABLE_LIST]);
+ std::ifstream hiveCreateExternalTableSql(logFilePaths_[HIVE_CREATE_EXTERNAL_TABLE]);
+ if(!hiveTableListFile.good() || !hiveCreateTableSql.good())
+ return;
+
+ //read hive sql file and create hive table
+ std::string str;
+ Int64 uid;
+ const QualifiedName * qualName = NULL;
+ Int64 * tableUID;
+ int counter = 0;
+ NAString trafName;
+
+ while(hiveTableListFile.good())
+ {
+ // read tableName and uid from the file
+ hiveTableListFile >> str >> uid;
+ // eofbit is not set until an attempt is made to read beyond EOF.
+ // Exit the loop if there was no data to read above.
+ if(!hiveTableListFile.good())
+ break;
+ NAString name = str.c_str();
+ qualName = new (heap_) QualifiedName(name,3);
+ tableUID = new Int64(uid);
+ hashDict_HiveTables_->insert(qualName, tableUID);
+ }
+
+ NAHashDictionaryIterator<const QualifiedName, Int64> iterator(*hashDict_HiveTables_);
+ //create hive schema and trafodion external schema and
+ //drop external tables and hive tables with same names
+ for(iterator.getNext(qualName, tableUID); qualName && tableUID; iterator.getNext(qualName, tableUID))
+ {
+ trafName = ComConvertNativeNameToTrafName(
+ qualName->getCatalogName(),
+ qualName->getSchemaName(),
+ qualName->getObjectName());
+
+ QualifiedName qualTrafName(trafName,3);
+ //create external table schema
+ NAString create_ext_schema = "CREATE SCHEMA IF NOT EXISTS ";
+ create_ext_schema += qualTrafName.getCatalogName();
+ create_ext_schema += ".\"";
+ create_ext_schema += qualTrafName.getSchemaName();
+ create_ext_schema += "\" AUTHORIZATION DB__ROOT";
+ retcode = executeFromMetaContext(create_ext_schema);
+ if(retcode < 0) {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("create hive external schema: %d %s", retcode, statement.data());
+ }
+ //drop external table
+ NAString dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += trafName;
+ retcode = executeFromMetaContext(dropStmt.data());
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("drop external table: %d", retcode);
+ }
+
+ //create hive schema
+ NAString hiveSchemaName;
+ qualName->getHiveSchemaName(hiveSchemaName);
+ NAString create_hive_schema = "CREATE SCHEMA IF NOT EXISTS ";
+ create_hive_schema += hiveSchemaName;
+ execHiveSQL(create_hive_schema.data());
+ //drop hive table
+ dropStmt = "DROP TABLE IF EXISTS ";
+ dropStmt += hiveSchemaName;
+ dropStmt += '.';
+ dropStmt += qualName->getObjectName();
+ execHiveSQL(dropStmt.data());//drop hive table
+ }
+ //create hive table
+ debugMessage("Begin creating hive tables\n");
+
+ while(readHiveStmt(hiveCreateTableSql, statement, comment))
+ {
+ if(statement.length() > 0)
+ {
+ debugMessage("%s\n", extractAsComment("CREATE TABLE", statement));
+ execHiveSQL(statement.data());//create hive table
+ debugMessage("done\n");
+ }
+ }
+
+ debugMessage("Begin creating hive external tables\n");
+
+ //create external table
+ while(readHiveStmt(hiveCreateExternalTableSql, statement, comment))
+ {
+ if(statement.length() > 0) {
+ debugMessage("%s\n", extractAsComment("CREATE EXTERNAL TABLE", statement));
+ retcode = executeFromMetaContext(statement.data()); //create hive external table
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("Create hive external table error: %d", retcode);
+ }
+ debugMessage("done\n");
+ }
+ }
+}
+
+//============================================================================
+// This method writes the information related to the NAClusterInfo class to a
+// logfile called "NAClusterInfo.txt".
+//============================================================================
+void NAClusterInfo::captureNAClusterInfo(ofstream & naclfile)
+{
+ CollIndex i, ci;
+ char filepath[OSIM_PATHMAX];
+ char filename[OSIM_FNAMEMAX];
+
+ // We don't capture data members that are computed during the compilation of
+ // a query. These include:
+ //
+ // * smpCount_;
+ // * tableToClusterMap_;
+ // * activeClusters_;
+ //
+
+ naclfile << "localCluster_: " << localCluster_ << endl
+ << "localSMP_: " << localSMP_ << endl;
+
+ CollIndex *key_collindex;
+ maps *val_maps;
+ // Iterator for logging all the entries in clusterToCPUMap_ HashDictionary.
+ NAHashDictionaryIterator<CollIndex, maps> C2CPUIter (*clusterToCPUMap_, NULL, NULL);
+ naclfile << "clusterToCPUMap_: " << C2CPUIter.entries() << " :" << endl;
+ if (C2CPUIter.entries() > 0)
+ {
+ // Write the header line for the table.
+ naclfile << " ";
+ naclfile.width(10);
+ naclfile << "clusterNum" << " ";
+ naclfile << "cpuList" << endl;
+ for (i=0; i<C2CPUIter.entries(); i++)
+ {
+ C2CPUIter.getNext(key_collindex, val_maps);
+ naclfile << " ";
+ naclfile.width(10); naclfile << *key_collindex << " ";
+ naclfile << val_maps->list->entries() << " : ";
+ for (ci=0; ci<val_maps->list->entries(); ci++)
+ {
+ naclfile.width(3); naclfile << (*(val_maps->list))[ci] << " ";
+ }
+ naclfile << endl;
+ }
+ }
+
+ Int32 * nodeID = NULL;
+ NAString* nodeName = NULL;
+ NAHashDictionaryIterator<Int32, NAString> nodeNameAndIDIter (*nodeIdToNodeNameMap_);
+ naclfile << "nodeIdAndNodeNameMap: " << nodeNameAndIDIter.entries() << endl;
+ for(nodeNameAndIDIter.getNext(nodeID, nodeName); nodeID && nodeName; nodeNameAndIDIter.getNext(nodeID, nodeName))
+ {
+ naclfile << *nodeID << " " << nodeName->data() << endl;
+ }
+
+ // Now save the OS-specific information to the NAClusterInfo.txt file
+ captureOSInfo(naclfile);
+}
+
+//============================================================================
+// This method reads the information needed for NAClusterInfo class from
+// a logfile called "NAClusterInfo.txt" and then populates the variables
+// accordigly.
+//============================================================================
+void NAClusterInfo::simulateNAClusterInfo()
+{
+ Int32 i, ci;
+ char var[256];
+
+ const char* filepath = CURRCONTEXT_OPTSIMULATOR->getLogFilePath(OptimizerSimulator::NACLUSTERINFO);
+
+ activeClusters_= NULL;
+ physicalSMPCount_ = -1;
+
+ ifstream naclfile(filepath);
+
+ if(!naclfile.good())
+ {
+ raiseOsimException("Unable to open %s file for reading data.", filepath);
+ }
+
+ while(naclfile.good())
+ {
+ // Read the variable name from the file.
+ naclfile.getline(var, sizeof(var), ':');
+ if(!strcmp(var, "localCluster_"))
+ {
+ naclfile >> localCluster_; naclfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "localSMP_"))
+ {
+ naclfile >> localSMP_; naclfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "clusterToCPUMap_"))
+ {
+ Int32 C2CPU_entries, clusterNum, cpuList_entries, cpuNum;
+
+ clusterToCPUMap_ = new(heap_) NAHashDictionary<CollIndex,maps>(&clusterNumHashFunc,17,TRUE, heap_);
+ naclfile >> C2CPU_entries; naclfile.ignore(OSIM_LINEMAX, '\n');
+ if(C2CPU_entries > 0)
+ {
+ // Read and ignore the header line.
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ for (i=0; i<C2CPU_entries; i++)
+ {
+ naclfile >> clusterNum;
+ naclfile >> cpuList_entries; naclfile.ignore(OSIM_LINEMAX, ':');
+ CollIndex *key_clusterNum = new(heap_) CollIndex(clusterNum);
+ maps *val_cpuList = new(heap_) maps(heap_);
+ for (ci=0; ci<cpuList_entries; ci++)
+ {
+ naclfile >> cpuNum;
+ val_cpuList->list->insert(cpuNum);
+ }
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ CollIndex *checkClusterNum = clusterToCPUMap_->insert(key_clusterNum, val_cpuList);
+ CMPASSERT(checkClusterNum);
+ }
+ }
+ }
+ else if(!strcmp(var, "nodeIdAndNodeNameMap"))
+ {
+ Int32 id_name_entries;
+ Int32 nodeId;
+ char nodeName[256];
+ nodeIdToNodeNameMap_ = new(heap_) NAHashDictionary<Int32, NAString>
+ (&intHashFunc, 101,TRUE,heap_);
+
+ nodeNameToNodeIdMap_ = new(heap_) NAHashDictionary<NAString, Int32>
+ (&NAString::hash, 101,TRUE,heap_);
+ naclfile >> id_name_entries;
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ physicalSMPCount_ = id_name_entries;
+ for(i = 0; i < id_name_entries; i++)
+ {
+ naclfile >> nodeId >> nodeName;
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+
+ //populate clusterId<=>clusterName map from file
+ Int32 * key_nodeId = new Int32(nodeId);
+ NAString * val_nodeName = new (heap_) NAString(nodeName, heap_);
+ Int32 * retId = nodeIdToNodeNameMap_->insert(key_nodeId, val_nodeName);
+ //CMPASSERT(retId);
+
+ NAString * key_nodeName = new (heap_) NAString(nodeName, heap_);
+ Int32 * val_nodeId = new Int32(nodeId);
+ NAString * retName = nodeNameToNodeIdMap_->insert(key_nodeName, val_nodeId);
+ //some node names are like g4t3024:0, g4t3024:1
+ //I don't know why we need to remove strings after ':' or '.' in node name,
+ //but if string after ':' or '.' is removed, same node names correspond to different node ids,
+ //this can cause problems here
+ //CMPASSERT(retName);
+ }
+ }
+ else
+ {
+ // This variable will either be read in simulateNAClusterInfoNSK()
+ // method of NAClusterInfoNSK class or is not the one that we want
+ // to read here in this method. So discard it and continue.
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ while (naclfile.peek() == ' ')
+ {
+ // The main variables are listed at the beginning of a line
+ // with additional information indented. If one or more spaces
+ // are seen at the beginning of the line upon the entry to this
+ // while loop, it is because of that additional information.
+ // So, ignore this line since the variable is being ignored.
+ naclfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ }
+ }
+}
+
+void NAClusterInfoLinux::simulateNAClusterInfoLinux()
+{
+ char var[256];
+
+ const char* filepath = CURRCONTEXT_OPTSIMULATOR->getLogFilePath(OptimizerSimulator::NACLUSTERINFO);
+
+ ifstream nacllinuxfile(filepath);
+
+ if(!nacllinuxfile.good())
+ {
+ raiseOsimException("Unable to open %s file for reading data.", filepath);
+ }
+
+ while(nacllinuxfile.good())
+ {
+ // Read the variable name from the file
+ nacllinuxfile.getline(var, sizeof(var), ':');
+ if(!strcmp(var, "frequency_"))
+ {
+ nacllinuxfile >> frequency_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "iorate_"))
+ {
+ nacllinuxfile >> iorate_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "seekTime_"))
+ {
+ nacllinuxfile >> seekTime_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "pageSize_"))
+ {
+ nacllinuxfile >> pageSize_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "totalMemoryAvailable_"))
+ {
+ nacllinuxfile >> totalMemoryAvailable_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else if (!strcmp(var, "numCPUcoresPerNode_"))
+ {
+ nacllinuxfile >> numCPUcoresPerNode_; nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ else
+ {
+ // This variable either may have been read in simulateNAClusterInfo()
+ // method of NAClusterInfo class or is not the one that we want to
+ // read here in this method. So discard it.
+ nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ while (nacllinuxfile.peek() == ' ')
+ {
+ // The main variables are listed at the beginning of a line
+ // with additional information indented. If one or more spaces
+ // are seen at the beginning of the line upon the entry to this
+ // while loop, it is because of that additional information.
+ // So, ignore this line since the variable is being ignored.
+ nacllinuxfile.ignore(OSIM_LINEMAX, '\n');
+ }
+ }
+ }
+}
+
+//use local table UID to replace UID in captured histogram file,
+//then read from files to histogram.
+NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashDictionary<NAString, QualifiedName> * modifiedPathList, NABoolean isHive)
{
int retcode;
NAString tmp = osimLogLocalDir_ + '/';
@@ -967,36 +1422,48 @@ NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashD
const char* histogramTableName = entry->getHistogram();
NAString * UIDModifiedPath = new (STMTHEAP) NAString(STMTHEAP);
+ //qualifiedName is used to create histogram tables
+ QualifiedName* qualifiedName;
+
+ qualifiedName = new (STMTHEAP) QualifiedName(histogramTableName, schema, catalog, STMTHEAP);
+
+ Int64 tableUID;
+ //in _MD_.OBJECTS, schema of hive table is _HV_HIVE_, catalog is TRAFODION
+ if(isHive)
+ {
+ NAString trafName;
+ trafName = ComConvertNativeNameToTrafName(
+ entry->getCatalog(),
+ entry->getSchema(),
+ entry->getTable());
+ QualifiedName qname (trafName, 3);
+ tableUID = getTableUID(qname.getCatalogName(), qname.getSchemaName(), qname.getObjectName());
+ }
+ else
+ tableUID = getTableUID(catalog, schema, table);
- QualifiedName* qualifiedName = new (STMTHEAP) QualifiedName(histogramTableName, schema, catalog, STMTHEAP);
-
- Int64 tableUID = getTableUID(catalog, schema, table);
if(tableUID < 0)
{
- NAString errMsg = "Get Table UID Error: " ;
- errMsg += std::to_string((long long)(tableUID)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Get Table UID Error: %d", tableUID);
}
NAString dataPath(STMTHEAP);
//get text file path within the dir
DIR * histogramDir = opendir(fullPath);
if(!histogramDir)
{
- NAString errMsg = "Error open ";
- errMsg += fullPath;
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", fullPath);
}
struct dirent * dataPathInfo = readdir(histogramDir);
while(dataPathInfo != NULL)
{
- if(dataPathInfo->d_name[0] != '.')
+ if(dataPathInfo->d_name[0] != '.' && dataPathInfo->d_type != DT_DIR)
{//there should be only one
dataPath = fullPath;
dataPath += '/';
dataPath += dataPathInfo->d_name;
break;
}
- dataPathInfo = readdir(histogramDir);
+ dataPathInfo = readdir(histogramDir);
}
closedir(histogramDir);
@@ -1020,9 +1487,7 @@ NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashD
std::ifstream infile (dataPath.data(), std::ifstream::binary);
if(!infile.good())
{
- NAString errMsg = "Error open ";
- errMsg += dataPath;
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", dataPath.data());
}
std::ofstream outfile (UIDModifiedPath->data(), std::ofstream::binary|std::ofstream::app);
//update table UID between files
@@ -1059,48 +1524,9 @@ NABoolean OptimizerSimulator::massageTableUID(OsimHistogramEntry* entry, NAHashD
outfile << V6 << endl;
}
else
- OsimLogException("Invalid format of histogram data.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Invalid format of histogram data.");
}
-#if 0
- enum workState { WRITEUID, READUID, RESTOFLINE };
- workState state = READUID;
- char a = ' ';
- char uidstr[256];
- snprintf(uidstr, 256, "%ld", tableUID);
- while(1)
- {
- switch(state)
- {
- case READUID :
- infile.get (a);
- if(infile.eof())
- return TRUE;
- else if('|' == a)
- state = WRITEUID;
- break;
- case WRITEUID :
- outfile.write(uidstr, strlen(uidstr));
- outfile.put(a);
- state = RESTOFLINE;
- break;
-
- case RESTOFLINE :
- infile.get (a);
- if(infile.eof())
- return TRUE;
- else if('\n' == a){
- outfile.put(a);
- state = READUID;
- }
- else if('\0' == a)
- continue;
- else
- outfile.put(a);
- break;
- }//switch
- }//while
-#endif
return TRUE;
}
@@ -1109,34 +1535,24 @@ void OptimizerSimulator::execHiveSQL(const char* hiveSQL)
HiveClient_JNI *hiveClient = CmpCommon::context()->getHiveClient();
if (hiveClient == NULL)
- {
- NAString errMsg;
- errMsg = "Error initialize hive client.";
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
+ raiseOsimException("Error initialize hive client.");
else
- {
- if (!CmpCommon::context()->execHiveSQL(hiveSQL))
- {
- NAString errMsg;
- errMsg = "Error running hive SQL.";
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
- }
- }
+ if (!CmpCommon::context()->execHiveSQL(hiveSQL))
+ raiseOsimException("Error running hive SQL.");
}
-short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen)
+short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen, NABoolean isHive)
{
debugMessage("loading %s\n", qualifiedName->getQualifiedNameAsString().data());
short retcode;
NAString cmd(STMTHEAP);
-
+ //drop hive sb_histogram table
cmd = "drop table "+qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
execHiveSQL(cmd.data());
-
+ //create hive sb_histogram table
cmd = "create table " + qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
- cmd += " ( table_uid bigint"
+ cmd += " (table_uid bigint"
",histogram_id int"
",col_position int"
",column_number int"
@@ -1163,13 +1579,17 @@ short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedN
" ) row format delimited fields terminated by '|' "
"tblproperties ('serialization.null.format' = '\\N')";
execHiveSQL(cmd.data());
-
+ //populate hive table with table UID modified file
cmd = "load data local inpath '" + *modifiedPath + "' into table ";
cmd += qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
execHiveSQL(cmd.data());
//create sb_histograms
- cmd = "CREATE TABLE IF NOT EXISTS " + qualifiedName->getQualifiedNameAsString();
+ cmd = "CREATE TABLE IF NOT EXISTS ";
+ if(isHive)
+ cmd += "TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd += qualifiedName->getQualifiedNameAsString();
cmd += " ( TABLE_UID LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , HISTOGRAM_ID INT UNSIGNED NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , COL_POSITION INT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
@@ -1201,36 +1621,43 @@ short OptimizerSimulator::loadHistogramsTable(NAString* modifiedPath, QualifiedN
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
- cmd = "upsert using load into " + qualifiedName->getQualifiedNameAsString() + " select * from hive.hive.";
- cmd += qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
+ cmd = "upsert using load into ";
+ if(isHive)
+ cmd += "TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd += qualifiedName->getQualifiedNameAsString();
+ //from hive table to trafodion table
+ cmd +=" select TABLE_UID, HISTOGRAM_ID, COL_POSITION, COLUMN_NUMBER, "
+ "COLCOUNT, INTERVAL_COUNT, ROWCOUNT, TOTAL_UEC, STATS_TIME, "
+ "CAST(LOW_VALUE AS VARCHAR(250) CHARACTER SET UCS2 NOT NULL), "
+ "CAST(HIGH_VALUE AS VARCHAR(250) CHARACTER SET UCS2 NOT NULL), "
+ "READ_TIME, READ_COUNT, SAMPLE_SECS, COL_SECS, SAMPLE_PERCENT, "
+ "CV, REASON, V1, V2, V3, V4, V5, V6 from hive.hive.";
+ cmd +=qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
retcode = executeFromMetaContext(cmd.data());
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
return retcode;
}
-short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen)
+short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, QualifiedName * qualifiedName, unsigned int bufLen, NABoolean isHive)
{
debugMessage("loading %s\n", qualifiedName->getQualifiedNameAsString().data());
short retcode;
NAString cmd(STMTHEAP);
-
+ //drop hive histogram table
cmd = "drop table "+qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
execHiveSQL(cmd.data());
-
+ //create hive histogram table
cmd = "create table " + qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
cmd += " ( table_uid bigint"
",histogram_id int"
@@ -1254,7 +1681,11 @@ short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, Qu
execHiveSQL(cmd.data());
//create sb_histogram_intervals
- cmd = "CREATE TABLE IF NOT EXISTS " + qualifiedName->getQualifiedNameAsString();
+ cmd = "CREATE TABLE IF NOT EXISTS ";
+ if(isHive)
+ cmd +="TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd += qualifiedName->getQualifiedNameAsString();
cmd += " ( TABLE_UID LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , HISTOGRAM_ID INT UNSIGNED NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
" , INTERVAL_NUMBER SMALLINT NO DEFAULT NOT NULL NOT DROPPABLE NOT SERIALIZED"
@@ -1276,42 +1707,39 @@ short OptimizerSimulator::loadHistogramIntervalsTable(NAString* modifiedPath, Qu
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
-
- cmd = "upsert using load into " + qualifiedName->getQualifiedNameAsString() + " select * from hive.hive.";
+ //from hive table to trafodion table.
+ cmd = "upsert using load into ";
+ if(isHive)
+ cmd +="TRAFODION.\"_HIVESTATS_\"." + qualifiedName->getObjectName();
+ else
+ cmd +=qualifiedName->getQualifiedNameAsString();
+ //from hive table to trafodion table
+ cmd += " select * from hive.hive.";
cmd += qualifiedName->getCatalogName()+"_"+qualifiedName->getSchemaName()+"_"+qualifiedName->getObjectName();
retcode = executeFromMetaContext(cmd.data());
if(retcode < 0)
{
CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
- NAString errMsg = "Load histogram data error: " ;
- errMsg += std::to_string((long long)(retcode)).c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Load histogram data error: %d", retcode);
}
-
return retcode;
}
-void OptimizerSimulator::loadHistograms()
+void OptimizerSimulator::loadHistograms(const char* histogramPath, NABoolean isHive)
{
debugMessage("loading histograms ...\n");
OsimElementMapper om;
OsimAllHistograms * allHistograms = NULL;
XMLDocument doc(STMTHEAP, om);
- std::ifstream s (sysCallLogFilePath_[HISTOGRAM_PATHS], std::ifstream::binary);
+ std::ifstream s (histogramPath, std::ifstream::binary);
if(!s.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[HISTOGRAM_PATHS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", histogramPath);
}
char * txt = new (STMTHEAP) char[1024];
- s.ignore(OSIM_LINEMAX, '\n');
- s.ignore(OSIM_LINEMAX, '\n');
s.read(txt, 1024);
while(s.gcount() > 0)
{
@@ -1326,41 +1754,52 @@ void OptimizerSimulator::loadHistograms()
}
if(!allHistograms)
{
- NAString errMsg = "Error parsing ";
- errMsg += sysCallLogFilePath_[HISTOGRAM_PATHS];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error parsing %s", histogramPath);
}
NAHashDictionary<NAString, QualifiedName> * modifiedPathDict =
new(STMTHEAP) NAHashDictionary<NAString, QualifiedName>
(&NAString::hash, 101, TRUE, STMTHEAP);
- for(CollIndex i = 0; i < allHistograms->getEntries().entries(); i++)
+ for(CollIndex i = 0; i < allHistograms->getHistograms().entries(); i++)
{
- OsimHistogramEntry * en = (allHistograms->getEntries())[i];
- massageTableUID(en, modifiedPathDict);
+ OsimHistogramEntry * en = (allHistograms->getHistograms())[i];
+ massageTableUID(en, modifiedPathDict, isHive);
}
-
+
//do load
NAHashDictionaryIterator<NAString, QualifiedName> iterator(*modifiedPathDict);
+ //create _HIVESTATS_ schema
+ if(isHive && iterator.entries() > 0)
+ {
+ //create hive stats schema
+ const char * create_stats_schema = "CREATE SCHEMA IF NOT EXISTS "
+ "TRAFODION.\"_HIVESTATS_\" AUTHORIZATION DB__ROOT";
+ short retcode = executeFromMetaContext(create_stats_schema);
+ if(retcode < 0)
+ {
+ CmpCommon::diags()->mergeAfter(*(cliInterface_->getDiagsArea()));
+ raiseOsimException("create hive stats schema: return code %d", retcode);
+ }
+ }
NAString* modifiedPath = NULL;
QualifiedName* qualifiedName = NULL;
Queue * dummyQueue = NULL;
- iterator.getNext(modifiedPath, qualifiedName) ;
- while ( modifiedPath && qualifiedName )
+ for (iterator.getNext(modifiedPath, qualifiedName); modifiedPath && qualifiedName; iterator.getNext(modifiedPath, qualifiedName))
{
if(qualifiedName->getObjectName().compareTo("SB_HISTOGRAMS", NAString::ignoreCase) == 0)
{
- loadHistogramsTable(modifiedPath, qualifiedName, OSIM_LINEMAX);
+ loadHistogramsTable(modifiedPath, qualifiedName, OSIM_LINEMAX, isHive);
}
else if(qualifiedName->getObjectName().compareTo("SB_HISTOGRAM_INTERVALS", NAString::ignoreCase) == 0)
{
- loadHistogramIntervalsTable(modifiedPath, qualifiedName, OSIM_LINEMAX);
+ loadHistogramIntervalsTable(modifiedPath, qualifiedName, OSIM_LINEMAX, isHive);
}
unlink(modifiedPath->data());
- iterator.getNext(modifiedPath, qualifiedName);
}
}
+
+
void OptimizerSimulator::initializeCLI()
{
if(!CLIInitialized_)
@@ -1376,24 +1815,17 @@ void OptimizerSimulator::readAndSetCQDs()
{
initializeCLI();
NABoolean isDir;
- if(!fileExists(sysCallLogFilePath_[CQD_DEFAULTSFILE], isDir))
+ if(!isFileExists(logFilePaths_[CQD_DEFAULTSFILE], isDir))
{
- char errMsg[38+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.",
- sysCallLogFilePath_[CQD_DEFAULTSFILE]);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[CQD_DEFAULTSFILE]);
}
- ifstream inLogfile(sysCallLogFilePath_[CQD_DEFAULTSFILE]);
+ ifstream inLogfile(logFilePaths_[CQD_DEFAULTSFILE]);
if(!inLogfile.good())
{
- NAString errMsg = "Error open ";
- errMsg += sysCallLogFilePath_[CQD_DEFAULTSFILE];
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error open %s", logFilePaths_[CQD_DEFAULTSFILE]);
}
- // Read and ignore the top 2 header lines.
- inLogfile.ignore(OSIM_LINEMAX, '\n');
- inLogfile.ignore(OSIM_LINEMAX, '\n');
+
Lng32 retcode;
std::string cqd;
while(inLogfile.good())
@@ -1408,9 +1840,7 @@ void OptimizerSimulator::readAndSetCQDs()
if(retcode < 0)
{
cliInterface_->retrieveSQLDiagnostics(CmpCommon::diags());
- NAString errMsg = "Error Setting CQD: ";
- errMsg += cqd.c_str();
- OsimLogException(errMsg.data(), __FILE__, __LINE__).throwException();
+ raiseOsimException("Error Setting CQD: %s", cqd.c_str());
}
}
}
@@ -1421,7 +1851,7 @@ Int64 OptimizerSimulator::getTableUID(const char * catName, const char * schName
Int64 retcode;
if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Switch Context.");
}
retcode = cmpSBD_->getObjectUID(cliInterface_, catName, schName, objName, "BT");
@@ -1437,7 +1867,7 @@ short OptimizerSimulator::fetchAllRowsFromMetaContext(Queue * &q, const char* qu
short retcode;
if (cmpSBD_->switchCompiler(CmpContextInfo::CMPCONTEXT_TYPE_META)) {
- OsimLogException("Errors Switch Context.", __FILE__, __LINE__).throwException();
+ raiseOsimException("Errors Switch Context.");
}
retcode = cliInterface_->fetchAllRows(queue_, query, 0, FALSE, FALSE, TRUE);
@@ -1524,6 +1954,77 @@ NABoolean OptimizerSimulator::readStmt(ifstream & DDLFile, NAString & stmt, NASt
}
}
+//Get a complete SQL statement and a line of comment in front of the SQL statement
+NABoolean OptimizerSimulator::readHiveStmt(ifstream & DDLFile, NAString & stmt, NAString & comment)
+{
+ char a = ' ';
+ long index = 0;
+ stmt = "";
+ comment = "";
+ enum readState
+ {
+ PROBABLY_COMMENT, CONSUME, EAT_CHAR, EOSTMT, EOFILE
+ };
+ readState state = EAT_CHAR;
+ while(1)
+ {
+ switch(state)
+ {
+ case EAT_CHAR:
+ DDLFile.get(a);
+ if(DDLFile.eof())
+ state = EOFILE;
+ else if(a == '/')
+ state = PROBABLY_COMMENT;
+ else if(a == ';') //end of statement
+ state = EOSTMT;
+ else
+ stmt += a;
+ break;
+ case PROBABLY_COMMENT :
+ {
+ char b = ' ';
+ DDLFile.get(b);
+ if( b == '*' )
+ state = CONSUME;
+ else //not comment
+ {
+ stmt += a;
+ stmt += b;
+ state = EAT_CHAR;
+ }
+ break;
+ }
+ case CONSUME:
+ //comment line, eat up rest of the line
+ while(DDLFile.get(a))
+ {
+ if(a == '*'){
+ char b = ' ';
+ DDLFile.get(b);
+ if(b == '/') //end of comment
+ state = EAT_CHAR;
+ else
+ comment += b;
+ break;
+ }
+ else if(DDLFile.eof()){
+ state = EOFILE;
+ break;
+ }
+ else
+ comment += a;
+ }
+ break;
+ case EOSTMT:
+ return TRUE;
+ case EOFILE:
+ return FALSE;
+ }
+ }
+}
+
+
void OptimizerSimulator::histogramHDFSToLocal()
{
Int32 status;
@@ -1566,19 +2067,14 @@ void OptimizerSimulator::histogramHDFSToLocal()
status = hdfsCopy(hdfs, src.data(), locfs, dst.data());
if(status != 0)
{
- NAString errMsg;
- errMsg = "Error getting histogram data from ";
- errMsg += src + " to " + dst;
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Error getting histogram data from %s to %s", src.data(), dst.data());
}
}
if( hdfsDisconnect(locfs) != 0 ||
hdfsDisconnect(hdfs) != 0)
{
- NAString errMsg;
- errMsg = "Error getting histogram data, disconneting";
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Error getting histogram data, disconneting");
}
}
@@ -1609,51 +2105,31 @@ void OptimizerSimulator::createLogDir()
{
case EACCES:
{
- char errMsg[37+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, permission denied.",
- osimLogLocalDir_.data());
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, permission denied.", osimLogLocalDir_.data());
}
break;
case ENOENT:
{
- char errMsg[58+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, a component of the path does not exist.",
- osimLogLocalDir_.data());
-
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, a component of the path does not exist.", osimLogLocalDir_.data());
}
break;
case EROFS:
{
- char errMsg[40+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, read-only filesystem.",
- osimLogLocalDir_.data());
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, read-only filesystem.", osimLogLocalDir_.data());
}
break;
case ENOTDIR:
{
- char errMsg[62+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create directory %s, a component of the path is not a directory.",
- osimLogLocalDir_.data());
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create directory %s, a component of the path is not a directory.", osimLogLocalDir_.data());
}
break;
default:
{
- char errMsg[58+OSIM_PATHMAX+1];
- snprintf (errMsg, sizeof(errMsg),
- "Could not create %s, errno is %d",
- osimLogLocalDir_.data(), error);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
+ raiseOsimException("Could not create %s, errno is %d", osimLogLocalDir_.data(), error);
}
break;
}
+ rval = mkdir(hiveTableStatsDir_.data(), S_IRWXU | S_IRWXG );
}
void OptimizerSimulator::readSysCallLogfiles()
@@ -1664,6 +2140,13 @@ void OptimizerSimulator::readSysCallLogfiles()
readLogFile_captureSysType();
}
+void OptimizerSimulator::setOsimLogdir(const char *localdir) {
+ if (localdir) {
+ osimLogLocalDir_ = localdir;
+ hiveTableStatsDir_ = osimLogLocalDir_ + OSIM_HIVE_TABLE_STATS_DIR;
+ }
+}
+
void OptimizerSimulator::initHashDictionaries()
{
// Initialize hash dictionary variables for all the system calls.
@@ -1682,48 +2165,37 @@ void OptimizerSimulator::initHashDictionaries()
hashDict_Synonyms_ = new(heap_) NAHashDictionary<const QualifiedName, Int32>
(&QualifiedName::hash, 101, TRUE, heap_);
- hashDict_TablesBeforeAction_ = new (heap_) NAHashDictionary<NAString, Int32>
- (&NAString::hash, 101, TRUE, heap_);
-
- hashDict_ViewsBeforeAction_= new (heap_) NAHashDictionary<NAString, Int32>
- (&NAString::hash, 101, TRUE, heap_);
+ hashDict_HiveTables_ = new(heap_) NAHashDictionary<const QualifiedName, Int64>
+ (&QualifiedName::hash, 101, TRUE, heap_);
hashDictionariesInitialized_ = TRUE;
}
}
-void OptimizerSimulator::setLogFilepath(sysCall sc)
+void OptimizerSimulator::createLogFilepath(OsimLog sc)
{
// Allocate memory for file pathname:
// dirname + '/' + syscallname + ".txt" + '\0'
- size_t pathLen = osimLogLocalDir_.length()+1+strlen(sysCallLogFileName_[sc])+4+1;
- sysCallLogFilePath_[sc] = new (heap_) char[pathLen];
+ size_t pathLen = osimLogLocalDir_.length()+1+strlen(logFileNames_[sc])+4+1;
+ logFilePaths_[sc] = new (heap_) char[pathLen];
// Construct an absolute pathname for the file.
- strcpy(sysCallLogFilePath_[sc], osimLogLocalDir_.data());
- strcat(sysCallLogFilePath_[sc], "/");
- strcat(sysCallLogFilePath_[sc], sysCallLogFileName_[sc]);
+ strcpy(logFilePaths_[sc], osimLogLocalDir_.data());
+ strcat(logFilePaths_[sc], "/");
+ strcat(logFilePaths_[sc], logFileNames_[sc]);
}
-void OptimizerSimulator::openAndAddHeaderToLogfile(sysCall sc)
+void OptimizerSimulator::openWriteLogStreams(OsimLog sc)
{
- NABoolean isDir;
- if(fileExists(sysCallLogFilePath_[sc],isDir))
- {
- char errMsg1[118+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- sprintf(errMsg1, "The target log file %s already exists. "
- "Delete this and other existing log files before "
- "running the OSIM in CAPTURE mode.", sysCallLogFilePath_[sc]);
- OsimLogException(errMsg1, __FILE__, __LINE__).throwException();
- }
-
- // Create the file and write header lines to it.
- writeSysCallStream_[sc] = new (heap_) ofstream(sysCallLogFilePath_[sc],ios::app);
-
- *writeSysCallStream_[sc] << "--" << sysCallLogFileName_[sc] << ":" << endl;
-
- // Indent the output.
- *writeSysCallStream_[sc] << " " << endl;
+ NABoolean isDir;
+ if(isFileExists(logFilePaths_[sc],isDir))
+ {
+ raiseOsimException("The target log file %s already exists. "
+ "Delete this and other existing log files before "
+ "running the OSIM in CAPTURE mode.", logFilePaths_[sc]);
+ }
+ // Create the file and write header lines to it.
+ writeLogStreams_[sc] = new (heap_) ofstream(logFilePaths_[sc],ios::app);
}
// Initialize the log files if OSIM is running under either CAPTURE
@@ -1732,26 +2204,23 @@ void OptimizerSimulator::openAndAddHeaderToLogfile(sysCall sc)
// to NULL if OSIM is not running(OFF).
void OptimizerSimulator::initLogFilePaths()
{
- for (sysCall sc=FIRST_SYSCALL; sc<NUM_OF_SYSCALLS; sc = sysCall(sc+1))
+ for (OsimLog sc=FIRST_LOG; sc<NUM_OF_LOGS; sc = OsimLog(sc+1))
{
switch (osimMode_)
{
case OFF:
- // OFF mode indicates no log files needed.
- sysCallLogFilePath_[sc] = NULL;
- break;
+ // OFF mode indicates no log files needed.
+ logFilePaths_[sc] = NULL;
+ break;
case CAPTURE:
- // Set log file path.
- setLogFilepath(sc);
- // Add header to the log file.
- openAndAddHeaderToLogfile(sc);
- break;
+ createLogFilepath(sc);
+ openWriteLogStreams(sc);
+ break;
case LOAD:
case UNLOAD:
case SIMULATE:
- // Set log file path.
- setLogFilepath(sc);
- break;;
+ createLogFilepath(sc);
+ break;;
}
}
}
@@ -1763,7 +2232,7 @@ void OptimizerSimulator::capture_MYSYSTEMNUMBER(short sysNum)
if (mySystemNumber_ == -1)
{
// Open file in append mode.
- ofstream * outLogfile = writeSysCallStream_[MYSYSTEMNUMBER];
+ ofstream * outLogfile = writeLogStreams_[MYSYSTEMNUMBER];
// Write data at the end of the file.
Int32 origWidth = (*outLogfile).width();
@@ -1779,19 +2248,10 @@ void OptimizerSimulator::readLogfile_MYSYSTEMNUMBER()
short sysNum;
NABoolean isDir;
- if(!fileExists(sysCallLogFilePath_[MYSYSTEMNUMBER],isDir))
- {
- char errMsg[38+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.",
- sysCallLogFilePath_[MYSYSTEMNUMBER]);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
- }
-
- ifstream inLogfile(sysCallLogFilePath_[MYSYSTEMNUMBER]);
+ if(!isFileExists(logFilePaths_[MYSYSTEMNUMBER],isDir))
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[MYSYSTEMNUMBER]);
- // Read and ignore the top 2 header lines.
- inLogfile.ignore(OSIM_LINEMAX, '\n');
- inLogfile.ignore(OSIM_LINEMAX, '\n');
+ ifstream inLogfile(logFilePaths_[MYSYSTEMNUMBER]);
if(inLogfile.good())
{
@@ -1808,6 +2268,8 @@ void OptimizerSimulator::readLogfile_MYSYSTEMNUMBER()
mySystemNumber_ = sysNum;
}
}
+ else
+ raiseOsimException("Unable to open %s, bad handle.", logFilePaths_[MYSYSTEMNUMBER]);
}
short OptimizerSimulator::simulate_MYSYSTEMNUMBER()
@@ -1839,29 +2301,13 @@ short OSIM_MYSYSTEMNUMBER()
break;
default:
// The OSIM must run under OFF (normal), CAPTURE or SIMULATE mode.
- OSIM_errorMessage("Invalid OSIM mode - It must be OFF or CAPTURE or SIMULATE.");
+ raiseOsimException("An invalid OSIM mode is encountered - The valid mode is OFF, CAPTURE or SIMULATE");
break;
}
return sysNum;
}
// END ************* System Call: MYSYSTEMNUMBER() *************
-
-// BEGIN *********** System Call: getEstimatedRows() ****************
-//
-void OSIM_captureEstimatedRows(const char *tableName, double estRows)
-{
- if(CURRCONTEXT_OPTSIMULATOR)
- CURRCONTEXT_OPTSIMULATOR->capture_getEstimatedRows(tableName, estRows);
-}
-
-double OSIM_simulateEstimatedRows(const char *tableName)
-{
- return CURRCONTEXT_OPTSIMULATOR ?
- CURRCONTEXT_OPTSIMULATOR->simulate_getEstimatedRows(tableName) :
- -1;
-}
-
void OptimizerSimulator::capture_getEstimatedRows(const char *tableName, double estRows)
{
NAString *key_tableName = new (heap_) NAString(tableName, heap_);
@@ -1879,7 +2325,7 @@ void OptimizerSimulator::capture_getEstimatedRows(const char *tableName, double
NAString *check = hashDict_getEstimatedRows_->insert(key_tableName,
val_estRows);
// Open file in append mode.
- ofstream * outLogfile = writeSysCallStream_[ESTIMATED_ROWS];
+ ofstream * outLogfile = writeLogStreams_[ESTIMATED_ROWS];
Int32 origWidth = (*outLogfile).width();
// Write data at the end of the file.
(*outLogfile) << " ";
@@ -1889,6 +2335,43 @@ void OptimizerSimulator::capture_getEstimatedRows(const char *tableName, double
}
}
+void OSIM_restoreHHDFSMasterHostList()
+{
+ if(CURRCONTEXT_OPTSIMULATOR &&
+ CURRCONTEXT_OPTSIMULATOR->getOsimMode() == OptimizerSimulator::SIMULATE)
+ CURRCONTEXT_OPTSIMULATOR->restoreHHDFSMasterHostList();
+}
+
+void OptimizerSimulator::restoreHHDFSMasterHostList()
+{
+ NABoolean isDir;
+ if(!isFileExists(logFilePaths_[HHDFS_MASTER_HOST_LIST],isDir))
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[HHDFS_MASTER_HOST_LIST]);
+
+ ifstream inLogfile(logFilePaths_[HHDFS_MASTER_HOST_LIST]);
+
+ if(inLogfile.good())
+ {
+ std::string name;
+ std::string value;
+ //read HHDFSMasterHostList::hasVirtualSQNodes_;
+ inLogfile >> name >> value;
+ CmpCommon::context()->setHasVirtualSQNodes(std::atoi(value.c_str()));
+ //read HHDFSMasterHostList::numSQNodes_;
+ inLogfile >> name >> value;
+ CmpCommon::context()->setNumSQNodes(std::atoi(value.c_str()));
+ inLogfile >> value;
+ if(value.length() > 0){
+ LIST(NAString) hosts(STMTHEAP);
+ NAString line = value.c_str();
+ line.split('|', hosts);
+ for(Int32 i = 0; i < hosts.entries(); i++)
+ HHDFSMasterHostList::getHostNumInternal(hosts[i].data());
+ }
+ }
+ else
+ raiseOsimException("Unable to open %s, bad handle.", logFilePaths_[HHDFS_MASTER_HOST_LIST]);
+}
void OptimizerSimulator::readLogfile_getEstimatedRows()
{
@@ -1896,19 +2379,10 @@ void OptimizerSimulator::readLogfile_getEstimatedRows()
double estRows;
NABoolean isDir;
- if(!fileExists(sysCallLogFilePath_[ESTIMATED_ROWS],isDir))
- {
- char errMsg[38+OSIM_PATHMAX+1]; // Error errMsg below + filename + '\0'
- snprintf(errMsg, sizeof(errMsg), "Unable to open %s file for reading data.",
- sysCallLogFilePath_[ESTIMATED_ROWS]);
- OsimLogException(errMsg, __FILE__, __LINE__).throwException();
- }
-
- ifstream inLogfile(sysCallLogFilePath_[ESTIMATED_ROWS]);
+ if(!isFileExists(logFilePaths_[ESTIMATED_ROWS],isDir))
+ raiseOsimException("Unable to open %s file for reading data.", logFilePaths_[ESTIMATED_ROWS]);
- // Read and ignore the top 2 header lines.
- inLogf
<TRUNCATED>