You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/06/29 05:48:14 UTC
[07/50] kylin git commit: minor, improve odbc performance
minor, improve odbc performance
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/51f22e28
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/51f22e28
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/51f22e28
Branch: refs/heads/master
Commit: 51f22e284467f23c8aeeed8ea79e5133ddd1141e
Parents: f32165e
Author: lidongsjtu <li...@apache.org>
Authored: Thu Jun 15 17:03:20 2017 +0800
Committer: Hongbin Ma <ma...@kyligence.io>
Committed: Thu Jun 15 21:17:54 2017 +0800
----------------------------------------------------------------------
odbc/Common/REST.cpp | 67 +++++++++++++++++++++--------------
odbc/Common/REST.h | 1 +
odbc/Driver/KO_DIAG.CPP | 17 ++++-----
odbc/Driver/KO_EXEC.CPP | 6 ++--
odbc/TestDLL/SimpleQueryTest.cpp | 6 ++--
5 files changed, 57 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Common/REST.cpp
----------------------------------------------------------------------
diff --git a/odbc/Common/REST.cpp b/odbc/Common/REST.cpp
index 858ecd4..8f0c7bb 100644
--- a/odbc/Common/REST.cpp
+++ b/odbc/Common/REST.cpp
@@ -280,34 +280,28 @@ std::unique_ptr <MetadataResponse> restGetMeta ( char* serverAddr, long port, ch
wstring cookQuery ( wchar_t* p )
{
- wchar_t* q = new wchar_t[wcslen ( p ) + 1];
- wcscpy ( q, p );
-
- for ( int i = 0; i < ( int ) wcslen ( q ); i++ )
- {
- if ( q[i] == '\r' || q[i] == '\n' || q[i] == '\t' )
- {
- q[i] = ' ';
- }
- }
-
- wstring ret ( q );
- delete[] q;
- size_t pos = 0;
+ std::wstringstream wss;
- for ( size_t pos = 0;; pos += 2 )
+ int l = wcslen ( p );
+ for ( int i = 0; i < l; i++ )
{
- pos = ret . find ( L"\"", pos );
-
- if ( pos == wstring::npos )
+ if ( p[i] == L'\r' || p[i] == L'\n' || p[i] == L'\t' )
{
- break;
- }
-
- ret . insert ( pos, L"\\" );
+ wss << L' ';
+ }
+
+ else if (p[i] == L'"')
+ {
+ wss << L"\\\"";
+ }
+
+ else
+ {
+ wss << p[i];
+ }
}
- return ret;
+ return wss.str();
}
wstring getBodyString ( http_response& response )
@@ -389,10 +383,15 @@ std::unique_ptr <SQLResponse> convertToSQLResponse ( int statusFlag,
wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username,
char* passwd,
char* project,
+ bool isPrepare,
int* statusFlag)
{
//using local cache to intercept probing queries
- const wchar_t* cachedQueryRes = loadCache ( rawSql );
+ const wchar_t* cachedQueryRes = NULL;
+
+ if (isPrepare) {
+ cachedQueryRes = loadCache ( rawSql );
+ }
if ( cachedQueryRes != NULL )
{
@@ -404,8 +403,24 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern
wstring serverAddrW = completeServerStr ( serverAddr, port );
http_client_config config;
config . set_timeout ( utility::seconds ( 36000 ) );
+
+ //uncomment these lines for debug with proxy
+ //wstring p = L"http://127.0.0.1:8888";
+ //config.set_proxy(web_proxy(p));
+
http_client session ( serverAddrW, config );
- http_request request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST );
+ http_request request;
+
+ if (!isPrepare)
+ {
+ request = makeRequest ( username, passwd, L"/kylin/api/query", methods::POST );
+ }
+
+ else
+ {
+ request = makeRequest ( username, passwd, L"/kylin/api/query/prestate", methods::POST );
+ }
+
wstring sql = cookQuery ( rawSql );
std::wstringstream wss;
wss << L"{ \"acceptPartial\": false, \"project\" : \"" << project << L"\", " << " \"sql\" : \"" << sql << L"\" }" ;
@@ -444,7 +459,7 @@ wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* usern
wstring ret = getBodyString ( response );
- if (*statusFlag == 1)
+ if (*statusFlag == 1 && isPrepare)
{
storeCache(rawSql, ret.c_str());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Common/REST.h
----------------------------------------------------------------------
diff --git a/odbc/Common/REST.h b/odbc/Common/REST.h
index f70d7d5..60c14a7 100644
--- a/odbc/Common/REST.h
+++ b/odbc/Common/REST.h
@@ -38,4 +38,5 @@ std::unique_ptr <SQLResponse> convertToSQLResponse ( int status,
wstring requestQuery ( wchar_t* rawSql, char* serverAddr, long port, char* username,
char* passwd,
char* project,
+ bool isPrepare,
int* statusFlag);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Driver/KO_DIAG.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_DIAG.CPP b/odbc/Driver/KO_DIAG.CPP
index ac313e7..def819c 100644
--- a/odbc/Driver/KO_DIAG.CPP
+++ b/odbc/Driver/KO_DIAG.CPP
@@ -644,7 +644,8 @@ RETCODE SQL_API SQLGetDiagRec ( SQLSMALLINT pHandleType,
pODBCDiagRow _SQLPutDiagRow ( pODBCDiag pDiag, StrPtr pFunc, StrPtr pState, Long pNativeErrorCode,
StrPtr pMsgArgs, va_list pArgs )
{
- Char s[4096]; // arbitary and maximum length of message
+ // to avoid: TST1003: The message for sqlstate '01000' is longer than SQL_MAX_MESSAGE_LENGTH - 1.
+ Char s[SQL_MAX_MESSAGE_LENGTH - 1]; // arbitary and maximum length of message
pODBCDiagRow l;
pODBCDiagRow r;
@@ -660,7 +661,7 @@ pODBCDiagRow _SQLPutDiagRow ( pODBCDiag pDiag, StrPtr pFunc, StrPtr pState, Long
// check if there is some message
if ( pMsgArgs )
{
- vsprintf ( s + strlen ( s ), pMsgArgs, pArgs );
+ vsnprintf ( s + strlen ( s ), SQL_MAX_MESSAGE_LENGTH - 2 - strlen(s), pMsgArgs, pArgs );
}
else
@@ -1110,13 +1111,13 @@ void _ODBCLogMsg ( LogLevel level, const char* pMsgArgs, ... )
break;
}
- time_t now = time ( 0 );
- struct tm tstruct;
- char buffer[100];
- tstruct = *localtime ( &now );
- strftime ( buffer, 100, "%Y-%m-%d.%X", &tstruct );
+ char ts[24];
+ SYSTEMTIME sys;
+ GetLocalTime( &sys );
+ sprintf( ts, "%4d-%02d-%02d %02d:%02d:%02d.%03d", sys.wYear, sys.wMonth, sys.wDay, sys.wHour, sys.wMinute, sys.wSecond, sys.wMilliseconds);
+
_write ( gLogFile, "[", 1 );
- _write ( gLogFile, buffer, strlen ( buffer ) );
+ _write ( gLogFile, ts, strlen ( ts ) );
_write ( gLogFile, "]", 1 );
// MSG PARSING
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/Driver/KO_EXEC.CPP
----------------------------------------------------------------------
diff --git a/odbc/Driver/KO_EXEC.CPP b/odbc/Driver/KO_EXEC.CPP
index 514375e..8a8992c 100644
--- a/odbc/Driver/KO_EXEC.CPP
+++ b/odbc/Driver/KO_EXEC.CPP
@@ -129,7 +129,7 @@ RETCODE SQL_API _SQLExecStmtFromReq ( pODBCStmt pStmt, bool pPrepared ) {
try {
__ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "start to call rest api" ) );
response = requestQuery ( pStmt->Stmt, pStmt->Conn->Server, pStmt->Conn->ServerPort, pStmt->Conn->UserName, pStmt->Conn->Password,
- pStmt->Conn->Project, &status );
+ pStmt->Conn->Project, pPrepared, &status );
__ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "received and uncompressed rest response:" ) );
p = convertToSQLResponse(status, response);
__ODBCLOG ( _ODBCLogMsg ( LogLevel_DEBUG, "parsed to SQLResponse" ) );
@@ -207,7 +207,7 @@ RETCODE SQL_API SQLPrepareW ( SQLHSTMT pStmt,
( ( pODBCStmt ) pStmt )->Prepared = 1;
- return _SQLExecStmtFromReq((pODBCStmt)pStmt,1);
+ return _SQLExecStmtFromReq((pODBCStmt)pStmt, 1);
//return SQL_SUCCESS;
}
@@ -230,7 +230,7 @@ RETCODE SQL_API SQLExecute ( HSTMT pStmt ) {
}
// excute the request
- x = _SQLExecStmtFromReq ( ( pODBCStmt ) pStmt, 1 );
+ x = _SQLExecStmtFromReq ( ( pODBCStmt ) pStmt, 0 );
return x;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/51f22e28/odbc/TestDLL/SimpleQueryTest.cpp
----------------------------------------------------------------------
diff --git a/odbc/TestDLL/SimpleQueryTest.cpp b/odbc/TestDLL/SimpleQueryTest.cpp
index 7f857a9..9705b5e 100644
--- a/odbc/TestDLL/SimpleQueryTest.cpp
+++ b/odbc/TestDLL/SimpleQueryTest.cpp
@@ -23,7 +23,7 @@ void simpleQueryTest ()
//Intercept query test
{
int status;
- wstring s = requestQuery ( L"SELECT 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status );
+ wstring s = requestQuery ( L"SELECT 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status );
std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s);
if ( ( int ) y -> results . size () != 1 )
@@ -34,7 +34,7 @@ void simpleQueryTest ()
//Ungzipped Query Test
{
int status;
- wstring s = requestQuery ( L"select cal_dt from test_kylin_fact limit 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status );
+ wstring s = requestQuery ( L"select cal_dt from test_kylin_fact limit 1", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status );
std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s);
if ( ( int ) y -> results . size () != 1 )
@@ -45,7 +45,7 @@ void simpleQueryTest ()
//zipped Query Test
{
int status;
- wstring s = requestQuery ( L"select * from test_kylin_fact limit 12", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, &status );
+ wstring s = requestQuery ( L"select * from test_kylin_fact limit 12", KServerAddr, KPort, KUserName, KPassword, KDefaultProject, false, &status );
std::unique_ptr <SQLResponse> y = convertToSQLResponse(status, s);
if ( ( int ) y -> results . size () != 12 )