You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by hz...@apache.org on 2017/06/26 17:28:00 UTC
[1/3] incubator-trafodion git commit: [TRAFODION-2637] Library
management with Cloudera parcels
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 498b89f74 -> c7dfd5a2e
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/regress/udr/TEST102
----------------------------------------------------------------------
diff --git a/core/sql/regress/udr/TEST102 b/core/sql/regress/udr/TEST102
index ad99838..ce89a37 100644
--- a/core/sql/regress/udr/TEST102
+++ b/core/sql/regress/udr/TEST102
@@ -53,7 +53,7 @@ execute get_schemas;
get libraries;
get procedures;
showddl procedure rm;
-set param ?h 'PUT';
+set param ?h 'PUTFILE';
call help (?h);
-- enable authorization
@@ -84,7 +84,7 @@ execute get_schemas;
get libraries;
get procedures;
showddl procedure rm;
-set param ?h 'PUT';
+set param ?h 'PUTFILE';
call help (?h);
-- ****************************************************************************
@@ -111,9 +111,28 @@ initialize trafodion, create library management;
execute get_schemas;
get procedures;
--- we are done
-exit;
+-- put a file in a single chunk
+call "_LIBMGR_".putfile('abcd', 'udr_test102_1', 1, 1, 1);
+
+-- put a file in three chunks
+call "_LIBMGR_".putfile('abcd', 'udr_test102_3', 1, 0, 1);
+call "_LIBMGR_".putfile('efgh', 'udr_test102_3', 0, 0, 1);
+call "_LIBMGR_".putfile('ijkl', 'udr_test102_3', 0, 1, 1);
+-- negative test, file already exists and no overwrite requested
+call "_LIBMGR_".putfile('abcd', 'udr_test102_1', 1, 1, 0);
+-- expect two lines with the contents of the two files below
+log;
+sh cat $$TRAF_HOME$$/udr/lib/DB__ROOT/udr_test102_1 >>LOG102;
+sh echo " " >>LOG102;
+sh cat $$TRAF_HOME$$/udr/lib/DB__ROOT/udr_test102_3 >>LOG102;
+sh echo " " >>LOG102;
+log LOG102;
+call "_LIBMGR_".rm('udr_test102_1');
+call "_LIBMGR_".rm('udr_test102_3');
+
+-- we are done
+exit;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp b/core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp
index 2ee9892..4386d94 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp
@@ -1641,10 +1641,10 @@ short CmpSeabaseDDL::createLibmgrProcs(ExeCliInterface * cliInterface)
{
Lng32 cliRC = 0;
- // Create the procedures if they don't already exist
+ // Create the UDRs if they don't already exist
for (Int32 i = 0; i < sizeof(allLibmgrRoutineInfo)/sizeof(LibmgrRoutineInfo); i++)
{
- // Get the next procedure routine details
+ // Get the next routine details
const LibmgrRoutineInfo &prd = allLibmgrRoutineInfo[i];
const QString * qs = NULL;
@@ -1698,8 +1698,11 @@ short CmpSeabaseDDL::grantLibmgrPrivs(ExeCliInterface *cliInterface)
// Get the next procedure routine details
const LibmgrRoutineInfo &prd = allLibmgrRoutineInfo[i];
- str_sprintf (queryBuf, "grant execute on procedure %s.\"%s\".%s to %s with grant option",
- getSystemCatalog(),SEABASE_LIBMGR_SCHEMA,prd.newName,
+ str_sprintf (queryBuf, "grant execute on %s %s.\"%s\".%s to %s with grant option",
+ prd.udrType,
+ getSystemCatalog(),
+ SEABASE_LIBMGR_SCHEMA,
+ prd.newName,
DB__LIBMGRROLE);
cliRC = cliInterface->executeImmediate(queryBuf);
if (cliRC < 0)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/sqlcomp/CmpSeabaseDDLroutine.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLroutine.h b/core/sql/sqlcomp/CmpSeabaseDDLroutine.h
index 18b6d7a..af47250 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLroutine.h
+++ b/core/sql/sqlcomp/CmpSeabaseDDLroutine.h
@@ -46,8 +46,10 @@
#define SYSTEM_PROC_LS "LS"
#define SYSTEM_PROC_LSALL "LSALL"
#define SYSTEM_PROC_PUT "PUT"
+#define SYSTEM_PROC_PUTFILE "PUTFILE"
#define SYSTEM_PROC_RM "RM"
#define SYSTEM_PROC_RMREX "RMREX"
+#define SYSTEM_TMUDF_SYNCLIBUDF "SYNCLIBUDF"
// Create procedure text for system procedures
static const QString seabaseProcAddlibDDL[] =
@@ -183,6 +185,25 @@ static const QString seabaseProcPutDDL[] =
{" ; "}
};
+static const QString seabaseProcPutFileDDL[] =
+{
+ {" CREATE PROCEDURE IF NOT EXISTS %s.\"%s\".PUTFILE "},
+ {" ( "},
+ {" IN FILEDATA VARCHAR(102400) CHARACTER SET ISO88591, "},
+ {" IN FILENAME VARCHAR(256) CHARACTER SET UTF8, "},
+ {" IN ISFIRSTCHUNK INTEGER, "},
+ {" IN ISLASTCHUNK INTEGER, "},
+ {" IN OVERWRITEEXISTINGFILE INTEGER) "},
+ {" EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.putFile(java.lang.String,java.lang.String,int,int,int)' "},
+ {" EXTERNAL SECURITY DEFINER "},
+ {" LIBRARY %s.\"%s\".%s "},
+ {" LANGUAGE JAVA "},
+ {" PARAMETER STYLE JAVA "},
+ {" READS SQL DATA "},
+ {" NO TRANSACTION REQUIRED "},
+ {" ; "}
+};
+
static const QString seabaseProcRmDDL[] =
{
{" CREATE PROCEDURE IF NOT EXISTS %s.\"%s\".RM "},
@@ -214,9 +235,21 @@ static const QString seabaseProcRmrexDDL[] =
{" ; "}
};
+ static const QString seabaseTMUDFSyncLibDDL[] =
+{
+ {" CREATE TABLE_MAPPING FUNCTION IF NOT EXISTS %s.\"%s\".SYNCLIBUDF() "},
+ {" EXTERNAL NAME 'org.trafodion.libmgmt.SyncLibUDF' "},
+ {" LIBRARY %s.\"%s\".%s "},
+ {" LANGUAGE JAVA "},
+ {" ; "}
+};
+
struct LibmgrRoutineInfo
{
- // name of the procedure
+ // type of the UDR (used in grant)
+ const char * udrType;
+
+ // name of the UDR
const char * newName;
// ddl stmt corresponding to the current ddl.
@@ -225,54 +258,76 @@ struct LibmgrRoutineInfo
};
static const LibmgrRoutineInfo allLibmgrRoutineInfo[] = {
- {SYSTEM_PROC_ADDLIB,
+ {"PROCEDURE",
+ SYSTEM_PROC_ADDLIB,
seabaseProcAddlibDDL,
sizeof(seabaseProcAddlibDDL)
},
- {SYSTEM_PROC_ALTERLIB,
+ {"PROCEDURE",
+ SYSTEM_PROC_ALTERLIB,
seabaseProcAlterlibDDL,
sizeof(seabaseProcAlterlibDDL)
},
- {SYSTEM_PROC_DROPLIB,
+ {"PROCEDURE",
+ SYSTEM_PROC_DROPLIB,
seabaseProcDroplibDDL,
sizeof(seabaseProcDroplibDDL)
},
- {SYSTEM_PROC_GETFILE,
+ {"PROCEDURE",
+ SYSTEM_PROC_GETFILE,
seabaseProcGetfileDDL,
sizeof(seabaseProcGetfileDDL)
},
- {SYSTEM_PROC_HELP,
+ {"PROCEDURE",
+ SYSTEM_PROC_HELP,
seabaseProcHelpDDL,
sizeof(seabaseProcHelpDDL)
},
- {SYSTEM_PROC_LS,
+ {"PROCEDURE",
+ SYSTEM_PROC_LS,
seabaseProcLsDDL,
sizeof(seabaseProcLsDDL)
},
- {SYSTEM_PROC_LSALL,
+ {"PROCEDURE",
+ SYSTEM_PROC_LSALL,
seabaseProcLsallDDL,
sizeof(seabaseProcLsallDDL)
},
- {SYSTEM_PROC_PUT,
+ {"PROCEDURE",
+ SYSTEM_PROC_PUT,
seabaseProcPutDDL,
sizeof(seabaseProcPutDDL)
},
- {SYSTEM_PROC_RM,
+ {"PROCEDURE",
+ SYSTEM_PROC_PUTFILE,
+ seabaseProcPutFileDDL,
+ sizeof(seabaseProcPutFileDDL)
+ },
+
+ {"PROCEDURE",
+ SYSTEM_PROC_RM,
seabaseProcRmDDL,
sizeof(seabaseProcRmDDL)
},
- {SYSTEM_PROC_RMREX,
+ {"PROCEDURE",
+ SYSTEM_PROC_RMREX,
seabaseProcRmrexDDL,
sizeof(seabaseProcRmrexDDL)
+ },
+
+ {"FUNCTION",
+ SYSTEM_TMUDF_SYNCLIBUDF,
+ seabaseTMUDFSyncLibDDL,
+ sizeof(seabaseTMUDFSyncLibDDL)
}
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/udrserv/udrglobals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/udrserv/udrglobals.cpp b/core/sql/udrserv/udrglobals.cpp
index b21a03a..c0817fb 100644
--- a/core/sql/udrserv/udrglobals.cpp
+++ b/core/sql/udrserv/udrglobals.cpp
@@ -49,7 +49,8 @@
#include "OperTypeEnum.h"
#include "UdrDebug.h"
#include "exp_function.h"
-
+#include "JavaObjectInterface.h"
+#include "Globals.h"
#include "udrdefs.h"
@@ -305,39 +306,79 @@ LmLanguageManager *UdrGlobals::getOrCreateLM(LmResult &result,
}
}
+// small helper class to use functions in JavaObjectInterface
+class UdrGlobalsJavaObj : public JavaObjectInterface
+{
+public:
+
+ UdrGlobalsJavaObj(NAHeap *heap) : JavaObjectInterface(heap, (jobject) -1) {}
+ JOI_RetCode initJVMForUDRServ(LmJavaOptions *options);
+};
+
+JOI_RetCode UdrGlobalsJavaObj::initJVMForUDRServ(LmJavaOptions *options)
+{
+ return initJVM(options);
+}
+
LmLanguageManagerJava *UdrGlobals::getOrCreateJavaLM(LmResult &result,
ComDiagsArea *diags)
{
+ result = LM_OK;
+
if (javaLanguageManager_)
- {
- result = LM_OK;
return javaLanguageManager_;
- }
- ComUInt32 lmJavaMax = 1;
+ // Note: For trusted UDRs, we use a T2 driver to do SQL from
+ // UDRs. This means that the UDR server contains the executor
+ // code. If we do SQL and that SQL needs another JVM, we need to
+ // share the JVM (and also the LmLanguageManagerJava). Therefore,
+ // create these objects in the executor and CLI globals.
+ TIMER_ON(initLangmanTimer);
+
+ // create a JavaObjectInterface, just so we can call its method
+ // to create or attach to a JVM
+ UdrGlobalsJavaObj helper(udrHeap_);
- TIMER_ON(initLangmanTimer)
- javaLanguageManager_ = new (getUdrHeap())
- LmLanguageManagerJava(result, commandLineMode_, lmJavaMax,
- getJavaOptions(), diags);
+ helper.initJVMForUDRServ(getJavaOptions());
- TIMER_OFF(initLangmanTimer, "Call Langman Constructor")
- LOG_FLUSH
+ // now that we know that a JVM exists, create the language manager
+ // in the CLI globals
+ CliGlobals *cliGlobals = GetCliGlobals();
+ if (!cliGlobals)
+ cliGlobals = CliGlobals::createCliGlobals(TRUE);
+
+ javaLanguageManager_ = cliGlobals->getLanguageManagerJava();
+
+ TIMER_OFF(initLangmanTimer, "Call Langman Constructor (trusted)");
+ LOG_FLUSH;
+
+ // At a later time, we may want to use a T4 driver for isolated
+ // UDRs and create a stand-alone LmLanguageManagerJava here,
+ // using the code below.
//
- // Make sure LM pointer and result are consistent. Either we return
- // a NULL pointer and an error result, or a non-NULL pointer and
- // LM_OK.
+ // ComUInt32 lmJavaMax = 1;
//
- if (javaLanguageManager_ && result != LM_OK)
- {
- delete javaLanguageManager_;
- javaLanguageManager_ = NULL;
- }
- if (!javaLanguageManager_ && result == LM_OK)
- {
+ // TIMER_ON(initLangmanTimer)
+ // javaLanguageManager_ = new (getUdrHeap())
+ // LmLanguageManagerJava(result, commandLineMode_, lmJavaMax,
+ // getJavaOptions(), diags);
+ //
+ // TIMER_OFF(initLangmanTimer, "Call Langman Constructor (isolated)")
+ // LOG_FLUSH
+ //
+ // // Make sure LM pointer and result are consistent. Either we return
+ // // a NULL pointer and an error result, or a non-NULL pointer and
+ // // LM_OK.
+ //
+ // if (javaLanguageManager_ && result != LM_OK)
+ // {
+ // delete javaLanguageManager_;
+ // javaLanguageManager_ = NULL;
+ // }
+
+ if (!javaLanguageManager_)
result = LM_ERR;
- }
return javaLanguageManager_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/udrserv/udrserv.cpp
----------------------------------------------------------------------
diff --git a/core/sql/udrserv/udrserv.cpp b/core/sql/udrserv/udrserv.cpp
index fef9d4d..0a3629a 100644
--- a/core/sql/udrserv/udrserv.cpp
+++ b/core/sql/udrserv/udrserv.cpp
@@ -230,35 +230,6 @@ void InitializeJavaOptionsFromEnvironment(LmJavaOptions &javaOptions
{
char *val;
- if ((val = getenv("CLASSPATH")))
- {
- javaOptions.addSystemProperty("java.class.path", val);
- }
- char maxHeapOption[64];
- int maxHeapEnvvarMB = DEFAULT_JVM_MAX_HEAP_SIZE;
- sprintf(maxHeapOption, "-Xmx%dm", maxHeapEnvvarMB);
- javaOptions.addOption((const char *)maxHeapOption, TRUE);
-
- char compressedClassSpaceSizeOptions[64];
- int compressedClassSpaceSize = 0;
- const char *compressedClassSpaceSizeStr = getenv("JVM_COMPRESSED_CLASS_SPACE_SIZE");
- if (compressedClassSpaceSizeStr)
- compressedClassSpaceSize = atoi(compressedClassSpaceSizeStr);
- if (compressedClassSpaceSize <= 0)
- compressedClassSpaceSize = DEFAULT_COMPRESSED_CLASSSPACE_SIZE;
- sprintf(compressedClassSpaceSizeOptions, "-XX:CompressedClassSpaceSize=%dm", compressedClassSpaceSize);
- javaOptions.addOption((const char *)compressedClassSpaceSizeOptions, TRUE);
-
- char maxMetaspaceSizeOptions[64];
- int maxMetaspaceSize = 0;
- const char *maxMetaspaceSizeStr = getenv("JVM_MAX_METASPACE_SIZE");
- if (maxMetaspaceSizeStr)
- maxMetaspaceSize = atoi(maxMetaspaceSizeStr);
- if (maxMetaspaceSize <= 0)
- maxMetaspaceSize = DEFAULT_MAX_METASPACE_SIZE;
- sprintf(maxMetaspaceSizeOptions, "-XX:MaxMetaspaceSize=%dm", maxMetaspaceSize);
- javaOptions.addOption((const char *)maxMetaspaceSizeOptions, TRUE);
-
/* Look for java startup options and envvars in configuration file */
if (UdrCfgParser::cfgFileIsOpen(initErrText))
{
[3/3] incubator-trafodion git commit: Merge [TRAFODION-2637] Library
management with Cloudera parcels PR-1141
Posted by hz...@apache.org.
Merge [TRAFODION-2637] Library management with Cloudera parcels PR-1141
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c7dfd5a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c7dfd5a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c7dfd5a2
Branch: refs/heads/master
Commit: c7dfd5a2e81488f0aa239ed4d2672191f47d2192
Parents: 498b89f ba20e3f
Author: Hans Zeller <hz...@apache.org>
Authored: Mon Jun 26 17:27:35 2017 +0000
Committer: Hans Zeller <hz...@apache.org>
Committed: Mon Jun 26 17:27:35 2017 +0000
----------------------------------------------------------------------
core/.gitignore | 2 +-
core/Makefile | 2 +-
core/sql/executor/JavaObjectInterface.cpp | 282 ++++++-----
core/sql/executor/JavaObjectInterface.h | 19 +-
core/sql/executor/OrcFileReader.cpp | 1 -
core/sql/executor/SequenceFileReader.cpp | 3 -
core/sql/langman/LmJavaOptions.cpp | 11 +
core/sql/langman/LmJavaOptions.h | 3 +
core/sql/lib_mgmt/Makefile | 4 -
core/sql/lib_mgmt/README.rst | 36 +-
core/sql/lib_mgmt/pom.xml | 27 +-
.../java/org/trafodion/libmgmt/FileMgmt.java | 487 +++++++++++--------
.../java/org/trafodion/libmgmt/SyncLibUDF.java | 170 +++++++
.../lib_mgmt/src/main/resources/init_libmgmt.sh | 206 --------
core/sql/optimizer/BindRelExpr.cpp | 7 -
core/sql/optimizer/OptPhysRelExpr.cpp | 14 +-
core/sql/optimizer/RelExeUtil.cpp | 1 -
core/sql/optimizer/RelExpr.cpp | 5 -
core/sql/optimizer/RelMisc.h | 9 -
core/sql/regress/udr/EXPECTED001 | 16 +-
core/sql/regress/udr/EXPECTED102 | 49 +-
core/sql/regress/udr/TEST102 | 27 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 11 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.h | 77 ++-
core/sql/udrserv/udrglobals.cpp | 85 +++-
core/sql/udrserv/udrserv.cpp | 29 --
26 files changed, 912 insertions(+), 671 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c7dfd5a2/core/sql/optimizer/OptPhysRelExpr.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c7dfd5a2/core/sql/optimizer/RelExeUtil.cpp
----------------------------------------------------------------------
[2/3] incubator-trafodion git commit: [TRAFODION-2637] Library
management with Cloudera parcels
Posted by hz...@apache.org.
[TRAFODION-2637] Library management with Cloudera parcels
Also in this commit:
[TRAFODION-2581] Combine JVM startup code of SQL executor and UDR server
We have library management SPJs that allow a user to deploy library
files from a client machine. These SPJs make use of the pdcp command
that is installed on many clusters, but not always. The new method
uses temporary HDFS files and a TMUDF to distribute the library
file to all nodes of the cluster.
For now I left the old PUT method in place and added a new PUTFILE
method. We can remove the old method later.
A few other things fixed in the process:
- Make sure that when a UDR server calls the executor and uses a JVM,
we share the same JVM. To do this, we need to use the same startup
method, and I needed to change the executor startup method to accept
the extra options used for the UDR server.
- Got rid of the "HBase compatibility mode" in JavaObjectInterface.
- A common method to execute SQL in the library management code.
- removed the obsolete init_libmgmt.sh file.
- We have special code that makes sure a UDR can request to run in
parallel, including a request to run on all nodes. Expanded this
special code from predefined UDFs to any UDF.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/ba20e3fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/ba20e3fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/ba20e3fa
Branch: refs/heads/master
Commit: ba20e3fabf1adf58fdb1287dc9b67055118aab23
Parents: 9dd45d2
Author: Hans Zeller <hz...@apache.org>
Authored: Thu Jun 22 21:58:59 2017 +0000
Committer: Hans Zeller <hz...@apache.org>
Committed: Thu Jun 22 21:58:59 2017 +0000
----------------------------------------------------------------------
core/.gitignore | 2 +-
core/Makefile | 2 +-
core/sql/executor/JavaObjectInterface.cpp | 282 ++++++-----
core/sql/executor/JavaObjectInterface.h | 19 +-
core/sql/executor/OrcFileReader.cpp | 1 -
core/sql/executor/SequenceFileReader.cpp | 3 -
core/sql/langman/LmJavaOptions.cpp | 11 +
core/sql/langman/LmJavaOptions.h | 3 +
core/sql/lib_mgmt/Makefile | 4 -
core/sql/lib_mgmt/README.rst | 36 +-
core/sql/lib_mgmt/pom.xml | 27 +-
.../java/org/trafodion/libmgmt/FileMgmt.java | 487 +++++++++++--------
.../java/org/trafodion/libmgmt/SyncLibUDF.java | 170 +++++++
.../lib_mgmt/src/main/resources/init_libmgmt.sh | 206 --------
core/sql/optimizer/BindRelExpr.cpp | 7 -
core/sql/optimizer/OptPhysRelExpr.cpp | 14 +-
core/sql/optimizer/RelExeUtil.cpp | 1 -
core/sql/optimizer/RelExpr.cpp | 5 -
core/sql/optimizer/RelMisc.h | 9 -
core/sql/regress/udr/EXPECTED001 | 16 +-
core/sql/regress/udr/EXPECTED102 | 49 +-
core/sql/regress/udr/TEST102 | 27 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 11 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.h | 77 ++-
core/sql/udrserv/udrglobals.cpp | 85 +++-
core/sql/udrserv/udrserv.cpp | 29 --
26 files changed, 912 insertions(+), 671 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/.gitignore
----------------------------------------------------------------------
diff --git a/core/.gitignore b/core/.gitignore
index fb6d0fe..c20d35c 100644
--- a/core/.gitignore
+++ b/core/.gitignore
@@ -3,4 +3,4 @@
*.o
*.jar
trafodion*.tgz
-/sqf/sql/scripts/init_libmgmt.sh
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/Makefile
----------------------------------------------------------------------
diff --git a/core/Makefile b/core/Makefile
index 86cac7e..19236dd 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -258,5 +258,5 @@ eclipseclean:
trafinstall:
cd $(TRAF_HOME)/sql/scripts && install_traf_components
-lib_mgmt:
+lib_mgmt: foundation
cd sql/lib_mgmt && $(MAKE) all
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/executor/JavaObjectInterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/JavaObjectInterface.cpp b/core/sql/executor/JavaObjectInterface.cpp
index c9a68c4..4417a6c 100644
--- a/core/sql/executor/JavaObjectInterface.cpp
+++ b/core/sql/executor/JavaObjectInterface.cpp
@@ -25,6 +25,7 @@
#include "QRLogger.h"
#include "Globals.h"
#include "ComUser.h"
+#include "LmJavaOptions.h"
// Changed the default to 256 to limit java heap size used by SQL processes.
// Keep this define in sync with udrserv/udrserv.cpp
@@ -96,146 +97,190 @@ char* JavaObjectInterface::buildClassPath()
char* classPathBuffer = (char*)malloc(size);
strcpy(classPathBuffer, "-Djava.class.path=");
- if (isHBaseCompatibilityMode())
- strcat(classPathBuffer, "/opt/home/tools/hbase-0.94.5/lib/hadoop-core-1.0.4.jar:");
-
strcat(classPathBuffer, classPath);
return classPathBuffer;
}
+// helper method for JavaObjectInterface::createJVM() below
+static NABoolean isDefinedInOptions(LmJavaOptions *options,
+ const char *prefix)
+{
+ return (options &&
+ options->entries() &&
+ options->findByPrefix(prefix) != NULL_COLL_INDEX);
+}
+
#define MAX_NO_JVM_OPTIONS 12
//////////////////////////////////////////////////////////////////////////////
// Create a new JVM instance.
//////////////////////////////////////////////////////////////////////////////
-int JavaObjectInterface::createJVM()
+int JavaObjectInterface::createJVM(LmJavaOptions *options)
{
JavaVMInitArgs jvm_args;
- JavaVMOption jvm_options[MAX_NO_JVM_OPTIONS];
-
- char* classPathArg = buildClassPath();
+ JavaVMOption jvm_options[
+ MAX_NO_JVM_OPTIONS + (options ? options->entries() : 0)];
int numJVMOptions = 0;
- jvm_options[numJVMOptions].optionString = classPathArg;
- QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG, "Using classpath: %s",
- jvm_options[numJVMOptions].optionString);
- numJVMOptions++;
-
- char maxHeapOptions[64];
- bool passMaxHeapToJVM = true;
- int maxHeapEnvvarMB = DEFAULT_JVM_MAX_HEAP_SIZE;
- const char *maxHeapSizeStr = getenv("JVM_MAX_HEAP_SIZE_MB");
- if (maxHeapSizeStr)
- {
- maxHeapEnvvarMB = atoi(maxHeapSizeStr);
- if (maxHeapEnvvarMB <= 0)
- passMaxHeapToJVM = false;
- }
- if (passMaxHeapToJVM)
- {
- sprintf(maxHeapOptions, "-Xmx%dm", maxHeapEnvvarMB);
- jvm_options[numJVMOptions].optionString = maxHeapOptions;
- QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
- "Max heap option: %s",
+ char* classPathArg = NULL;
+
+ // call the helper method defined above to check for already defined options
+ if (!isDefinedInOptions(options, "-Djava.class.path="))
+ {
+ classPathArg = buildClassPath();
+ jvm_options[numJVMOptions].optionString = classPathArg;
+ QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG, "Using classpath: %s",
jvm_options[numJVMOptions].optionString);
- numJVMOptions++;
- }
+ numJVMOptions++;
+ }
+
+ if (!isDefinedInOptions(options, "-Xmx"))
+ {
+ char maxHeapOptions[64];
+ bool passMaxHeapToJVM = true;
+ int maxHeapEnvvarMB = DEFAULT_JVM_MAX_HEAP_SIZE;
+ const char *maxHeapSizeStr = getenv("JVM_MAX_HEAP_SIZE_MB");
+ if (maxHeapSizeStr)
+ {
+ maxHeapEnvvarMB = atoi(maxHeapSizeStr);
+ if (maxHeapEnvvarMB <= 0)
+ passMaxHeapToJVM = false;
+ }
+ if (passMaxHeapToJVM)
+ {
+ snprintf(maxHeapOptions, sizeof(maxHeapOptions),
+ "-Xmx%dm", maxHeapEnvvarMB);
+ jvm_options[numJVMOptions].optionString = maxHeapOptions;
+ QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
+ "Max heap option: %s",
+ jvm_options[numJVMOptions].optionString);
+ numJVMOptions++;
+ }
+ }
- char compressedClassSpaceSizeOptions[64];
- int compressedClassSpaceSize = 0;
- const char *compressedClassSpaceSizeStr = getenv("JVM_COMPRESSED_CLASS_SPACE_SIZE");
- if (compressedClassSpaceSizeStr)
- compressedClassSpaceSize = atoi(compressedClassSpaceSizeStr);
- if (compressedClassSpaceSize <= 0)
- compressedClassSpaceSize = DEFAULT_COMPRESSED_CLASSSPACE_SIZE;
- sprintf(compressedClassSpaceSizeOptions, "-XX:CompressedClassSpaceSize=%dm", compressedClassSpaceSize);
- jvm_options[numJVMOptions].optionString = compressedClassSpaceSizeOptions;
- QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
+ if (!isDefinedInOptions(options, "-XX:CompressedClassSpaceSize="))
+ {
+ char compressedClassSpaceSizeOptions[64];
+ int compressedClassSpaceSize = 0;
+ const char *compressedClassSpaceSizeStr = getenv("JVM_COMPRESSED_CLASS_SPACE_SIZE");
+ if (compressedClassSpaceSizeStr)
+ compressedClassSpaceSize = atoi(compressedClassSpaceSizeStr);
+ if (compressedClassSpaceSize <= 0)
+ compressedClassSpaceSize = DEFAULT_COMPRESSED_CLASSSPACE_SIZE;
+ snprintf(compressedClassSpaceSizeOptions,
+ sizeof(compressedClassSpaceSizeOptions),
+ "-XX:CompressedClassSpaceSize=%dm", compressedClassSpaceSize);
+ jvm_options[numJVMOptions].optionString = compressedClassSpaceSizeOptions;
+ QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
"CompressedClassSpaceSize: %s",
jvm_options[numJVMOptions].optionString);
- numJVMOptions++;
-
- char maxMetaspaceSizeOptions[64];
- int maxMetaspaceSize = 0;
- const char *maxMetaspaceSizeStr = getenv("JVM_MAX_METASPACE_SIZE");
- if (maxMetaspaceSizeStr)
- maxMetaspaceSize = atoi(maxMetaspaceSizeStr);
- if (maxMetaspaceSize <= 0)
- maxMetaspaceSize = DEFAULT_MAX_METASPACE_SIZE;
- sprintf(maxMetaspaceSizeOptions, "-XX:MaxMetaspaceSize=%dm", maxMetaspaceSize);
- jvm_options[numJVMOptions].optionString = maxMetaspaceSizeOptions;
- QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
- "MaxMetaspaceSize: %s",
- jvm_options[numJVMOptions].optionString);
- numJVMOptions++;
+ numJVMOptions++;
+ }
- char initHeapOptions[64];
- const char *initHeapSizeStr = getenv("JVM_INIT_HEAP_SIZE_MB");
- if (initHeapSizeStr)
- {
- const int initHeapEnvvarMB = atoi(initHeapSizeStr);
- if (initHeapEnvvarMB > 0)
+ if (!isDefinedInOptions(options, "-XX:MaxMetaspaceSize="))
{
- sprintf(initHeapOptions, "-Xms%dm", initHeapEnvvarMB);
- jvm_options[numJVMOptions].optionString = initHeapOptions;
+ char maxMetaspaceSizeOptions[64];
+ int maxMetaspaceSize = 0;
+ const char *maxMetaspaceSizeStr = getenv("JVM_MAX_METASPACE_SIZE");
+ if (maxMetaspaceSizeStr)
+ maxMetaspaceSize = atoi(maxMetaspaceSizeStr);
+ if (maxMetaspaceSize <= 0)
+ maxMetaspaceSize = DEFAULT_MAX_METASPACE_SIZE;
+ snprintf(maxMetaspaceSizeOptions, sizeof(maxMetaspaceSizeOptions),
+ "-XX:MaxMetaspaceSize=%dm", maxMetaspaceSize);
+ jvm_options[numJVMOptions].optionString = maxMetaspaceSizeOptions;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
- "Init heap option: %s",
+ "MaxMetaspaceSize: %s",
jvm_options[numJVMOptions].optionString);
numJVMOptions++;
}
- }
- int debugPort = 0;
- const char *debugPortStr = getenv("JVM_DEBUG_PORT");
- if (debugPortStr != NULL)
- debugPort = atoi(debugPortStr);
- if (debugPort > 0
-#ifdef NDEBUG
- // in a release build, only DB__ROOT can debug a process
- && ComUser::isRootUserID()
-#endif
- )
- {
- const char *debugTimeoutStr = getenv("JVM_DEBUG_TIMEOUT");
- if (debugTimeoutStr != NULL)
- debugTimeout_ = atoi(debugTimeoutStr);
- const char *suspendOnDebug = getenv("JVM_SUSPEND_ON_DEBUG");
- char debugOptions[300];
-
- debugPort_ = debugPort;
- // to allow debugging multiple processes at the same time,
- // specify a port that is a multiple of 1000 and the code will
- // add pid mod 1000 to the port number to use
- if (debugPort_ % 1000 == 0)
- debugPort_ += (GetCliGlobals()->myPin() % 1000);
- sprintf(debugOptions,"-agentlib:jdwp=transport=dt_socket,address=%d,server=y,timeout=%d",
- debugPort_, debugTimeout_);
- if (suspendOnDebug != NULL)
- strcat(debugOptions, ",suspend=y");
- else
- strcat(debugOptions, ",suspend=n");
- jvm_options[numJVMOptions].optionString = debugOptions;
- QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_WARN,
- "Debugging JVM with options: %s",
- jvm_options[numJVMOptions].optionString);
- numJVMOptions++;
- }
+ if (!isDefinedInOptions(options, "-Xms"))
+ {
+ char initHeapOptions[64];
+ const char *initHeapSizeStr = getenv("JVM_INIT_HEAP_SIZE_MB");
+ if (initHeapSizeStr)
+ {
+ const int initHeapEnvvarMB = atoi(initHeapSizeStr);
+ if (initHeapEnvvarMB > 0)
+ {
+ snprintf(initHeapOptions, sizeof(initHeapOptions),
+ "-Xms%dm", initHeapEnvvarMB);
+ jvm_options[numJVMOptions].optionString = initHeapOptions;
+ QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
+ "Init heap option: %s",
+ jvm_options[numJVMOptions].optionString);
+ numJVMOptions++;
+ }
+ }
+ }
+
+ if (!isDefinedInOptions(options, "-agentlib:jdwp="))
+ {
+ int debugPort = 0;
+ const char *debugPortStr = getenv("JVM_DEBUG_PORT");
+ if (debugPortStr != NULL)
+ debugPort = atoi(debugPortStr);
+ if (debugPort > 0)
+ {
+ const char *debugTimeoutStr = getenv("JVM_DEBUG_TIMEOUT");
+ if (debugTimeoutStr != NULL)
+ debugTimeout_ = atoi(debugTimeoutStr);
+ const char *suspendOnDebug = getenv("JVM_SUSPEND_ON_DEBUG");
+ char debugOptions[300];
+
+ debugPort_ = debugPort;
+ // to allow debugging multiple processes at the same time,
+ // specify a port that is a multiple of 1000 and the code will
+ // add pid mod 1000 to the port number to use
+ if (debugPort_ % 1000 == 0)
+ debugPort_ += (GetCliGlobals()->myPin() % 1000);
+ sprintf(debugOptions,"-agentlib:jdwp=transport=dt_socket,address=%d,server=y,timeout=%d",
+ debugPort_, debugTimeout_);
+ if (suspendOnDebug != NULL)
+ strcat(debugOptions, ",suspend=y");
+ else
+ strcat(debugOptions, ",suspend=n");
+ jvm_options[numJVMOptions].optionString = debugOptions;
+ QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_WARN,
+ "Debugging JVM with options: %s",
+ jvm_options[numJVMOptions].optionString);
+ numJVMOptions++;
+ }
+ }
const char *oomOption = "-XX:+HeapDumpOnOutOfMemoryError";
- jvm_options[numJVMOptions].optionString = (char *)oomOption;
- numJVMOptions++;
- char *mySqRoot = getenv("TRAF_HOME");
- int len;
- char *oomDumpDir = NULL;
- if (mySqRoot != NULL)
- {
- len = strlen(mySqRoot);
- oomDumpDir = new (heap_) char[len+50];
- strcpy(oomDumpDir, "-XX:HeapDumpPath=");
- strcat(oomDumpDir, mySqRoot);
- strcat(oomDumpDir, "/logs");
- jvm_options[numJVMOptions].optionString = (char *)oomDumpDir;
- numJVMOptions++;
- }
+ if (!isDefinedInOptions(options, oomOption))
+ {
+ jvm_options[numJVMOptions].optionString = (char *)oomOption;
+ numJVMOptions++;
+ }
+
+ if (!isDefinedInOptions(options, "-XX:HeapDumpPath="))
+ {
+ char *mySqRoot = getenv("TRAF_HOME");
+ int len;
+ char *oomDumpDir = NULL;
+ if (mySqRoot != NULL)
+ {
+ len = strlen(mySqRoot);
+ oomDumpDir = new (heap_) char[len+50];
+ strcpy(oomDumpDir, "-XX:HeapDumpPath=");
+ strcat(oomDumpDir, mySqRoot);
+ strcat(oomDumpDir, "/logs");
+ jvm_options[numJVMOptions].optionString = (char *)oomDumpDir;
+ numJVMOptions++;
+ }
+ }
+
+ if (options)
+ for (CollIndex o=0; o<options->entries(); o++)
+ {
+ jvm_options[numJVMOptions].optionString = (char *) options->getOption(o);
+ QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
+ "Option passed to JavaObjectInterface::createJVM(): %s",
+ jvm_options[numJVMOptions].optionString);
+ numJVMOptions++;
+ }
jvm_args.version = JNI_VERSION_1_6;
jvm_args.options = jvm_options;
@@ -243,14 +288,15 @@ int JavaObjectInterface::createJVM()
jvm_args.ignoreUnrecognized = 1;
int ret = JNI_CreateJavaVM(&jvm_, (void**)&jenv_, &jvm_args);
- free(classPathArg);
+ if (classPathArg)
+ free(classPathArg);
return ret;
}
//////////////////////////////////////////////////////////////////////////////
//
//////////////////////////////////////////////////////////////////////////////
-JOI_RetCode JavaObjectInterface::initJVM()
+JOI_RetCode JavaObjectInterface::initJVM(LmJavaOptions *options)
{
jint result;
if (jvm_ == NULL)
@@ -266,7 +312,7 @@ JOI_RetCode JavaObjectInterface::initJVM()
if (jvm_count == 0)
{
// No - create a new one.
- result = createJVM();
+ result = createJVM(options);
if (result != JNI_OK)
{
GetCliGlobals()->setJniErrorStr(getErrorText(JOI_ERROR_CHECK_JVM));
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/executor/JavaObjectInterface.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/JavaObjectInterface.h b/core/sql/executor/JavaObjectInterface.h
index de5b451..772148b 100644
--- a/core/sql/executor/JavaObjectInterface.h
+++ b/core/sql/executor/JavaObjectInterface.h
@@ -32,6 +32,8 @@
#include "jni.h"
#include "Platform.h"
+class LmJavaOptions;
+
#ifndef SEQ_TESTING
#include "ex_god.h"
#endif
@@ -82,7 +84,6 @@ protected:
,javaObj_(NULL)
,needToDetach_(false)
,isInitialized_(false)
- ,isHBaseCompatibilityMode_(TRUE)
,debugPort_(debugPort)
,debugTimeout_(debugTimeout)
{
@@ -95,7 +96,6 @@ protected:
,javaObj_(NULL)
,needToDetach_(false)
,isInitialized_(false)
- ,isHBaseCompatibilityMode_(TRUE)
,debugPort_(0)
,debugTimeout_(0)
{
@@ -113,10 +113,10 @@ protected:
virtual ~JavaObjectInterface();
// Create a new JVM
- int createJVM();
+ int createJVM(LmJavaOptions *options);
// Initialize the JVM.
- JOI_RetCode initJVM();
+ JOI_RetCode initJVM(LmJavaOptions *options = NULL);
// Initialize JVM and all the JNI configuration.
// Must be called.
@@ -132,16 +132,6 @@ protected:
void logError(std::string &cat, const char* methodName, jstring jresult);
void logError(std::string &cat, const char* file, int line);
- void setHBaseCompatibilityMode(bool val)
- {
- isHBaseCompatibilityMode_ = val;
- }
-
- bool isHBaseCompatibilityMode()
- {
- return isHBaseCompatibilityMode_;
- }
-
JOI_RetCode initJNIEnv();
char* buildClassPath();
@@ -178,7 +168,6 @@ protected:
jobject javaObj_;
bool needToDetach_;
bool isInitialized_;
- bool isHBaseCompatibilityMode_;
int debugPort_;
int debugTimeout_;
pid_t tid_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/executor/OrcFileReader.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/OrcFileReader.cpp b/core/sql/executor/OrcFileReader.cpp
index 9dfafd5..7696589 100644
--- a/core/sql/executor/OrcFileReader.cpp
+++ b/core/sql/executor/OrcFileReader.cpp
@@ -112,7 +112,6 @@ OFR_RetCode OrcFileReader::init()
JavaMethods_[JM_CLOSE ].jm_name = "close";
JavaMethods_[JM_CLOSE ].jm_signature = "()Ljava/lang/String;";
- setHBaseCompatibilityMode(FALSE);
return (OFR_RetCode)JavaObjectInterface::init(className, javaClass_, JavaMethods_, (Int32)JM_LAST, FALSE);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/executor/SequenceFileReader.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/SequenceFileReader.cpp b/core/sql/executor/SequenceFileReader.cpp
index aff2945..0ef0399 100644
--- a/core/sql/executor/SequenceFileReader.cpp
+++ b/core/sql/executor/SequenceFileReader.cpp
@@ -114,7 +114,6 @@ SFR_RetCode SequenceFileReader::init()
JavaMethods_[JM_CLOSE ].jm_name = "close";
JavaMethods_[JM_CLOSE ].jm_signature = "()Ljava/lang/String;";
- setHBaseCompatibilityMode(FALSE);
rc = (SFR_RetCode)JavaObjectInterface::init(className, javaClass_, JavaMethods_, (Int32)JM_LAST, javaMethodsInitialized_);
javaMethodsInitialized_ = TRUE;
pthread_mutex_unlock(&javaMethodsInitMutex_);
@@ -523,8 +522,6 @@ SFW_RetCode SequenceFileWriter::init()
JavaMethods_[JM_HDFS_DELETE_PATH].jm_name = "hdfsDeletePath";
JavaMethods_[JM_HDFS_DELETE_PATH].jm_signature = "(Ljava/lang/String;)Z";
-
- setHBaseCompatibilityMode(FALSE);
rc = (SFW_RetCode)JavaObjectInterface::init(className, javaClass_, JavaMethods_, (Int32)JM_LAST, javaMethodsInitialized_);
javaMethodsInitialized_ = TRUE;
pthread_mutex_unlock(&javaMethodsInitMutex_);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/langman/LmJavaOptions.cpp
----------------------------------------------------------------------
diff --git a/core/sql/langman/LmJavaOptions.cpp b/core/sql/langman/LmJavaOptions.cpp
index 37d6d62..e36e03a 100644
--- a/core/sql/langman/LmJavaOptions.cpp
+++ b/core/sql/langman/LmJavaOptions.cpp
@@ -134,6 +134,17 @@ void LmJavaOptions::addOptions(const char *options,
NADELETEBASIC(copy, collHeap());
}
+CollIndex LmJavaOptions::findByPrefix(const char *prefix) const
+{
+ size_t prefixLen = strlen(prefix);
+
+ for (CollIndex i=0; i<options_.entries(); i++)
+ if (strncmp(options_[i], prefix, prefixLen) == 0)
+ return i;
+
+ return NULL_COLL_INDEX;
+}
+
void LmJavaOptions::addSystemProperty(const char *name,
const char *value)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/langman/LmJavaOptions.h
----------------------------------------------------------------------
diff --git a/core/sql/langman/LmJavaOptions.h b/core/sql/langman/LmJavaOptions.h
index 800a101..9b39594 100644
--- a/core/sql/langman/LmJavaOptions.h
+++ b/core/sql/langman/LmJavaOptions.h
@@ -90,6 +90,9 @@ public:
// whitespace from each individual option before inserting it.
void addOptions(const char *options, const char *delimiters, NABoolean trim);
+ // find an option by its prefix and return its index or NULL_COLL_INDEX
+ CollIndex findByPrefix(const char *prefix) const;
+
// Convenience function to add a single "-Dname=value" option
void addSystemProperty(const char *name, const char *value);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/lib_mgmt/Makefile
----------------------------------------------------------------------
diff --git a/core/sql/lib_mgmt/Makefile b/core/sql/lib_mgmt/Makefile
index 72a44ac..3dd2100 100644
--- a/core/sql/lib_mgmt/Makefile
+++ b/core/sql/lib_mgmt/Makefile
@@ -20,16 +20,12 @@
# @@@ END COPYRIGHT @@@
INSTALL_LIBDIR := ${TRAF_HOME}/export/lib
-INSTALL_SCRIPTSDIR := ${TRAF_HOME}/sql/scripts
CP :=/bin/cp -f
all:
@$(MAVEN) package -DskipTests
mkdir -p ${INSTALL_LIBDIR}
${CP} target/lib_mgmt-${TRAFODION_VER}.jar ${INSTALL_LIBDIR}/lib_mgmt.jar
- ${CP} src/main/resources/init_libmgmt.sh ${INSTALL_SCRIPTSDIR}/
- @chmod +x ${INSTALL_SCRIPTSDIR}/init_libmgmt.sh
clean:
@rm -f ${INSTALL_LIBDIR}/lib_mgmt.jar
- @rm -f ${INSTALL_SCRIPTSDIR}/init_libmgmt.sh
@$(MAVEN) clean
cleanall: clean
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/lib_mgmt/README.rst
----------------------------------------------------------------------
diff --git a/core/sql/lib_mgmt/README.rst b/core/sql/lib_mgmt/README.rst
index 2eacac5..25959a9 100644
--- a/core/sql/lib_mgmt/README.rst
+++ b/core/sql/lib_mgmt/README.rst
@@ -31,33 +31,31 @@ make clean && make all
Initialization
-cds && ./init_libmgmt.sql
+sqlci
+initialize trafodion, create library management;
+
+ if already initialized, do the following:
+
+sqlci
+initialize trafodion, upgrade library management;
+
SPJ Example
-DB_LIBMGR.HELP(INOUT COMMANDNAME VARCHAR)
+"_LIBMGR_".HELP(INOUT COMMANDNAME VARCHAR)
E.g.
trafci>set param ?p1 help;
-trafci>call DB_LIBMGR.HELP(?p1)
-
-PUT - Upload a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]PUT for more info.
-
-LS - List JARs. SHOWDDL PROCEDURE [SCHEMA NAME.]LS for more info.
-
-LSALL - List all JARs. SHOWDDL PROCEDURE [SCHEMA NAME.]LSALL for more info.
-
-RM - Remove a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]RM for more info.
-
-RMREX - Remove JARs by a perticular pattern. SHOWDDL PROCEDURE [SCHEMA NAME.]RMREX for more info.
-
-GETFILE - Download a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]GETFILE for more info.
-
+trafci>call "_LIBMGR_".HELP(?p1);
+
+PUTFILE - Upload a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]PUTFILE for more info.
+LS - List library files. SHOWDDL PROCEDURE [SCHEMA NAME.]LS for more info.
+LSALL - List all library files. SHOWDDL PROCEDURE [SCHEMA NAME.]LSALL for more info.
+RM - Remove a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]RM for more info.
+RMREX - Remove library files by a perticular pattern. SHOWDDL PROCEDURE [SCHEMA NAME.]RMREX for more info.
+GETFILE - Download a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]GETFILE for more info.
ADDLIB - Create a library. SHOWDDL PROCEDURE [SCHEMA NAME.]ADDLIB for more info.
-
ALTERLIB - Update a library. SHOWDDL PROCEDURE [SCHEMA NAME.]ALTERLIB for more info.
-
DROPLIB - Drop a library. SHOWDDL PROCEDURE [SCHEMA NAME.]DROPLIB for more info.
-
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/lib_mgmt/pom.xml
----------------------------------------------------------------------
diff --git a/core/sql/lib_mgmt/pom.xml b/core/sql/lib_mgmt/pom.xml
index d01ae09..58ff63d 100644
--- a/core/sql/lib_mgmt/pom.xml
+++ b/core/sql/lib_mgmt/pom.xml
@@ -27,6 +27,11 @@
<groupId>org.trafodion</groupId>
<artifactId>lib_mgmt</artifactId>
<version>${env.TRAFODION_VER}</version>
+ <properties>
+ <maven.antrun.plugin.version>1.6</maven.antrun.plugin.version>
+ <hbase.version>${env.HBASE_DEP_VER_CDH}</hbase.version>
+ <hadoop.version>${env.HADOOP_DEP_VER}</hadoop.version>
+ </properties>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
@@ -34,10 +39,26 @@
<version>1.7.12</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.trafodion</groupId>
+ <artifactId>trafodion-sql</artifactId>
+ <version>${env.TRAFODION_VER}</version>
+ <scope>system</scope>
+ <systemPath>${env.TRAF_HOME}/export/lib/trafodion-sql-apache-${env.TRAFODION_VER}.jar</systemPath>
+ </dependency>
</dependencies>
- <properties>
- <maven.antrun.plugin.version>1.6</maven.antrun.plugin.version>
- </properties>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/FileMgmt.java
----------------------------------------------------------------------
diff --git a/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/FileMgmt.java b/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/FileMgmt.java
index ba5d886..990fe1f 100644
--- a/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/FileMgmt.java
+++ b/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/FileMgmt.java
@@ -42,6 +42,12 @@ import java.util.Arrays;
import java.util.Date;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,8 +58,7 @@ public class FileMgmt {
private static final long MAX_JAR_FILE_SIZE = 104857600;
private static final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
private static final int MaxDataSize = 12800;
- private static final String CHARTSET = "ISO-8859-1";
- private static final String DEL_POSTFIX = ".DELETE";
+ private static final String CHARSET = "ISO-8859-1";
/**
* Print help info
@@ -63,18 +68,18 @@ public class FileMgmt {
*/
public static void help(String[] helps) {
String[] help = new String[] {
- "PUT - Upload a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]PUT for more info.",
- "LS - List JARs. SHOWDDL PROCEDURE [SCHEMA NAME.]LS for more info.",
- "LSALL - List all JARs. SHOWDDL PROCEDURE [SCHEMA NAME.]LSALL for more info.",
- "RM - Remove a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]RM for more info.",
- "RMREX - Remove JARs by a perticular pattern. SHOWDDL PROCEDURE [SCHEMA NAME.]RMREX for more info.",
- "GETFILE - Download a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]GETFILE for more info.",
+ "PUTFILE - Upload a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]PUTFILE for more info.",
+ "LS - List library files. SHOWDDL PROCEDURE [SCHEMA NAME.]LS for more info.",
+ "LSALL - List all library files. SHOWDDL PROCEDURE [SCHEMA NAME.]LSALL for more info.",
+ "RM - Remove a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]RM for more info.",
+ "RMREX - Remove library files by a perticular pattern. SHOWDDL PROCEDURE [SCHEMA NAME.]RMREX for more info.",
+ "GETFILE - Download a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]GETFILE for more info.",
"ADDLIB - Create a library. SHOWDDL PROCEDURE [SCHEMA NAME.]ADDLIB for more info.",
"ALTERLIB - Update a library. SHOWDDL PROCEDURE [SCHEMA NAME.]ALTERLIB for more info.",
"DROPLIB - Drop a library. SHOWDDL PROCEDURE [SCHEMA NAME.]DROPLIB for more info."
};
List<String> index = new ArrayList<String>(help.length);
- index.add("PUT");
+ index.add("PUTFILE");
index.add("LS");
index.add("LSALL");
index.add("RM");
@@ -121,37 +126,18 @@ public class FileMgmt {
String localFile) throws SQLException {
checkFileName(fileName);
Connection conn = getConn();
- Statement st = null;
- String sql = "";
- try {
- st = conn.createStatement();
- String userPath = getCodeFilePath(conn);
- sql = "create library " + libName + " file '" + userPath
+ String userPath = getCodeFilePath(conn);
+ String sql = "create library " + libName + " file '" + userPath
+ fileName + "'";
- if (hostName != null && !"".equals(hostName.trim())) {
- sql += " HOST NAME '" + hostName + "'";
- }
- if (localFile != null && !"".equals(localFile.trim())) {
- sql += " LOCAL FILE '" + localFile + "'";
- }
- st.execute(sql);
- } catch(SQLException e){
- LOG.error(sql,e);
- throw e;
- }finally {
- if (st != null) {
- try {
- st.close();
- } catch (Exception e) {
- }
- }
- if (conn != null){
- try {
- conn.close();
- } catch (Exception e) {
- }
- }
- }
+
+ if (hostName != null && !"".equals(hostName.trim())) {
+ sql += " HOST NAME '" + hostName + "'";
+ }
+ if (localFile != null && !"".equals(localFile.trim())) {
+ sql += " LOCAL FILE '" + localFile + "'";
+ }
+
+ execSQL(sql, conn);
}
/**
@@ -169,7 +155,6 @@ public class FileMgmt {
String hostName, String localFile) throws SQLException {
checkFileName(fileName);
Connection conn = getConn();
- Statement st = null;
String userPath = getCodeFilePath(conn);
String sql = "alter library " + libName + " FILE '" + userPath
+ fileName + "'";
@@ -180,22 +165,8 @@ public class FileMgmt {
if (localFile != null && !"".equals(localFile.trim())) {
sql += " LOCAL FILE '" + localFile + "'";
}
- try {
- st = conn.createStatement();
- st.execute(sql);
- } catch(SQLException e){
- LOG.error(sql,e);
- throw e;
- } finally {
- if (st != null)
- st.close();
- if (conn != null){
- try {
- conn.close();
- } catch (Exception e) {
- }
- }
- }
+
+ execSQL(sql, conn);
}
/**
@@ -207,35 +178,25 @@ public class FileMgmt {
* @throws SQLException
*/
public static void dropLib(String libName, String mode) throws SQLException {
- String sql = null;
- Connection con = getConn();
- Statement st = null;
- try {
- st = con.createStatement();
- sql = "drop library " + libName;
- if (mode != null)
- if (mode.trim().equalsIgnoreCase("RESTRICT"))
- sql += " RESTRICT";
- else if (mode.trim().equalsIgnoreCase("CASCADE"))
- sql += " CASCADE";
-
- st.execute(sql);
- } catch(SQLException e){
- LOG.error(sql,e);
- throw e;
- } finally {
- if (st != null)
- st.close();
- if (con != null){
- try {
- con.close();
- } catch (Exception e) {
- }
- }
- }
+ String sql = "drop library " + libName;
+
+ if (mode != null)
+ if (mode.trim().equalsIgnoreCase("RESTRICT"))
+ sql += " RESTRICT";
+ else if (mode.trim().equalsIgnoreCase("CASCADE"))
+ sql += " CASCADE";
+
+ execSQL(sql, getConn());
}
public static void syncJar(String userPath, String fileName) throws SQLException, IOException {
+ // the local staging directory is the same as the destination directory
+ syncJarPdcp(userPath, userPath, fileName);
+ }
+
+ public static void syncJarPdcp(String stagingPath,
+ String dstDir,
+ String fileName) throws SQLException, IOException {
checkFileName(fileName);
String nodes = System.getenv("MY_NODES");
LOG.info("syncJars " + fileName + ", MY_NODES=" + nodes);
@@ -249,23 +210,68 @@ public class FileMgmt {
if (pdsh == null) {
pdsh = "/usr/bin/pdsh";
}
- execShell(pdsh + " " + nodes + " mkdir -p " + userPath);
- execShell(pdcp + " " + nodes + " " + userPath + fileName.trim() + " " + userPath + " ");
- execShell(pdsh + " " + nodes + " chmod 755 " + userPath + fileName.trim());
+ execShell(pdsh + " " + nodes + " mkdir -p " + dstDir);
+ execShell(pdcp + " " + nodes + " " + stagingPath + fileName.trim() + " " + dstDir);
+ execShell(pdsh + " " + nodes + " chmod 755 " + dstDir + fileName.trim());
}
}
- public static void rmJar(String userPath, String fileName) throws SQLException, IOException {
+ static void syncLibViaHDFS(String fileName, Connection conn) throws SQLException, IOException {
checkFileName(fileName);
- LOG.info("syncJars " + fileName);
- String nodes = System.getenv("MY_NODES");
- if (nodes != null && !"".equals(nodes.trim())) {
- String pdsh = System.getenv("SQ_PDSH");
- if (pdsh == null) {
- pdsh = "/usr/bin/pdsh";
- }
- execShell(pdsh + " " + nodes + " rm -rf " + userPath + fileName.trim());
- }
+
+ String stagingDirPath = getLocalStagingDirPath(conn);
+ Path srcLocalPath = new Path(stagingDirPath, fileName);
+ Path dstHdfsDir = new Path(getHdfsStagingDirName(getCurrentUser(conn)));
+ Path dstHdfsPath = new Path(dstHdfsDir, fileName);
+
+ try {
+ // using HDFS Java interface
+ Configuration conf = new Configuration(true);
+ FileSystem fs = FileSystem.get(dstHdfsPath.toUri(),conf);
+
+ fs.mkdirs(dstHdfsDir, new FsPermission(FsAction.ALL,
+ FsAction.NONE,
+ FsAction.NONE));
+ fs.copyFromLocalFile(false, // do not delete src
+ true, // overwrite tgt
+ srcLocalPath,
+ dstHdfsPath);
+
+ LOG.info("syncJar " + fileName + ", target=" + dstHdfsDir.toString());
+
+ // now execute a TMUDF that will copy the file across all nodes
+
+ String sql = "select count(*) from udf(\"_LIBMGR_\".SyncLibUDF('c','"+ fileName + "'))";
+ String result = execSQL(sql, conn, true);
+
+ fs.delete(dstHdfsPath, false);
+ LOG.info("syncJar " + fileName + ", numSyncedNodes=" + result);
+ }
+ catch (Exception e1) {
+ try {
+ LOG.warn("syncJar " + fileName + ", sync via HDFS failed, trying pdcp");
+ // try using pdcp instead, this may not
+ // work on all systems
+ syncJarPdcp(stagingDirPath,
+ getCodeFilePath(conn),
+ fileName);
+ } catch (Exception e2) {
+ LOG.error(e1.getMessage(), e1);
+ LOG.error(e2.getMessage(), e2);
+ throw new SQLException(e1);
+ }
+ }
+ }
+
+ public static void rmJar(String fileName, Connection conn) throws SQLException, IOException {
+ checkFileName(fileName);
+
+ // execute a TMUDF that will delete the file across all nodes
+
+ String sql = "select count(*) from udf(\"_LIBMGR_\".SyncLibUDF('d','"+ fileName + "'))";
+ String result = execSQL(sql, conn, true);
+
+ LOG.info("rmJar " + fileName + ", numSyncedNodes=" + result);
}
private static String execShell(String cmd) throws IOException {
@@ -332,7 +338,7 @@ public class FileMgmt {
byte bArray[] = new byte[MaxDataSize];
int bytesRead = rAFile.read(bArray, 0, MaxDataSize);
if (bytesRead != -1) {
- fileData[0] = new String(Arrays.copyOf(bArray, bytesRead), CHARTSET);
+ fileData[0] = new String(Arrays.copyOf(bArray, bytesRead), CHARSET);
fileLength[0] = file.length();
LOG.info("Download: " + fileName + ", offset:" + offset + ",compressed length:" + fileData[0].length()
+ ",file length:" + fileLength[0]);
@@ -364,33 +370,19 @@ public class FileMgmt {
Connection conn = getConn();
LOG.info("Remove " + fileName);
String userPath = getCodeFilePath(conn);
- close(conn);
- File file = new File(userPath + fileName);
- File delFile = new File(fileName + DEL_POSTFIX);
- boolean isSuccess = false;
- if (file.exists()) {
- try {
- boolean isRenamed = file.renameTo(delFile);
- if (isRenamed)
- rmJar(userPath, fileName);
- else {
- throw new IOException("Delete " + fileName + " failed. File metrics: CanRead is " + file.canRead()
- + ", canWrite is " + file.canWrite() + ", canExecute is " + file.canExecute());
- }
- LOG.info("Remove " + fileName + " successfully!");
- isSuccess = true;
- return;
- } finally {
- if (isSuccess) {
- delFile.delete();
- } else {
- delFile.renameTo(file);
- }
- }
- } else {
- LOG.error("No such file[" + fileName + "]");
- throw new SQLException("No such file[" + fileName + "]");
- }
+ try {
+ File file = new File(userPath + fileName);
+ if (file.exists()) {
+ rmJar(fileName, conn);
+ LOG.info("Removed " + fileName + " successfully!");
+ return;
+ } else {
+ LOG.error("No such file[" + fileName + "]");
+ throw new SQLException("No such file[" + fileName + "]");
+ }
+ } finally {
+ close(conn);
+ }
}
/**
@@ -408,36 +400,21 @@ public class FileMgmt {
Connection conn = getConn();
LOG.info("Try to remove files[" + pattern + "]");
String userPath = getCodeFilePath(conn);
- close(conn);
File[] files = getFiles(pattern, new File(userPath));
- File[] delFiles = new File[files.length];
StringBuilder sb = new StringBuilder();
sb.append("<rmRex>");
sb.append(toXML(files, "rmList"));
sb.append("<message>");
- boolean hasError = false;
- boolean isSuccess = false;
- try {
- for (int i = 0; i < files.length; i++) {
- delFiles[i] = new File(files[i].getAbsolutePath() + DEL_POSTFIX);
- files[i].renameTo(delFiles[i]);
- }
- rmJar(userPath, pattern);
- isSuccess = true;
- } finally {
- if (isSuccess) {
- for (int i = 0; i < delFiles.length; i++) {
- delFiles[i].delete();
- }
- } else {
- for (int i = 0; i < delFiles.length; i++) {
- delFiles[i].renameTo(files[i]);
- }
- }
- }
- if (!hasError) {
- sb.append("Remove the files successfully!");
- }
+
+ try {
+ for (int i = 0; i < files.length; i++) {
+ rmJar(files[i].getName(), conn);
+ }
+ } finally {
+ close(conn);
+ }
+
+ sb.append("Removed the files successfully!");
sb.append("</message>");
sb.append("</rmRex>");
names[0] = sb.toString();
@@ -490,7 +467,7 @@ public class FileMgmt {
public static void put(String fileData, String fileName, int appendFlag, int overwriteOnCreate) throws SQLException, IOException {
checkFileName(fileName);
try {
- byte[] data = fileData.getBytes(CHARTSET);
+ byte[] data = fileData.getBytes(CHARSET);
Connection conn = getConn();
LOG.info("Put " + fileName + ", length: " + data.length + ", file string length:" + fileData.length());
@@ -534,35 +511,142 @@ public class FileMgmt {
}
}
- private static void checkFileName(String fileName) throws SQLException {
+ /**
+ * upload a library (jar or DLL) in one or more chunks, passed in as strings
+ *
+ * @param fileData a string (chunk) of bytes to be added to the file on
+ * the server side. These bytes are assumed to be in the
+ * ISO-8859-1 character set, which allows any bit combination,
+ * so passing arbitrary bit patterns is ok.
+ * @param fileName unqualified name of the target file (library file)
+ * @param isFirstChunk indicates whether this is the first chunk for a file
+ * @param isLastChunk indicates whether this is the last chunk for a file
+ * @param overwriteExistingFile (used only for the first chunk) indicates
+ * whether we should silently overwrite an existing file
+ * (true) or raise an exception, if file "fileName"
+ * already exists (false)
+ * @throws SQLException
+ * @throws IOException
+ */
+ public static void putFile(String fileData,
+ String fileName,
+ int isFirstChunk,
+ int isLastChunk,
+ int overwriteExistingFile)
+ throws SQLException, IOException {
+ checkFileName(fileName);
+ try {
+ byte[] data = fileData.getBytes(CHARSET);
+
+ Connection conn = getConn();
+ String stagingPath = getLocalStagingDirPath(conn);
+ String fname = stagingPath + fileName;
+ String dstFileName = getCodeFilePath(conn) + fileName;
+ boolean isFirst = (isFirstChunk != 0);
+ boolean isLast = (isLastChunk != 0);
+
+ LOG.info("PutFile " + fileName +
+ (isFirst && isLast ? "(single chunk)" :
+ (isFirst ? "(first chunk)" :
+ (isLast ? "(last chunk)" : "(intermediate chunk)"))) +
+ ", length: " + data.length + ", file string length:" + fileData.length());
+
+ if (isFirst &&
+ overwriteExistingFile == 0 &&
+ new File(dstFileName).exists()) {
+ throw new IOException("File " + fileName + " already exists!");
+ }
+ checkFile(fname, data.length);
+ File stagingDir = null;
+ FileOutputStream fos = null;
+ FileChannel channel = null;
+ FileLock lock = null;
+ try {
+ if (isFirst) {
+ stagingDir = new File(stagingPath);
+
+ if (!stagingDir.exists())
+ stagingDir.mkdir();
+ }
+
+ fos = new FileOutputStream(fname, !isFirst);
+ channel = fos.getChannel();
+ lock = channel.tryLock();
+ if (lock != null) {
+ fos.write(data);
+ fos.flush();
+ }else{
+ throw new SQLException("File "+fileName+" is locked, please try again later.");
+ }
+ if (isLast)
+ syncLibViaHDFS(fileName, conn);
+ } finally {
+ if(lock != null){
+ lock.release();
+ }
+ if(channel !=null){
+ channel.close();
+ }
+ if (fos != null)
+ fos.close();
+ if (isLast) {
+ // delete the file in the staging area
+ new File(fname).delete();
+ }
+ close(conn);
+ }
+
+ LOG.info("PutFile method out !!! " + fileName);
+ } catch (Throwable t) {
+ LOG.error(t.getMessage(), t);
+ throw new SQLException(t.getMessage());
+ }
+ }
+
+ static void checkFileName(String fileName) throws SQLException {
if (fileName.contains("/") || fileName.contains("\\"))
throw new SQLException("Illegal file name: " + fileName
+ ". File name must not contain \"/\".");
}
- private static void checkFile(String fname, int dataSize) throws SQLException {
- File jar = new File(fname);
- if (jar.length() + dataSize > MAX_JAR_FILE_SIZE) {
- LOG.error("Jar file size is over the threshold[100Mb]");
- throw new SQLException("Jar file size is over the threshold[100Mb]");
- }
- }
+ static String getHdfsStagingDirName(String userName) {
+ return "/user/trafodion/udr/lib/staging/" + userName;
+ }
- private static String getCodeFilePath(Connection conn) throws SQLException {
- String user = getCurrentUser(conn);
+ static String getLocalLibDirName(String userName) throws SQLException {
String root = System.getenv("TRAF_HOME");
if (root == null || "".equals(root.trim())) {
LOG.error("Cant get your traf installation path!");
throw new SQLException("Cant get your traf installation path!");
}
- File file = new File(root + "/udr/lib/" + user);
+ File file = new File(root + "/udr/lib/" + userName);
if (!file.exists()) {
file.mkdirs();
} else if (!file.isDirectory()) {
- throw new SQLException("User Directory is not valide or you dont have permission!");
+ throw new SQLException("User Directory is not valid or you dont have permission!");
}
- LOG.info("SPJ JARs location: " + file.getAbsolutePath());
return file.getAbsolutePath() + "/";
+ }
+
+ private static void checkFile(String fname, int dataSize) throws SQLException {
+ File jar = new File(fname);
+ if (jar.length() + dataSize > MAX_JAR_FILE_SIZE) {
+ LOG.error("Jar file size is over the threshold[100Mb]");
+ throw new SQLException("Jar file size is over the threshold[100Mb]");
+ }
+ }
+
+ private static String getCodeFilePath(Connection conn) throws SQLException {
+ String user = getCurrentUser(conn);
+ String result = getLocalLibDirName(user);
+ LOG.info("SPJ JARs location: " + result);
+ return result;
+ }
+
+ private static String getLocalStagingDirPath(Connection conn) throws SQLException {
+ // this is the local directory where the put() method
+ // stores the file as it is being assembled from chunks
+ return getCodeFilePath(conn) + "staging/";
}
private static Connection getConn() throws SQLException {
@@ -577,37 +661,54 @@ public class FileMgmt {
return conn;
}
+ private static void execSQL(String sqlString,
+ Connection conn) throws SQLException {
+ execSQL(sqlString, conn, false);
+ }
+
+ private static String execSQL(String sqlString,
+ Connection conn,
+ boolean executeQuery) throws SQLException {
+ Statement st = null;
+ ResultSet rs = null;
+ String result = null;
+
+ try {
+ st = conn.createStatement();
+
+ if (executeQuery) {
+ rs = st.executeQuery(sqlString);
+ if (rs.next()) {
+ result = rs.getString(1);
+ }
+ }
+ else
+ st.execute(sqlString);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new SQLException(e);
+ } finally {
+ if (rs != null) {
+ try {
+ rs.close();
+ } catch (Exception e) {
+ LOG.warn(e.getMessage(), e);
+ }
+ }
+ if (st != null) {
+ try {
+ st.close();
+ } catch (Exception e) {
+ LOG.warn(e.getMessage(), e);
+ }
+ }
+ }
+ return result;
+ }
+
private static String getCurrentUser(Connection conn) throws SQLException {
- Statement st = null;
- ResultSet rs = null;
- String user = null;
- try {
- st = conn.createStatement();
- rs = st.executeQuery("values(session_user)");
- if (rs.next()) {
- user = rs.getString(1);
- }
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new SQLException(e);
- } finally {
- if (rs != null) {
- try {
- rs.close();
- } catch (Exception e) {
- LOG.warn(e.getMessage(), e);
- }
- }
- if (st != null) {
- try {
- st.close();
- } catch (Exception e) {
- LOG.warn(e.getMessage(), e);
- }
- }
- }
-
- return user.replaceAll("[\\\\/]", "_");
+ String user = execSQL("values(session_user)", conn, true);
+ return user.replaceAll("[\\\\/]", "_");
}
private static File[] getFiles(String pattern, File dir) {
@@ -616,7 +717,7 @@ public class FileMgmt {
@Override
public boolean accept(File name) {
- if (name == null || !name.isFile() || name.getName().endsWith(DEL_POSTFIX)) {
+ if (name == null || !name.isFile()) {
return false;
}
return name.getName().trim().toUpperCase().matches(p);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/SyncLibUDF.java
----------------------------------------------------------------------
diff --git a/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/SyncLibUDF.java b/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/SyncLibUDF.java
new file mode 100644
index 0000000..3f8daf0
--- /dev/null
+++ b/core/sql/lib_mgmt/src/main/java/org/trafodion/libmgmt/SyncLibUDF.java
@@ -0,0 +1,170 @@
+/**
+* @@@ START COPYRIGHT @@@
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+* @@@ END COPYRIGHT @@@
+ */
+package org.trafodion.libmgmt;
+
+import java.io.File;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import org.trafodion.sql.udr.UDR;
+import org.trafodion.sql.udr.UDRInvocationInfo;
+import org.trafodion.sql.udr.UDRPlanInfo;
+import org.trafodion.sql.udr.UDRException;
+
+public class SyncLibUDF extends UDR {
+
+ // default constructor
+ public SyncLibUDF()
+ {}
+
+ @Override
+ public void describeParamsAndColumns(UDRInvocationInfo info)
+ throws UDRException
+ {
+ // this TMUDF takes no table-valued inputs, two string
+ // parameters, and generates a table with a single integer
+ // value. It assumes that it was created with the following
+ // DDL, without specifying parameters or return values:
+ //
+ // create table_mapping function SyncLibUDF()
+ // external name 'org.trafodion.libmgmt.SyncLibUDF'
+ // library ...;
+ //
+ // The call is like this: select ... from udf(synclibudf(<op>,<file>))
+ //
+ // <op> is 'c' to create/copy a file from the HDFS staging area for
+ // the current user into a local file on every node, or
+ // 'd' to delete/drop a local file on every node
+ // <file> is an unqualified file name (no '/' in the name)
+ //
+ // Example:
+ //
+ // select * from udf("_LIBMGR_".synclibudf('c', 'mylib.jar'));
+ //
+ // INSTANCE_NUM
+ // ------------
+ //
+ // 0
+ // 1
+ //
+ // --- 2 row(s) selected.
+ // >>
+
+
+ if (info.par().getNumColumns() != 2)
+ throw new UDRException(38970, "This UDF needs to be called with two input parameters");
+ if (info.getNumTableInputs() > 0)
+ throw new UDRException(38971, "This UDF needs to be called without table-valued inputs");
+
+ for (int i=0; i<2; i++)
+ info.addFormalParameter(info.par().getColumn(i));
+ info.out().addIntColumn("INSTANCE_NUM", false);
+ }
+
+ @Override
+ public void describeDesiredDegreeOfParallelism(UDRInvocationInfo info,
+ UDRPlanInfo plan)
+ throws UDRException
+ {
+ boolean usesVirtualNodes = false;
+
+ // check for configurations with virtual nodes. Run the UDF serially
+ // in those cases, since all the virtual nodes share the same node.
+ String cmd[] = new String[] { "grep",
+ "^[ \t]*_virtualnodes ",
+ System.getenv("TRAF_HOME") +
+ "/sql/scripts/sqconfig" };
+
+ try {
+ Process p = Runtime.getRuntime().exec(cmd);
+ int exitValue = p.waitFor();
+
+ usesVirtualNodes = (exitValue == 0);
+ } catch (Exception e) {
+ // if this fails, we assume that we are using no virtual nodes
+ }
+
+ if (!usesVirtualNodes)
+ // this TMUDF needs to run once on each node, since every
+ // parallel instance will be writing to a local file on
+ // that node
+ plan.setDesiredDegreeOfParallelism(
+ UDRPlanInfo.SpecialDegreeOfParallelism.
+ ONE_INSTANCE_PER_NODE.getSpecialDegreeOfParallelism());
+ else
+ plan.setDesiredDegreeOfParallelism(1);
+ }
+
+ @Override
+ public void processData(UDRInvocationInfo info,
+ UDRPlanInfo plan)
+ throws UDRException
+ {
+ String operation = info.par().getString(0);
+ String fileName = info.par().getString(1);
+ Path srcHdfsPath = null;
+ Path dstLocalPath = null;
+
+ try {
+ FileMgmt.checkFileName(fileName);
+
+ String user = info.getSessionUser();
+ File dstLocalDir = new File(FileMgmt.getLocalLibDirName(user));
+ File dstLocalFile = new File(dstLocalDir, fileName);
+
+ srcHdfsPath = new Path(FileMgmt.getHdfsStagingDirName(user), fileName);
+ dstLocalPath = new Path(dstLocalDir.getPath(), fileName);
+
+ // create the local directory or remove an existing file,
+ // this is common for all operations
+ if (!dstLocalDir.isDirectory())
+ dstLocalDir.mkdirs();
+ if (dstLocalFile.exists())
+ dstLocalFile.delete();
+
+ if (operation.compareTo("c") == 0) {
+ // Using the HDFS Java interface, copy the file to a local file.
+ // Note: The Hadoop configuration file (core-site.xml) needs
+ // to be in the classpath
+ Configuration conf = new Configuration(true);
+ FileSystem fs = FileSystem.get(srcHdfsPath.toUri(),conf);
+
+ fs.copyToLocalFile(false, // do not delete src
+ srcHdfsPath,
+ dstLocalPath);
+ }
+ }
+ catch (Exception e) {
+ throw new UDRException(38973, "Error copying HDFS file %s to local file %s, reason: %s",
+ (srcHdfsPath != null) ? srcHdfsPath.toString() : "null",
+ (dstLocalPath != null) ? dstLocalPath.toString() : "null",
+ e.getMessage());
+ }
+
+ // return my own instance number as the result
+ info.out().setInt(0, info.getMyInstanceNum());
+ emitRow(info);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/lib_mgmt/src/main/resources/init_libmgmt.sh
----------------------------------------------------------------------
diff --git a/core/sql/lib_mgmt/src/main/resources/init_libmgmt.sh b/core/sql/lib_mgmt/src/main/resources/init_libmgmt.sh
deleted file mode 100644
index 55e315c..0000000
--- a/core/sql/lib_mgmt/src/main/resources/init_libmgmt.sh
+++ /dev/null
@@ -1,206 +0,0 @@
-#!/bin/bash
-# @@@ START COPYRIGHT @@@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# @@@ END COPYRIGHT @@@
-
-SERVER_JAR=${TRAF_HOME}/export/lib/lib_mgmt.jar
-CI=sqlci
-CATALOG_NAME=TRAFODION
-LIB_SCHEMA="DB__LIBMGR"
-DB__LIBMGRROLE=DB_LIBMGRROLE
-LIB_NAME=DB__LIBMGRNAME
-
-function dropAndCreateSchema {
- echo "Creating Schema for ${LIB_SCHEMA}"
- ${CI} << sqlciEOF
-
- cqd CAT_IGNORE_ALREADY_EXISTS_ERROR 'on';
- cqd CAT_IGNORE_DOES_NOT_EXIST_ERROR 'on';
-
- DROP SCHEMA IF EXISTS $CATALOG_NAME.$LIB_SCHEMA CASCADE;
- CREATE SCHEMA $CATALOG_NAME.$LIB_SCHEMA;
- set schema $CATALOG_NAME.$LIB_SCHEMA;
- DROP LIBRARY ${LIB_NAME} CASCADE;
- CREATE LIBRARY ${LIB_NAME} FILE '${SERVER_JAR}';
- CREATE ROLE ${DB__LIBMGRROLE};
-
- cqd CAT_IGNORE_ALREADY_EXISTS_ERROR 'off';
- cqd CAT_IGNORE_DOES_NOT_EXIST_ERROR 'off';
-
- exit;
-sqlciEOF
-}
-
-function createProcedures {
-
- echo "Creating Procedures in schema '$CATALOG_NAME.$LIB_SCHEMA' "
- ${CI} << procEOF
-
- cqd CAT_IGNORE_ALREADY_EXISTS_ERROR 'on';
- cqd CAT_IGNORE_DOES_NOT_EXIST_ERROR 'on';
-
- -- Creating Procedures
- set schema $CATALOG_NAME.$LIB_SCHEMA;
-
- CREATE PROCEDURE HELP (
- INOUT COMMANDNAME VARCHAR(2560) CHARACTER SET ISO88591)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.help (java.lang.String[])'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- READS SQL DATA
- ;
- GRANT EXECUTE ON PROCEDURE HELP TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE ADDLIB (
- IN LIBNAME VARCHAR(1024) CHARACTER SET UTF8,
- IN FILENAME VARCHAR(1024) CHARACTER SET UTF8,
- IN HOSTNAME VARCHAR(1024) CHARACTER SET UTF8,
- IN LOCALFILE VARCHAR(1024) CHARACTER SET UTF8)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.addLib (java.lang.String,java.lang.String,java.lang.String,java.lang.String)'
- LIBRARY ${LIB_NAME}
- EXTERNAL SECURITY DEFINER
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- CONTAINS SQL
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE ADDLIB TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE DROPLIB (
- IN LIBNAME VARCHAR(1024) CHARACTER SET UTF8,
- IN MODETYPE VARCHAR(1024) CHARACTER SET ISO88591)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.dropLib (java.lang.String,java.lang.String)'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- CONTAINS SQL
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE DROPLIB TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE ALTERLIB (
- IN LIBNAME VARCHAR(1024) CHARACTER SET UTF8,
- IN FILENAME VARCHAR(1024) CHARACTER SET UTF8,
- IN HOSTNAME VARCHAR(1024) CHARACTER SET UTF8,
- IN LOCALFILE VARCHAR(1024) CHARACTER SET UTF8)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.alterLib (java.lang.String,java.lang.String,java.lang.String,java.lang.String)'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- CONTAINS SQL
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE ALTERLIB TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE PUT (
- IN FILEDATA VARCHAR(102400) CHARACTER SET ISO88591,
- IN FILENAME VARCHAR(256) CHARACTER SET UTF8,
- IN CREATEFLAG INTEGER,
- IN FILEOVERWRITE INTEGER)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.put(java.lang.String,java.lang.String,int,int)'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- READS SQL DATA
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE PUT TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE LS (
- IN FILENAME VARCHAR(256) CHARACTER SET UTF8,
- OUT FILENAMES VARCHAR(10240) CHARACTER SET ISO88591)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.ls(java.lang.String,java.lang.String[])'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- READS SQL DATA
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE LS TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE LSALL (
- OUT FILENAMES VARCHAR(10240) CHARACTER SET ISO88591)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.lsAll(java.lang.String[])'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- READS SQL DATA
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE LSALL TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE RM (
- IN FILENAME VARCHAR(256) CHARACTER SET UTF8)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.rm(java.lang.String)'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- READS SQL DATA
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE RM TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE RMREX (
- IN FILENAME VARCHAR(256) CHARACTER SET UTF8,
- OUT FILENAMES VARCHAR(10240) CHARACTER SET ISO88591)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.rmRex(java.lang.String, java.lang.String[])'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- READS SQL DATA
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE RMREX TO ${DB__LIBMGRROLE};
-
- CREATE PROCEDURE GETFILE (
- IN FILENAME VARCHAR(256) CHARACTER SET UTF8,
- IN OFFSET INTEGER,
- OUT FILEDATA VARCHAR(12800) CHARACTER SET ISO88591,
- OUT DATALENGTH LARGEINT)
- EXTERNAL NAME 'org.trafodion.libmgmt.FileMgmt.get (java.lang.String,int,java.lang.String[],long[])'
- EXTERNAL SECURITY DEFINER
- LIBRARY ${LIB_NAME}
- LANGUAGE JAVA
- PARAMETER STYLE JAVA
- READS SQL DATA
- NO TRANSACTION REQUIRED
- ;
- GRANT EXECUTE ON PROCEDURE GETFILE TO ${DB__LIBMGRROLE};
-
- cqd CAT_IGNORE_ALREADY_EXISTS_ERROR 'off';
- cqd CAT_IGNORE_DOES_NOT_EXIST_ERROR 'off';
-
- exit;
-procEOF
-
-}
-
-dropAndCreateSchema
-createProcedures
-echo "Successfully completed installation of procedures."
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index 585330b..45fa83d 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -17532,13 +17532,6 @@ RelExpr *TableMappingUDF::bindNode(BindWA *bindWA)
return NULL;
}
- // if this is a maintenance-type operation that must run on
- // all nodes of the cluster or must run in parallel, regardless
- // of the ATTEMPT_ESP_PARALLELISM CQD, then set a flag in the
- // root node
- if (getOperatorType() == REL_TABLE_MAPPING_BUILTIN_LOG_READER)
- bindWA->getTopRoot()->setMustUseESPs(TRUE);
-
// add the routine to the UdrStoiList. The UdrStoi list is used
// to check valid privileges
LIST(OptUdrOpenInfo *) udrList = bindWA->getUdrStoiList ();
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/optimizer/OptPhysRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/OptPhysRelExpr.cpp b/core/sql/optimizer/OptPhysRelExpr.cpp
index 392ee8d..922035c 100644
--- a/core/sql/optimizer/OptPhysRelExpr.cpp
+++ b/core/sql/optimizer/OptPhysRelExpr.cpp
@@ -12278,6 +12278,8 @@ Context* RelRoot::createContextForAChild(Context* myContext,
// the optimizer should attempt ESP parallelism.
NADefaults &defs = ActiveSchemaDB()->getDefaults();
DefaultToken attESPPara = CURRSTMT_OPTDEFAULTS->attemptESPParallelism();
+ NABoolean hasTMUDFsOrLRU = (getGroupAttr()->getNumTMUDFs() > 0 ||
+ containsLRU());
// heuristics for nice context we want to detect the group which is just
// below root. For example if group by is just below root we want it to
@@ -12318,9 +12320,11 @@ Context* RelRoot::createContextForAChild(Context* myContext,
// Regardless of how the ATTEMPT_ESP_PARALLELISM CQD is set, some operations
// like LRU always execute under ESPs for partitioned tables.
- // These are indicated by mustUseESPs() == TRUE.
- // Note that some conditions below may still override this.
- if ( ((CURRSTMT_OPTDEFAULTS->attemptESPParallelism() != DF_OFF) OR mustUseESPs())
+ // Note that TMUDFs can control their own degree of parallelism,
+ // and that the method to do this is called after we reach here.
+ // Note also that some conditions below may still override this.
+ if ( ((CURRSTMT_OPTDEFAULTS->attemptESPParallelism() != DF_OFF) OR
+ hasTMUDFsOrLRU)
// QSTUFF
// we don't support paralled execution in the get_next
// protocol yet - but for streams there should not be
@@ -12372,7 +12376,7 @@ Context* RelRoot::createContextForAChild(Context* myContext,
(childOpType != REL_EXE_UTIL) &&
(numExtractStreams_ == 0) &&
!isFastLoadIntoTrafodion &&
- !mustUseESPs())
+ !hasTMUDFsOrLRU)
{
countOfCPUs =
CURRSTMT_OPTDEFAULTS->getMaximumDegreeOfParallelism();
@@ -12504,7 +12508,7 @@ Context* RelRoot::createContextForAChild(Context* myContext,
// To be replaced later by a different CQD
if ( CmpCommon::getDefault(HBASE_RANGE_PARTITIONING) == DF_OFF ||
isASON ||
- mustUseESPs())
+ hasTMUDFsOrLRU)
canAdjustDoP = FALSE;
// Adjust DoP based on table size, if possible
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/optimizer/RelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelExeUtil.cpp b/core/sql/optimizer/RelExeUtil.cpp
index b59c58c..b588fca 100644
--- a/core/sql/optimizer/RelExeUtil.cpp
+++ b/core/sql/optimizer/RelExeUtil.cpp
@@ -5668,7 +5668,6 @@ RelExpr * ExeUtilLongRunning::bindNode(BindWA *bindWA)
// indicate that this is a an LRU statement
bindWA->getTopRoot()->setContainsLRU(TRUE);
- bindWA->getTopRoot()->setMustUseESPs(TRUE);
return boundExpr;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/optimizer/RelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelExpr.cpp b/core/sql/optimizer/RelExpr.cpp
index 0846e80..c14fc1c 100644
--- a/core/sql/optimizer/RelExpr.cpp
+++ b/core/sql/optimizer/RelExpr.cpp
@@ -10788,7 +10788,6 @@ RelRoot::RelRoot(RelExpr *input,
avoidHalloween_(FALSE),
containsOnStatementMV_(FALSE),
containsLRU_(FALSE),
- mustUseESPs_(FALSE),
disableESPParallelism_(FALSE),
hasOlapFunctions_(FALSE),
hasTDFunctions_(FALSE),
@@ -10860,7 +10859,6 @@ RelRoot::RelRoot(RelExpr *input,
avoidHalloween_(FALSE),
containsOnStatementMV_(FALSE),
containsLRU_(FALSE),
- mustUseESPs_(FALSE),
disableESPParallelism_(FALSE),
hasOlapFunctions_(FALSE),
hasTDFunctions_(FALSE),
@@ -10938,7 +10936,6 @@ RelRoot::RelRoot(const RelRoot & other)
disableESPParallelism_(other.disableESPParallelism_),
containsOnStatementMV_(other.containsOnStatementMV_),
containsLRU_(other.containsLRU_),
- mustUseESPs_(other.mustUseESPs_),
hasOlapFunctions_(other.hasOlapFunctions_),
hasTDFunctions_(other.hasTDFunctions_ ),
isAnalyzeOnly_(FALSE),
@@ -11151,8 +11148,6 @@ RelExpr * RelRoot::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap)
result->containsLRU_ = containsLRU_;
- result->mustUseESPs_ = mustUseESPs_;
-
result->isAnalyzeOnly_ = isAnalyzeOnly_;
result->hasMandatoryXP_ = hasMandatoryXP_ ;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/optimizer/RelMisc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelMisc.h b/core/sql/optimizer/RelMisc.h
index d892e45..fef9038 100644
--- a/core/sql/optimizer/RelMisc.h
+++ b/core/sql/optimizer/RelMisc.h
@@ -482,12 +482,6 @@ public:
inline void setContainsLRU(NABoolean h)
{ containsLRU_ = h; }
- inline NABoolean mustUseESPs() const
- { return mustUseESPs_; }
-
- inline void setMustUseESPs(NABoolean h = TRUE)
- { mustUseESPs_ = h; }
-
// MVQR
inline NABoolean isAnalyzeOnly()
{ return isAnalyzeOnly_; }
@@ -771,9 +765,6 @@ public:
// TRUE, if the IUD statement is an LRU
NABoolean containsLRU_;
- // TRUE, if one of the children must execute in parallel
- NABoolean mustUseESPs_;
-
// MVQR
// TRUE if the query result should be the query descriptor used for MV Query Reqrite.
NABoolean isAnalyzeOnly_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/regress/udr/EXPECTED001
----------------------------------------------------------------------
diff --git a/core/sql/regress/udr/EXPECTED001 b/core/sql/regress/udr/EXPECTED001
index e184bd7..004274c 100644
--- a/core/sql/regress/udr/EXPECTED001
+++ b/core/sql/regress/udr/EXPECTED001
@@ -371,9 +371,11 @@ SESSION_ID SEQUENCE_NO USERID TS
LC RC OP OPERATOR OPT DESCRIPTION CARD
---- ---- ---- -------------------- -------- -------------------- ---------
-3 . 4 root 1.79E+001
-2 . 3 tmudf XO 5.00E+001
-1 . 2 sort 1.00E+002
+5 . 6 root 1.79E+001
+4 . 5 esp_exchange 1:4(hash2) 5.00E+001
+3 . 4 tmudf XO 5.00E+001
+2 . 3 sort 1.00E+002
+1 . 2 esp_exchange 4(hash2):1 1.00E+002
. . 1 trafodion_scan CLICKS 1.00E+002
--- SQL operation complete.
@@ -459,9 +461,11 @@ SESSION_ID SEQUENCE_NO USERID TS
LC RC OP OPERATOR OPT DESCRIPTION CARD
---- ---- ---- -------------------- -------- -------------------- ---------
-3 . 4 root 1.79E+001
-2 . 3 tmudf XO 5.00E+001
-1 . 2 sort 1.00E+002
+5 . 6 root 1.79E+001
+4 . 5 esp_exchange 1:4(hash2) 5.00E+001
+3 . 4 tmudf XO 5.00E+001
+2 . 3 sort 1.00E+002
+1 . 2 esp_exchange 4(hash2):1 1.00E+002
. . 1 trafodion_scan CLICKS 1.00E+002
--- SQL operation complete.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ba20e3fa/core/sql/regress/udr/EXPECTED102
----------------------------------------------------------------------
diff --git a/core/sql/regress/udr/EXPECTED102 b/core/sql/regress/udr/EXPECTED102
index 04063b0..f0de120 100644
--- a/core/sql/regress/udr/EXPECTED102
+++ b/core/sql/regress/udr/EXPECTED102
@@ -66,6 +66,7 @@ HELP
LS
LSALL
PUT
+PUTFILE
RM
RMREX
@@ -88,13 +89,13 @@ CREATE PROCEDURE TRAFODION."_LIBMGR_".RM
;
--- SQL operation complete.
->>set param ?h 'PUT';
+>>set param ?h 'PUTFILE';
>>call help (?h);
COMMANDNAME
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-PUT - Upload a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]PUT for more info.
+PUTFILE - Upload a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]PUTFILE for more info.
--- SQL operation complete.
>>
@@ -247,6 +248,7 @@ HELP
LS
LSALL
PUT
+PUTFILE
RM
RMREX
@@ -273,13 +275,13 @@ CREATE PROCEDURE TRAFODION."_LIBMGR_".RM
GRANT OPTION;
--- SQL operation complete.
->>set param ?h 'PUT';
+>>set param ?h 'PUTFILE';
>>call help (?h);
COMMANDNAME
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-PUT - Upload a JAR. SHOWDDL PROCEDURE [SCHEMA NAME.]PUT for more info.
+PUTFILE - Upload a library file. SHOWDDL PROCEDURE [SCHEMA NAME.]PUTFILE for more info.
--- SQL operation complete.
>>
@@ -311,6 +313,7 @@ DROPLIB
GETFILE
HELP
LSALL
+PUTFILE
RM
RMREX
@@ -338,6 +341,7 @@ HELP
LS
LSALL
PUT
+PUTFILE
RM
RMREX
@@ -409,11 +413,48 @@ HELP
LS
LSALL
PUT
+PUTFILE
RM
RMREX
--- SQL operation complete.
>>
+>>
+>>-- put a file in a single chunk
+>>call "_LIBMGR_".putfile('abcd', 'udr_test102_1', 1, 1, 1);
+
+--- SQL operation complete.
+>>
+>>-- put a file in three chunks
+>>call "_LIBMGR_".putfile('abcd', 'udr_test102_3', 1, 0, 1);
+
+--- SQL operation complete.
+>>call "_LIBMGR_".putfile('efgh', 'udr_test102_3', 0, 0, 1);
+
+--- SQL operation complete.
+>>call "_LIBMGR_".putfile('ijkl', 'udr_test102_3', 0, 1, 1);
+
+--- SQL operation complete.
+>>
+>>-- negative test, file already exists and no overwrite requested
+>>call "_LIBMGR_".putfile('abcd', 'udr_test102_1', 1, 1, 0);
+
+*** ERROR[11220] A Java method completed with an uncaught java.sql.SQLException with invalid SQLSTATE. The uncaught exception had a SQLCODE of 0 and SQLSTATE of . Details: java.sql.SQLException: File udr_test102_1 already exists!
+
+--- SQL operation failed with errors.
+>>
+>>-- expect two lines with the contents of the two files below
+>>log;
+abcd
+abcdefghijkl
+>>
+>>call "_LIBMGR_".rm('udr_test102_1');
+
+--- SQL operation complete.
+>>call "_LIBMGR_".rm('udr_test102_3');
+
+--- SQL operation complete.
+>>
>>-- we are done
>>exit;