You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/03/27 23:13:52 UTC
[3/5] incubator-impala git commit: IMPALA-4673: Use
--local_library_dir for tzdb startup scratch space
IMPALA-4673: Use --local_library_dir for tzdb startup scratch space
Currently we hardcode the the path to /tmp which may not be
accessible on certain environments.
Change-Id: I7ab0238e6ce65ca39c167349e3b79aa70fb39d2f
Reviewed-on: http://gerrit.cloudera.org:8080/5523
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/77dbdc3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/77dbdc3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/77dbdc3f
Branch: refs/heads/master
Commit: 77dbdc3f572260a90af174dcb9112cb991ec296c
Parents: 8e45c4a
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Thu Dec 15 12:03:16 2016 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Mon Mar 27 21:53:04 2017 +0000
----------------------------------------------------------------------
be/src/common/init.cc | 6 ++++--
be/src/exprs/expr-test.cc | 2 ++
be/src/exprs/timezone_db.cc | 15 ++++++++++-----
be/src/exprs/timezone_db.h | 2 +-
be/src/runtime/lib-cache.cc | 3 +--
be/src/service/impalad-main.cc | 2 ++
6 files changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/common/init.cc
----------------------------------------------------------------------
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 1e90f53..16f0f41 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -24,7 +24,6 @@
#include "common/status.h"
#include "exec/kudu-util.h"
#include "exprs/expr.h"
-#include "exprs/timezone_db.h"
#include "gutil/atomicops.h"
#include "rpc/authentication.h"
#include "rpc/thrift-util.h"
@@ -76,6 +75,10 @@ DEFINE_int64(pause_monitor_warn_threshold_ms, 10000, "If the pause monitor sleep
"more than this time period, a warning is logged. If set to 0 or less, pause monitor"
" is disabled.");
+DEFINE_string(local_library_dir, "/tmp",
+ "Scratch space for local fs operations. Currently used for copying "
+ "UDF binaries locally from HDFS and also for initializing the timezone db");
+
// Defined by glog. This allows users to specify the log level using a glob. For
// example -vmodule=*scanner*=3 would enable full logging for scanners. If redaction
// is enabled, this option won't be allowed because some logging dumps table data
@@ -198,7 +201,6 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
impala::InitThreading();
impala::TimestampParser::Init();
impala::SeedOpenSSLRNG();
- ABORT_IF_ERROR(impala::TimezoneDatabase::Initialize());
ABORT_IF_ERROR(impala::InitAuth(argv[0]));
// Initialize maintenance_thread after InitGoogleLoggingSafe and InitThreading.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 0ca2da2..0de6833 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -38,6 +38,7 @@
#include "exprs/null-literal.h"
#include "exprs/string-functions.h"
#include "exprs/timestamp-functions.h"
+#include "exprs/timezone_db.h"
#include "gen-cpp/Exprs_types.h"
#include "gen-cpp/hive_metastore_types.h"
#include "rpc/thrift-client.h"
@@ -7072,6 +7073,7 @@ TEST_F(ExprTest, UuidTest) {
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
InitCommonRuntime(argc, argv, true, TestInfo::BE_TEST);
+ ABORT_IF_ERROR(TimezoneDatabase::Initialize());
InitFeSupport(false);
impala::LlvmCodeGen::InitializeLlvm();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/exprs/timezone_db.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/timezone_db.cc b/be/src/exprs/timezone_db.cc
index c7ae888..110d23b 100644
--- a/be/src/exprs/timezone_db.cc
+++ b/be/src/exprs/timezone_db.cc
@@ -19,6 +19,7 @@
#include <boost/date_time/compiler_config.hpp>
#include <boost/date_time/local_time/posix_time_zone.hpp>
+#include <boost/filesystem.hpp>
#include "exprs/timestamp-functions.h"
#include "gutil/strings/substitute.h"
@@ -29,6 +30,8 @@ using boost::local_time::tz_database;
using boost::local_time::time_zone_ptr;
using boost::local_time::posix_time_zone;
+DECLARE_string(local_library_dir);
+
namespace impala {
tz_database TimezoneDatabase::tz_database_;
@@ -666,14 +669,16 @@ const char* TimezoneDatabase::TIMEZONE_DATABASE_STR = "\"ID\",\"STD ABBR\",\"STD
Status TimezoneDatabase::Initialize() {
// Create a temporary file and write the timezone information. The boost
- // interface only loads this format from a file. We don't want to raise
- // an error here since this is done when the backend is created and this
- // information might not actually get used by any queries.
- char filestr[] = "/tmp/impala.tzdb.XXXXXXX";
+ // interface only loads this format from a file. We abort the startup if
+ // this initialization fails for some reason.
+ char *filestr = const_cast<char*>((boost::filesystem::path(FLAGS_local_library_dir)
+ / string("impala.tzdb.XXXXXXX")).string().c_str());
FILE* file;
int fd;
if ((fd = mkstemp(filestr)) == -1) {
- return Status(Substitute("Could not create temporary timezone file: $0", filestr));
+ return Status(Substitute("Could not create temporary timezone file: $0. Check that "
+ "the directory $1 is writable by the user running Impala.", filestr,
+ FLAGS_local_library_dir));
}
if ((file = fopen(filestr, "w")) == NULL) {
unlink(filestr);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/exprs/timezone_db.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/timezone_db.h b/be/src/exprs/timezone_db.h
index 20ea205..3a1178a 100644
--- a/be/src/exprs/timezone_db.h
+++ b/be/src/exprs/timezone_db.h
@@ -32,7 +32,7 @@ namespace impala {
class TimezoneDatabase {
public:
/// Set up the static timezone database.
- static Status Initialize();
+ static Status Initialize() WARN_UNUSED_RESULT;
/// Converts the name of a timezone to a boost timezone object. In some cases, the
/// timestamp is required to determine the timezone because occasionally timezone
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/runtime/lib-cache.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/lib-cache.cc b/be/src/runtime/lib-cache.cc
index 9a8d873..d694c49 100644
--- a/be/src/runtime/lib-cache.cc
+++ b/be/src/runtime/lib-cache.cc
@@ -33,8 +33,7 @@
namespace filesystem = boost::filesystem;
-DEFINE_string(local_library_dir, "/tmp",
- "Local directory to copy UDF libraries from HDFS into");
+DECLARE_string(local_library_dir);
namespace impala {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/77dbdc3f/be/src/service/impalad-main.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impalad-main.cc b/be/src/service/impalad-main.cc
index 341399c..4848b45 100644
--- a/be/src/service/impalad-main.cc
+++ b/be/src/service/impalad-main.cc
@@ -27,6 +27,7 @@
#include "exec/hbase-table-scanner.h"
#include "exec/hbase-table-writer.h"
#include "exprs/hive-udf-call.h"
+#include "exprs/timezone_db.h"
#include "runtime/hbase-table.h"
#include "codegen/llvm-codegen.h"
#include "common/status.h"
@@ -59,6 +60,7 @@ DECLARE_bool(enable_rm);
int ImpaladMain(int argc, char** argv) {
InitCommonRuntime(argc, argv, true);
+ ABORT_IF_ERROR(TimezoneDatabase::Initialize());
ABORT_IF_ERROR(LlvmCodeGen::InitializeLlvm());
JniUtil::InitLibhdfs();
ABORT_IF_ERROR(HBaseTableScanner::Init());