You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2012/07/14 02:36:33 UTC
svn commit: r1361455 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project: ./ hadoop-hdfs/
hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main/native/
hadoop-hdfs/src/main/webapps/datanode/ hadoop-hdfs/src/main/webapps/hdfs/
hadoop-hdfs/src/main/webap...
Author: eli
Date: Sat Jul 14 00:36:32 2012
New Revision: 1361455
URL: http://svn.apache.org/viewvc?rev=1361455&view=rev
Log:
HDFS-3609. libhdfs: don't force the URI to look like hdfs://hostname:port. Contributed by Colin Patrick McCabe
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.c
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1361454
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1361454
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1361455&r1=1361454&r2=1361455&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sat Jul 14 00:36:32 2012
@@ -330,6 +330,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3492. fix some misuses of InputStream#skip.
(Colin Patrick McCabe via eli)
+ HDFS-3609. libhdfs: don't force the URI to look like hdfs://hostname:port.
+ (Colin Patrick McCabe via eli)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1361454
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1361454
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.c?rev=1361455&r1=1361454&r2=1361455&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.c (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.c Sat Jul 14 00:36:32 2012
@@ -406,6 +406,57 @@ hdfsFS hdfsConnectAsUserNewInstance(cons
return hdfsBuilderConnect(bld);
}
+
+/**
+ * Calculate the effective URI to use, given a builder configuration.
+ *
+ * If there is not already a URI scheme, we prepend 'hdfs://'.
+ *
+ * If there is not already a port specified, and a port was given to the
+ * builder, we suffix that port. If there is a port specified but also one in
+ * the URI, that is an error.
+ *
+ * @param bld The hdfs builder object
+ * @param uri (out param) dynamically allocated string representing the
+ * effective URI
+ *
+ * @return 0 on success; error code otherwise
+ */
+static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri)
+{
+ const char *scheme;
+ char suffix[64];
+ const char *lastColon;
+ char *u;
+ size_t uriLen;
+
+ if (!bld->nn)
+ return EINVAL;
+ scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://";
+ if (bld->port == 0) {
+ suffix[0] = '\0';
+ } else {
+ lastColon = rindex(bld->nn, ':');
+ if (lastColon && (strspn(lastColon + 1, "0123456789") ==
+ strlen(lastColon + 1))) {
+ fprintf(stderr, "port %d was given, but URI '%s' already "
+ "contains a port!\n", bld->port, bld->nn);
+ return EINVAL;
+ }
+ snprintf(suffix, sizeof(suffix), ":%d", bld->port);
+ }
+
+ uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix);
+ u = malloc((uriLen + 1) * (sizeof(char)));
+ if (!u) {
+ fprintf(stderr, "calcEffectiveURI: out of memory");
+ return ENOMEM;
+ }
+ snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix);
+ *uri = u;
+ return 0;
+}
+
hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld)
{
JNIEnv *env = 0;
@@ -414,7 +465,6 @@ hdfsFS hdfsBuilderConnect(struct hdfsBui
jstring jURIString = NULL, jUserString = NULL;
jvalue jVal;
jthrowable jExc = NULL;
- size_t cURILen;
char *cURI = 0;
int ret = 0;
@@ -472,14 +522,9 @@ hdfsFS hdfsBuilderConnect(struct hdfsBui
jURI = jVal.l;
} else {
// fs = FileSystem::get(URI, conf, ugi);
- cURILen = strlen(bld->nn) + 16;
- cURI = malloc(cURILen);
- if (!cURI) {
- fprintf(stderr, "failed to allocate memory for HDFS URI\n");
- ret = ENOMEM;
+ ret = calcEffectiveURI(bld, &cURI);
+ if (ret)
goto done;
- }
- snprintf(cURI, cURILen, "hdfs://%s:%d", bld->nn, (int)(bld->port));
jURIString = (*env)->NewStringUTF(env, cURI);
if (invokeMethod(env, &jVal, &jExc, STATIC, NULL, JAVA_NET_URI,
"create", "(Ljava/lang/String;)Ljava/net/URI;",
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h?rev=1361455&r1=1361454&r2=1361455&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native/hdfs.h Sat Jul 14 00:36:32 2012
@@ -175,11 +175,21 @@ extern "C" {
*
* @param bld The HDFS builder
* @param nn The NameNode to use.
+ *
* If the string given is 'default', the default NameNode
* configuration will be used (from the XML configuration files)
+ *
* If NULL is given, a LocalFileSystem will be created.
- * Otherwise, the string will be interpreted as a hostname or IP
- * address.
+ *
+ * If the string starts with a protocol type such as file:// or
+ * hdfs://, this protocol type will be used. If not, the
+ * hdfs:// protocol type will be used.
+ *
+ * You may specify a NameNode port in the usual way by
+ * passing a string of the format hdfs://<hostname>:<port>.
+ * Alternately, you may set the port with
+ * hdfsBuilderSetNameNodePort. However, you must not pass the
+ * port in two different ways.
*/
void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn);
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1361454
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1361454
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1361454
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1361454