You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2020/02/05 16:57:20 UTC
[tez] branch branch-0.9 updated: Revert "TEZ-4082. Reduce excessive
getFileLinkInfo calls in Tez"
This is an automated email from the ASF dual-hosted git repository.
jeagles pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new a15b740 Revert "TEZ-4082. Reduce excessive getFileLinkInfo calls in Tez"
a15b740 is described below
commit a15b74043f2c740b92c35500e415a9bf8a86b2da
Author: Jonathan Eagles <je...@apache.org>
AuthorDate: Wed Feb 5 10:55:40 2020 -0600
Revert "TEZ-4082. Reduce excessive getFileLinkInfo calls in Tez"
This reverts commit 67cb0a4e387e23dbbfe53838bf5069fab8c6be2b.
---
.../java/org/apache/tez/client/TezClientUtils.java | 57 ++++++++++++++--------
.../java/org/apache/tez/common/TezCommonUtils.java | 17 ++++++-
.../org/apache/tez/common/TestTezCommonUtils.java | 36 ++++----------
tez-ui/src/main/webapp/bower-shrinkwrap.json | 44 ++++++++---------
4 files changed, 84 insertions(+), 70 deletions(-)
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 07f0994..ae4f4a9 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -19,7 +19,6 @@
package org.apache.tez.client;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
@@ -122,6 +121,31 @@ public class TezClientUtils {
private static Logger LOG = LoggerFactory.getLogger(TezClientUtils.class);
private static final int UTF8_CHUNK_SIZE = 16 * 1024;
+ private static FileStatus[] getLRFileStatus(String fileName, Configuration conf) throws
+ IOException {
+ URI uri;
+ try {
+ uri = new URI(fileName);
+ } catch (URISyntaxException e) {
+ String message = "Invalid URI defined in configuration for"
+ + " location of TEZ jars. providedURI=" + fileName;
+ LOG.error(message);
+ throw new TezUncheckedException(message, e);
+ }
+
+ Path p = new Path(uri);
+ FileSystem fs = p.getFileSystem(conf);
+ p = fs.resolvePath(p.makeQualified(fs.getUri(),
+ fs.getWorkingDirectory()));
+ FileSystem targetFS = p.getFileSystem(conf);
+ if (targetFS.isDirectory(p)) {
+ return targetFS.listStatus(p);
+ } else {
+ FileStatus fStatus = targetFS.getFileStatus(p);
+ return new FileStatus[]{fStatus};
+ }
+ }
+
/**
* Setup LocalResource map for Tez jars based on provided Configuration
*
@@ -193,16 +217,8 @@ public class TezClientUtils {
}
Path p = new Path(u);
FileSystem remoteFS = p.getFileSystem(conf);
- FileStatus targetStatus = remoteFS.getFileLinkStatus(p);
- p = targetStatus.getPath();
-
- FileStatus[] fileStatuses;
- FileSystem targetFS = p.getFileSystem(conf);
- if (targetStatus.isDirectory()) {
- fileStatuses = targetFS.listStatus(p);
- } else {
- fileStatuses = new FileStatus[]{targetStatus};
- }
+ p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
+ remoteFS.getWorkingDirectory()));
LocalResourceType type = null;
@@ -216,6 +232,8 @@ public class TezClientUtils {
type = LocalResourceType.FILE;
}
+ FileStatus [] fileStatuses = getLRFileStatus(configUri, conf);
+
for (FileStatus fStatus : fileStatuses) {
String linkName;
if (fStatus.isDirectory()) {
@@ -311,16 +329,13 @@ public class TezClientUtils {
Path stagingArea)
throws IOException {
FileSystem fs = stagingArea.getFileSystem(conf);
+ String realUser;
+ String currentUser;
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
- String realUser = ugi.getShortUserName();
- String currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
- FileStatus fsStatus = null;
- try {
- fsStatus = fs.getFileStatus(stagingArea);
- } catch (FileNotFoundException e) {
- TezCommonUtils.mkDirForAM(fs, stagingArea);
- }
- if (fsStatus != null) {
+ realUser = ugi.getShortUserName();
+ currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
+ if (fs.exists(stagingArea)) {
+ FileStatus fsStatus = fs.getFileStatus(stagingArea);
String owner = fsStatus.getOwner();
if (!(owner.equals(currentUser) || owner.equals(realUser))) {
throw new IOException("The ownership on the staging directory "
@@ -335,6 +350,8 @@ public class TezClientUtils {
+ TezCommonUtils.TEZ_AM_DIR_PERMISSION);
fs.setPermission(stagingArea, TezCommonUtils.TEZ_AM_DIR_PERMISSION);
}
+ } else {
+ TezCommonUtils.mkDirForAM(fs, stagingArea);
}
return fs;
}
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
index fc4789f..16165e2 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
@@ -79,8 +79,21 @@ public class TezCommonUtils {
* @return Fully qualified staging directory
*/
public static Path getTezBaseStagingPath(Configuration conf) {
- return new Path(conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
- TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
+ String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
+ TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);
+ Path baseStagingDir;
+ try {
+ Path p = new Path(stagingDirStr);
+ FileSystem fs = p.getFileSystem(conf);
+ if (!fs.exists(p)) {
+ mkDirForAM(fs, p);
+ LOG.info("Stage directory " + p + " doesn't exist and is created");
+ }
+ baseStagingDir = fs.resolvePath(p);
+ } catch (IOException e) {
+ throw new TezUncheckedException(e);
+ }
+ return baseStagingDir;
}
/**
diff --git a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
index 52df2c6..3929c4b 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
@@ -57,7 +57,7 @@ public class TestTezCommonUtils {
LOG.info("Starting mini clusters");
try {
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
- dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).racks(null)
+ dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).racks(null)
.build();
remoteFs = dfsCluster.getFileSystem();
RESOLVED_STAGE_DIR = remoteFs.getUri() + STAGE_DIR;
@@ -80,23 +80,19 @@ public class TestTezCommonUtils {
}
// Testing base staging dir
- @Test
+ @Test(timeout = 5000)
public void testTezBaseStagingPath() throws Exception {
Configuration localConf = new Configuration();
// Check if default works with localFS
localConf.unset(TezConfiguration.TEZ_AM_STAGING_DIR);
localConf.set("fs.defaultFS", "file:///");
Path stageDir = TezCommonUtils.getTezBaseStagingPath(localConf);
- TezCommonUtils.mkDirForAM(stageDir.getFileSystem(localConf), stageDir);
- Path resolveStageDir = stageDir.getFileSystem(localConf).resolvePath(stageDir);
- Assert.assertEquals(resolveStageDir.toString(), "file:" + TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);
+ Assert.assertEquals(stageDir.toString(), "file:" + TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);
// check if user set something, indeed works
conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGE_DIR);
stageDir = TezCommonUtils.getTezBaseStagingPath(conf);
- TezCommonUtils.mkDirForAM(stageDir.getFileSystem(conf), stageDir);
- resolveStageDir = stageDir.getFileSystem(conf).resolvePath(stageDir);
- Assert.assertEquals(resolveStageDir.toString(), RESOLVED_STAGE_DIR);
+ Assert.assertEquals(stageDir.toString(), RESOLVED_STAGE_DIR);
}
// Testing System staging dir if createed
@@ -115,8 +111,7 @@ public class TestTezCommonUtils {
}
Assert.assertFalse(fs.exists(stagePath));
Path stageDir = TezCommonUtils.createTezSystemStagingPath(conf, strAppId);
- Path resolveStageDir = stageDir.getFileSystem(conf).resolvePath(stageDir);
- Assert.assertEquals(resolveStageDir.toString(), expectedStageDir);
+ Assert.assertEquals(stageDir.toString(), expectedStageDir);
Assert.assertTrue(fs.exists(stagePath));
}
@@ -127,9 +122,7 @@ public class TestTezCommonUtils {
Path stageDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
String expectedStageDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId;
- TezCommonUtils.mkDirForAM(stageDir.getFileSystem(conf), stageDir);
- Path resolvedStageDir = stageDir.getFileSystem(conf).resolvePath(stageDir);
- Assert.assertEquals(resolvedStageDir.toString(), expectedStageDir);
+ Assert.assertEquals(stageDir.toString(), expectedStageDir);
}
// Testing conf staging dir
@@ -141,9 +134,7 @@ public class TestTezCommonUtils {
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId + Path.SEPARATOR
+ TezConstants.TEZ_PB_BINARY_CONF_NAME;
- TezCommonUtils.mkDirForAM(confStageDir.getFileSystem(conf), confStageDir);
- Path resolvedConfStageDir = confStageDir.getFileSystem(conf).resolvePath(confStageDir);
- Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
+ Assert.assertEquals(confStageDir.toString(), expectedDir);
}
// Testing session jars staging dir
@@ -155,9 +146,7 @@ public class TestTezCommonUtils {
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId + Path.SEPARATOR
+ TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME;
- TezCommonUtils.mkDirForAM(confStageDir.getFileSystem(conf), confStageDir);
- Path resolvedConfStageDir = confStageDir.getFileSystem(conf).resolvePath(confStageDir);
- Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
+ Assert.assertEquals(confStageDir.toString(), expectedDir);
}
// Testing bin plan staging dir
@@ -169,9 +158,7 @@ public class TestTezCommonUtils {
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId + Path.SEPARATOR
+ TezConstants.TEZ_PB_PLAN_BINARY_NAME;
- TezCommonUtils.mkDirForAM(confStageDir.getFileSystem(conf), confStageDir);
- Path resolvedConfStageDir = confStageDir.getFileSystem(conf).resolvePath(confStageDir);
- Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
+ Assert.assertEquals(confStageDir.toString(), expectedDir);
}
// Testing text plan staging dir
@@ -180,15 +167,12 @@ public class TestTezCommonUtils {
String strAppId = "testAppId";
String dagPBName = "testDagPBName";
Path tezSysStagingPath = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
- TezCommonUtils.mkDirForAM(tezSysStagingPath.getFileSystem(conf), tezSysStagingPath);
Path confStageDir =
TezCommonUtils.getTezTextPlanStagingPath(tezSysStagingPath, strAppId, dagPBName);
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId + Path.SEPARATOR
+ strAppId + "-" + dagPBName + "-" + TezConstants.TEZ_PB_PLAN_TEXT_NAME;
- TezCommonUtils.createFileForAM(confStageDir.getFileSystem(conf), confStageDir);
- Path resolvedConfStageDir = confStageDir.getFileSystem(conf).resolvePath(confStageDir);
- Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
+ Assert.assertEquals(confStageDir.toString(), expectedDir);
}
// Testing recovery path staging dir
diff --git a/tez-ui/src/main/webapp/bower-shrinkwrap.json b/tez-ui/src/main/webapp/bower-shrinkwrap.json
index edcbae3..357d576 100644
--- a/tez-ui/src/main/webapp/bower-shrinkwrap.json
+++ b/tez-ui/src/main/webapp/bower-shrinkwrap.json
@@ -1,72 +1,72 @@
{
"https://github.com/FortAwesome/Font-Awesome.git": {
- "4.5.0": "4.5.0"
+ "4.5.0": "593ad563a987977f14102be935d0abc2a172903e"
},
"https://github.com/Teleborder/FileSaver.js.git": {
- "1.20150507.2": "1.20150507.2"
+ "1.20150507.2": "b7cf622909258086bc63ad764d08fcaed780ab42"
},
"https://github.com/adamwdraper/Numeral-js.git": {
- "1.5.3": "1.5.3"
+ "1.5.3": "f97f14bb8bab988f28f1d854525b4cfeff8ec9e1"
},
"https://github.com/components/codemirror.git": {
- "5.11.0": "5.11.0"
+ "5.11.0": "7d43f32bb56f83a9c47addb3f91170b3102f3ead"
},
"https://github.com/components/ember-data.git": {
- "2.1.0": "2.1.0"
+ "2.1.0": "d8b4d3092f67afe22d9d374c40d719d557915fa3"
},
"https://github.com/components/ember.git": {
- "2.2.0": "2.2.0"
+ "2.2.0": "49e042ca89922ed96b27488c2a98add280ae7123"
},
"https://github.com/components/jqueryui.git": {
- "1.11.4": "1.11.4"
+ "1.11.4": "c34f8dbf3ba57b3784b93f26119f436c0e8288e1"
},
"https://github.com/dockyard/ember-qunit-notifications.git": {
- "0.1.0": "0.1.0"
+ "0.1.0": "a83277aa7a1c0545c66e6d133caebb9a620e71ad"
},
"https://github.com/dockyard/qunit-notifications.git": {
- "0.1.1": "0.1.1"
+ "0.1.1": "7a13f6dba5a340e1cb9e0b64c1c711e4d7edaca1"
},
"https://github.com/ember-cli/ember-cli-shims.git": {
- "0.0.6": "0.0.6"
+ "0.0.6": "dcab43b58d5698690050bb9a46ead5c8663c7da1"
},
"https://github.com/ember-cli/ember-cli-test-loader.git": {
- "0.2.1": "0.2.1"
+ "0.2.1": "3348d801089279296c38f31ae14d9c4d115ce154"
},
"https://github.com/ember-cli/ember-load-initializers.git": {
- "0.1.7": "0.1.7"
+ "0.1.7": "7bb21488563bd1bba23e903a812bf5815beddd1a"
},
"https://github.com/jquery/jquery-dist.git": {
- "2.1.4": "2.1.4"
+ "2.1.4": "7751e69b615c6eca6f783a81e292a55725af6b85"
},
"https://github.com/jquery/jquery-mousewheel.git": {
- "3.1.13": "3.1.13"
+ "3.1.13": "67289b6b2aa0066d7d78a5807f520387135ffb22"
},
"https://github.com/jquery/qunit.git": {
- "1.19.0": "1.19.0"
+ "1.19.0": "467e7e34652ad7d5883ce9c568461cf8c5e172a8"
},
"https://github.com/moment/moment-timezone.git": {
- "0.5.0": "0.5.0"
+ "0.5.0": "74a2e9378ecf4a31a168f3049f086565c8d66814"
},
"https://github.com/moment/moment.git": {
- "2.12.0": "2.12.0"
+ "2.12.0": "d3d7488b4d60632854181cb0a9af325d57fb3d51"
},
"https://github.com/rwjblue/ember-qunit-builds.git": {
- "0.4.16": "0.4.16"
+ "0.4.16": "142c4066a5458bef9dfcb92b70152b9c01d79188"
},
"https://github.com/sreenaths/more-js.git": {
"0.8.6": "f1d9ccdaf7ff74c26b6ee341067a5a5ef33bd64a",
"0.8.8": "0.8.8"
},
"https://github.com/sreenaths/snippet-ss.git": {
- "1.11.0": "1.11.0"
+ "1.11.0": "c1abc566f4e001b7f1939b6dbdd911eadc969cf9"
},
"https://github.com/sreenaths/zip.js.git": {
- "1.0.0": "1.0.0"
+ "1.0.0": "ec67ad22eba116083ea3ef2fe0b40ccc953513ce"
},
"https://github.com/stefanpenner/loader.js.git": {
- "3.3.0": "3.3.0"
+ "3.3.0": "ac909550c9544325632542bbea97531cc60bc628"
},
"https://github.com/twbs/bootstrap.git": {
- "3.3.6": "3.3.6"
+ "3.3.6": "81df608a40bf0629a1dc08e584849bb1e43e0b7a"
}
}
\ No newline at end of file