You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2015/03/11 02:22:07 UTC
[11/12] tajo git commit: TAJO-1384: Duplicated output file path
problem. (jihoon)
TAJO-1384: Duplicated output file path problem. (jihoon)
Closes #406
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/513a8c30
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/513a8c30
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/513a8c30
Branch: refs/heads/index_support
Commit: 513a8c3086b889d9b701d5c97eda3fb8c5453834
Parents: 3b9a2e5
Author: Jihoon Son <ji...@apache.org>
Authored: Wed Mar 11 10:07:01 2015 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Mar 11 10:07:40 2015 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/storage/StorageUtil.java | 30 ++++------
.../apache/tajo/storage/TestStorageUtil.java | 58 ++++++++++++++++++++
3 files changed, 72 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index dc1a5f7..68aabf5 100644
--- a/CHANGES
+++ b/CHANGES
@@ -16,6 +16,8 @@ Release 0.11.0 - unreleased
BUG FIXES
+ TAJO-1384: Duplicated output file path problem. (jihoon)
+
TAJO-1386: CURRENT_DATE generates parsing errors sometimes.
(Contributed by navis, Committed by hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
index 68e96d8..926b5d3 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java
@@ -18,15 +18,10 @@
package org.apache.tajo.storage;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.util.FileUtil;
import sun.nio.ch.DirectBuffer;
import java.io.DataInput;
@@ -34,8 +29,6 @@ import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
public class StorageUtil extends StorageConstants {
@@ -119,32 +112,33 @@ public class StorageUtil extends StorageConstants {
}
int maxValue = -1;
- List<Path> fileNamePatternMatchedList = new ArrayList<Path>();
for (FileStatus eachFile: files) {
// In the case of partition table, return largest value within all partition dirs.
+ int value;
if (eachFile.isDirectory() && recursive) {
- int value = getMaxFileSequence(fs, eachFile.getPath(), recursive);
+ value = getMaxFileSequence(fs, eachFile.getPath(), recursive);
if (value > maxValue) {
maxValue = value;
}
} else {
if (eachFile.getPath().getName().matches(fileNamePatternV08) ||
eachFile.getPath().getName().matches(fileNamePatternV09)) {
- fileNamePatternMatchedList.add(eachFile.getPath());
+ value = getSequence(eachFile.getPath().getName());
+ if (value > maxValue) {
+ maxValue = value;
+ }
}
}
}
- if (fileNamePatternMatchedList.isEmpty()) {
- return maxValue;
- }
- Path lastFile = fileNamePatternMatchedList.get(fileNamePatternMatchedList.size() - 1);
- String pathName = lastFile.getName();
+ return maxValue;
+ }
- // 0.8: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>
- // 0.9: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>-<Sequence>
- String[] pathTokens = pathName.split("-");
+ // 0.8: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>
+ // 0.9: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>-<Sequence>
+ private static int getSequence(String name) {
+ String[] pathTokens = name.split("-");
if (pathTokens.length == 3) {
return -1;
} else if(pathTokens.length == 4) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java
new file mode 100644
index 0000000..4b7c929
--- /dev/null
+++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.util.CommonTestingUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestStorageUtil {
+ private TajoConf conf;
+ private static String TEST_PATH = "target/test-data/TestStorageUtil";
+ private Path testDir;
+ private FileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new TajoConf();
+ testDir = CommonTestingUtil.getTestDir(TEST_PATH);
+ fs = testDir.getFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test
+ public final void testGetMaxFileSequence() throws IOException {
+ for (int i = 0; i < 7; i++) {
+ fs.create(new Path(testDir, "part-00-00000-00"+i), false);
+ }
+
+ assertEquals(6, StorageUtil.getMaxFileSequence(fs, testDir, true));
+ }
+}