You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2013/09/05 09:56:57 UTC
git commit: TAJO-80: Remove codes of legacy data paths. (jihoon)
Updated Branches:
refs/heads/master 29ecb62b5 -> 14d1f87de
TAJO-80: Remove codes of legacy data paths. (jihoon)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tajo/commit/14d1f87d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tajo/tree/14d1f87d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tajo/diff/14d1f87d
Branch: refs/heads/master
Commit: 14d1f87dec16ae9be436a8e5b0bd518851eeadce
Parents: 29ecb62
Author: Jihoon Son <ji...@apache.org>
Authored: Thu Sep 5 16:55:39 2013 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Thu Sep 5 16:55:39 2013 +0900
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../org/apache/tajo/catalog/TestTableDesc.java | 10 ++++-----
.../apache/tajo/engine/query/ResultSetImpl.java | 2 +-
.../tajo/master/TajoMasterClientService.java | 22 ++++----------------
.../tajo/master/querymaster/Repartitioner.java | 3 +--
.../tajo/master/querymaster/SubQuery.java | 5 -----
.../apache/tajo/LocalTajoTestingUtility.java | 4 +---
.../org/apache/tajo/TajoTestingCluster.java | 8 ++-----
.../org/apache/tajo/storage/TestRowFile.java | 4 +---
.../org/apache/tajo/storage/StorageManager.java | 15 ++++---------
.../apache/tajo/storage/index/TestBSTIndex.java | 2 +-
.../index/TestSingleCSVFileBSTIndex.java | 2 +-
12 files changed, 23 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 919008f..537c281 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -126,6 +126,8 @@ Release 0.2.0 - unreleased
BUG FIXES
+ TAJO-80: Remove codes of legacy data paths. (jihoon)
+
TAJO-159: QueryMaster cannot connect the remote CatalogServer. (hyunsik)
TAJO-158: Can't allocate worker when single SubQuery requests more than
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
index cf943c4..0471b04 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
@@ -39,7 +39,7 @@ public class TestTableDesc {
schema.addColumn("addr", Type.TEXT);
info = CatalogUtil.newTableMeta(schema, StoreType.CSV);
- desc = new TableDescImpl("table1", info, new Path("/nta/data"));
+ desc = new TableDescImpl("table1", info, new Path("/table1"));
}
@Test
@@ -50,17 +50,17 @@ public class TestTableDesc {
TableMeta info = CatalogUtil.newTableMeta(schema, StoreType.CSV);
testClone(info);
- TableDesc desc = new TableDescImpl("table1", info, new Path("/tajo/data"));
+ TableDesc desc = new TableDescImpl("table1", info, new Path("/tajo"));
assertEquals("table1", desc.getName());
- assertEquals(new Path("/tajo/data"), desc.getPath());
+ assertEquals(new Path("/tajo"), desc.getPath());
assertEquals(info, desc.getMeta());
testClone(desc);
}
@Test
public void testGetProto() throws CloneNotSupportedException {
- TableDesc desc = new TableDescImpl("table1", info, new Path("/tajo/data"));
+ TableDesc desc = new TableDescImpl("table1", info, new Path("/tajo"));
CatalogProtos.TableDescProto proto = (CatalogProtos.TableDescProto) desc.getProto();
TableDesc fromProto = new TableDescImpl(proto);
@@ -69,7 +69,7 @@ public class TestTableDesc {
@Test
public void testToJson() throws CloneNotSupportedException {
- TableDesc desc = new TableDescImpl("table1", info, new Path("/tajo/data"));
+ TableDesc desc = new TableDescImpl("table1", info, new Path("/tajo"));
String json = desc.toJson();
TableDesc fromJson = CatalogGsonHelper.fromJson(json, TableDesc.class);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/ResultSetImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/ResultSetImpl.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/ResultSetImpl.java
index 8cbe956..22186e2 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/ResultSetImpl.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/query/ResultSetImpl.java
@@ -138,7 +138,7 @@ public class ResultSetImpl implements ResultSet {
private Fragment[] getFragments(TableMeta meta, Path tablePath)
throws IOException {
List<Fragment> fraglist = Lists.newArrayList();
- FileStatus[] files = fs.listStatus(new Path(tablePath, "data"));
+ FileStatus[] files = fs.listStatus(tablePath);
Arrays.sort(files, new FileNameComparator());
String tbname = tablePath.getName();
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
index e65c1f3..c9cc904 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
@@ -328,31 +328,17 @@ public class TajoMasterClientService extends AbstractService {
throw new AlreadyExistsTableException(request.getName());
}
- Path path = new Path(request.getPath());
+ Path tablePath = new Path(request.getPath());
- LOG.info(path.toUri());
+ LOG.info(tablePath.toUri());
TableMeta meta;
try {
- meta = TableUtil.getTableMeta(conf, path);
+ meta = TableUtil.getTableMeta(conf, tablePath);
} catch (IOException e) {
throw new RemoteException(e);
}
- FileSystem fs;
-
- // for legacy table structure
- Path tablePath = new Path(path, "data");
- try {
- fs = path.getFileSystem(conf);
- if (!fs.exists(tablePath)) {
- tablePath = path;
- }
- } catch (IOException e) {
- LOG.error(e);
- return null;
- }
-
if (meta.getStat() == null) {
long totalSize = 0;
try {
@@ -368,7 +354,7 @@ public class TajoMasterClientService extends AbstractService {
meta.setStat(stat);
}
- desc = new TableDescImpl(request.getName(), meta, path);
+ desc = new TableDescImpl(request.getName(), meta, tablePath);
catalog.addTable(desc);
LOG.info("Table " + desc.getName() + " is attached ("
+ meta.getStat().getNumBytes() + ")");
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index 7dbc37d..cc5f848 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -83,8 +83,7 @@ public class Repartitioner {
CatalogUtil.newTableMeta(scans[i].getInSchema(), StoreType.CSV), 0, 0);
} else {
fragments[i] = subQuery.getStorageManager().getSplits(scans[i].getTableId(),
- tableDesc.getMeta(),
- new Path(tablePath, "data")).get(0);
+ tableDesc.getMeta(), tablePath).get(0);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index ecde59c..0cd5d64 100644
--- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -650,11 +650,6 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
meta = desc.getMeta();
// TODO - should be change the inner directory
- Path oldPath = new Path(inputPath, "data");
- FileSystem fs = inputPath.getFileSystem(subQuery.context.getConf());
- if (fs.exists(oldPath)) {
- inputPath = oldPath;
- }
List<Fragment> fragments = subQuery.getStorageManager().getSplits(scan.getTableId(), meta, inputPath);
QueryUnit queryUnit;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
index 1667813..2af7fb8 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -62,9 +62,7 @@ public class LocalTajoTestingUtility {
Path localPath = new Path(tablepaths[i]);
Path tablePath = new Path(rootDir, names[i]);
fs.mkdirs(tablePath);
- Path dataPath = new Path(tablePath, "data");
- fs.mkdirs(dataPath);
- Path dfsPath = new Path(dataPath, localPath.getName());
+ Path dfsPath = new Path(tablePath, localPath.getName());
fs.copyFromLocalFile(localPath, dfsPath);
TableMeta meta = CatalogUtil.newTableMeta(schemas[i],
CatalogProtos.StoreType.CSV, option);
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
index ab8d4b4..0adf22a 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/TajoTestingCluster.java
@@ -491,9 +491,7 @@ public class TajoTestingCluster {
for (int i = 0; i < tableNames.length; i++) {
File tableDir = new File(tmpDir,tableNames[i]);
tableDir.mkdirs();
- File dataDir = new File(tableDir, "data");
- dataDir.mkdirs();
- File tableFile = new File(dataDir, tableNames[i]);
+ File tableFile = new File(tableDir, tableNames[i]);
writeLines(tableFile, tables[i]);
TableMeta meta = CatalogUtil
.newTableMeta(schemas[i], CatalogProtos.StoreType.CSV, option);
@@ -528,9 +526,7 @@ public class TajoTestingCluster {
for (int i = 0; i < names.length; i++) {
Path tablePath = new Path(rootDir, names[i]);
fs.mkdirs(tablePath);
- Path dataPath = new Path(tablePath, "data");
- fs.mkdirs(dataPath);
- Path dfsPath = new Path(dataPath, names[i] + ".tbl");
+ Path dfsPath = new Path(tablePath, names[i] + ".tbl");
FSDataOutputStream out = fs.create(dfsPath);
for (int j = 0; j < tables[i].length; j++) {
out.write((tables[i][j]+"\n").getBytes());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
index 0b85fdf..e44ca99 100644
--- a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
+++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/storage/TestRowFile.java
@@ -73,11 +73,9 @@ public class TestRowFile {
Path tablePath = new Path("hdfs:///test");
Path metaPath = new Path(tablePath, ".meta");
- Path dataDir = new Path(tablePath, "data");
- Path dataPath = new Path(dataDir, "test.tbl");
+ Path dataPath = new Path(tablePath, "test.tbl");
FileSystem fs = tablePath.getFileSystem(conf);
fs.mkdirs(tablePath);
- fs.mkdirs(dataDir);
FileUtil.writeProto(util.getDefaultFileSystem(), metaPath, meta.getProto());
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
index 6de58e7..5fc6c46 100644
--- a/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-core/tajo-core-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -242,7 +242,7 @@ public class StorageManager {
List<Fragment> listTablets = new ArrayList<Fragment>();
Fragment tablet;
- FileStatus[] fileLists = fs.listStatus(new Path(tablePath, "data"));
+ FileStatus[] fileLists = fs.listStatus(tablePath);
for (FileStatus file : fileLists) {
tablet = new Fragment(tablePath.getName(), file.getPath(), meta, 0, file.getLen());
listTablets.add(tablet);
@@ -272,7 +272,7 @@ public class StorageManager {
List<Fragment> listTablets = new ArrayList<Fragment>();
Fragment tablet;
- FileStatus[] fileLists = fs.listStatus(new Path(tablePath, "data"));
+ FileStatus[] fileLists = fs.listStatus(tablePath);
for (FileStatus file : fileLists) {
long remainFileSize = file.getLen();
long start = 0;
@@ -339,16 +339,9 @@ public class StorageManager {
public long calculateSize(Path tablePath) throws IOException {
FileSystem fs = tablePath.getFileSystem(conf);
long totalSize = 0;
- Path oldPath = new Path(tablePath, "data");
- Path dataPath;
- if (fs.exists(oldPath)) {
- dataPath = oldPath;
- } else {
- dataPath = tablePath;
- }
- if (fs.exists(dataPath)) {
- for (FileStatus status : fs.listStatus(dataPath)) {
+ if (fs.exists(tablePath)) {
+ for (FileStatus status : fs.listStatus(tablePath)) {
totalSize += status.getLen();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
index 436c50e..d97a27c 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestBSTIndex.java
@@ -46,7 +46,7 @@ public class TestBSTIndex {
private static final int TUPLE_NUM = 10000;
private static final int LOAD_NUM = 100;
- private static final String TEST_PATH = "target/test-data/TestIndex/data";
+ private static final String TEST_PATH = "target/test-data/TestIndex";
private Path testDir;
private FileSystem fs;
http://git-wip-us.apache.org/repos/asf/incubator-tajo/blob/14d1f87d/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
----------------------------------------------------------------------
diff --git a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
index bfaf24e..18a69e6 100644
--- a/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
+++ b/tajo-core/tajo-core-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java
@@ -50,7 +50,7 @@ public class TestSingleCSVFileBSTIndex {
private static final int TUPLE_NUM = 10000;
private static final int LOAD_NUM = 100;
- private static final String TEST_PATH = "target/test-data/TestSingleCSVFileBSTIndex/data";
+ private static final String TEST_PATH = "target/test-data/TestSingleCSVFileBSTIndex";
private Path testDir;
public TestSingleCSVFileBSTIndex() {