You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2014/12/02 04:56:34 UTC
[01/14] tajo git commit: TAJO-1163: TableDesc should use URI instead
of Path.
Repository: tajo
Updated Branches:
refs/heads/index_support 1e9c38ded -> 708453cd8
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 42338a8..d507b97 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@ -45,7 +45,10 @@ import org.apache.tajo.engine.planner.global.DataChannel;
import org.apache.tajo.engine.planner.global.MasterPlan;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.master.session.Session;
-import org.apache.tajo.plan.*;
+import org.apache.tajo.plan.LogicalOptimizer;
+import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.LogicalPlanner;
+import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.expr.AggregationFunctionCallEval;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.plan.util.PlannerUtil;
@@ -140,7 +143,7 @@ public class TestPhysicalPlanner {
employee = new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), employeeSchema, employeeMeta,
- employeePath);
+ employeePath.toUri());
catalog.createTable(employee);
Path scorePath = new Path(testDir, "score");
@@ -149,7 +152,7 @@ public class TestPhysicalPlanner {
appender.init();
score = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta,
- scorePath);
+ scorePath.toUri());
tuple = new VTuple(scoreSchema.size());
int m = 0;
for (int i = 1; i <= 5; i++) {
@@ -192,7 +195,7 @@ public class TestPhysicalPlanner {
appender.init();
largeScore = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score_large"), scoreSchmea, scoreLargeMeta,
- scoreLargePath);
+ scoreLargePath.toUri());
Tuple tuple = new VTuple(scoreSchmea.size());
int m = 0;
@@ -244,7 +247,7 @@ public class TestPhysicalPlanner {
@Test
public final void testCreateScanPlan() throws IOException, PlanningException {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
- employee.getPath(), Integer.MAX_VALUE);
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
@@ -275,7 +278,7 @@ public class TestPhysicalPlanner {
@Test
public final void testCreateScanWithFilterPlan() throws IOException, PlanningException {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
- employee.getPath(), Integer.MAX_VALUE);
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateScanWithFilterPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
@@ -303,7 +306,7 @@ public class TestPhysicalPlanner {
@Test
public final void testGroupByPlan() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -334,7 +337,7 @@ public class TestPhysicalPlanner {
@Test
public final void testHashGroupByPlanWithALLField() throws IOException, PlanningException {
// TODO - currently, this query does not use hash-based group operator.
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir(
"target/test-data/testHashGroupByPlanWithALLField");
@@ -364,7 +367,7 @@ public class TestPhysicalPlanner {
@Test
public final void testSortGroupByPlan() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortGroupByPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -427,7 +430,7 @@ public class TestPhysicalPlanner {
@Test
public final void testStorePlan() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -475,7 +478,7 @@ public class TestPhysicalPlanner {
assertTrue("Checking meaningfulness of test", stats.getNumBytes() > StorageUnit.MB);
FileFragment[] frags = StorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(),
- largeScore.getPath(), Integer.MAX_VALUE);
+ new Path(largeScore.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithMaxOutputFileSize");
QueryContext queryContext = new QueryContext(conf, session);
@@ -525,7 +528,7 @@ public class TestPhysicalPlanner {
@Test
public final void testStorePlanWithRCFile() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlanWithRCFile");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -566,7 +569,7 @@ public class TestPhysicalPlanner {
@Test
public final void testEnforceForDefaultColumnPartitionStorePlan() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -593,7 +596,7 @@ public class TestPhysicalPlanner {
Enforcer enforcer = new Enforcer();
enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.HASH_PARTITION);
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -617,7 +620,7 @@ public class TestPhysicalPlanner {
Enforcer enforcer = new Enforcer();
enforcer.enforceColumnPartitionAlgorithm(createTableNode.getPID(), ColumnPartitionAlgorithm.SORT_PARTITION);
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testStorePlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -633,7 +636,7 @@ public class TestPhysicalPlanner {
@Test
public final void testPartitionedStorePlan() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan);
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf), id, new FileFragment[] { frags[0] },
@@ -703,7 +706,7 @@ public class TestPhysicalPlanner {
// Preparing working dir and input fragments
FileFragment[] frags = StorageManager.splitNG(conf, "default.score_large", largeScore.getMeta(),
- largeScore.getPath(), Integer.MAX_VALUE);
+ new Path(largeScore.getPath()), Integer.MAX_VALUE);
QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testPartitionedStorePlanWithMaxFileSize");
@@ -766,7 +769,7 @@ public class TestPhysicalPlanner {
@Test
public final void testPartitionedStorePlanWithEmptyGroupingSet()
throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
QueryUnitAttemptId id = LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan);
@@ -833,7 +836,7 @@ public class TestPhysicalPlanner {
@Test
public final void testAggregationFunction() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testAggregationFunction");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -864,7 +867,7 @@ public class TestPhysicalPlanner {
@Test
public final void testCountFunction() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCountFunction");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -892,7 +895,7 @@ public class TestPhysicalPlanner {
@Test
public final void testGroupByWithNullValue() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByWithNullValue");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -917,8 +920,8 @@ public class TestPhysicalPlanner {
@Test
public final void testUnionPlan() throws IOException, PlanningException, CloneNotSupportedException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testUnionPlan");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
@@ -982,8 +985,8 @@ public class TestPhysicalPlanner {
//@Test
public final void testCreateIndex() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
@@ -1009,8 +1012,8 @@ public class TestPhysicalPlanner {
@Test
public final void testDuplicateEliminate() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(),
+ new Path(score.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testDuplicateEliminate");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -1044,7 +1047,7 @@ public class TestPhysicalPlanner {
@Test
public final void testIndexedStoreExec() throws IOException, PlanningException {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
- employee.getPath(), Integer.MAX_VALUE);
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testIndexedStoreExec");
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
@@ -1137,7 +1140,7 @@ public class TestPhysicalPlanner {
@Test
public final void testSortEnforcer() throws IOException, PlanningException {
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
- employee.getPath(), Integer.MAX_VALUE);
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testSortEnforcer");
Expr context = analyzer.parse(SORT_QUERY[0]);
@@ -1187,8 +1190,8 @@ public class TestPhysicalPlanner {
@Test
public final void testGroupByEnforcer() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), score.getPath(), Integer.MAX_VALUE);
-
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.score", score.getMeta(), new Path(score.getPath()),
+ Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testGroupByEnforcer");
Expr context = analyzer.parse(QUERIES[7]);
LogicalPlan plan = planner.createPlan(defaultContext, context);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
index 9da9dee..a23a2d1 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java
@@ -109,7 +109,7 @@ public class TestProgressExternalSortExec {
testDataStats = appender.getStats();
employee = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, employeeMeta,
- employeePath);
+ employeePath.toUri());
catalog.createTable(employee);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
@@ -136,8 +136,8 @@ public class TestProgressExternalSortExec {
}
private void testProgress(int sortBufferBytesNum) throws Exception {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
index df1732c..9ebe871 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java
@@ -29,14 +29,17 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.parser.SQLAnalyzer;
-import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.engine.planner.PhysicalPlanner;
import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
-import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.engine.planner.enforce.Enforcer;
-import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.storage.*;
+import org.apache.tajo.plan.LogicalPlanner;
+import org.apache.tajo.plan.PlanningException;
+import org.apache.tajo.plan.logical.LogicalNode;
+import org.apache.tajo.storage.Appender;
+import org.apache.tajo.storage.StorageManager;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.VTuple;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.TUtil;
@@ -229,9 +232,9 @@ public class TestRightOuterHashJoinExec {
@Test
public final void testRightOuter_HashJoinExec0() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(),
+ FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags);
@@ -270,9 +273,9 @@ public class TestRightOuterHashJoinExec {
@Test
public final void testRightOuter_HashJoinExec1() throws IOException, PlanningException {
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(),
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
@@ -311,9 +314,9 @@ public class TestRightOuterHashJoinExec {
@Test
public final void testRightOuter_HashJoinExec2() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(),
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
index 77e4e4e..4956b7f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java
@@ -313,8 +313,10 @@ public class TestRightOuterMergeJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
- FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
+ Integer.MAX_VALUE);
+ FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()),
+ Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin0");
@@ -349,9 +351,9 @@ public class TestRightOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] job3Frags =
- StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin1");
@@ -385,9 +387,9 @@ public class TestRightOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] job3Frags =
- StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuterMergeJoin2");
@@ -421,9 +423,9 @@ public class TestRightOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] dep4Frags =
- StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), dep4.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testRightOuter_MergeJoin3");
@@ -458,9 +460,9 @@ public class TestRightOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] phone3Frags =
- StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
+ StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
@@ -495,8 +497,8 @@ public class TestRightOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
- FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
+ FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
index 3ed5997..afa7430 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java
@@ -97,7 +97,7 @@ public class TestSortExec {
TableDesc desc = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, employeeMeta,
- tablePath);
+ tablePath.toUri());
catalog.createTable(desc);
analyzer = new SQLAnalyzer();
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index 27d59a9..0e02079 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -71,7 +71,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(testBase.getTestingCluster().getConfiguration());
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
assertTrue(fs.isDirectory(path));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
@@ -114,7 +114,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
assertEquals("key", partitionDesc.getExpressionSchema().getColumns().get(0).getSimpleName());
FileSystem fs = FileSystem.get(cluster.getConfiguration());
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
assertTrue(fs.isDirectory(path));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
@@ -225,7 +225,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
TableDesc desc = client.getTableDesc("managed_table1");
assertNotNull(desc);
- assertEquals("managed_table1", desc.getPath().getName());
+ assertEquals("managed_table1", new Path(desc.getPath()).getName());
} else {
assertFalse(client.existTable("managed_Table1"));
assertTrue(client.existTable("MANAGED_TABLE1"));
@@ -233,7 +233,7 @@ public class TestCTASQuery extends QueryTestCaseBase {
TableDesc desc = client.getTableDesc("MANAGED_TABLE1");
assertNotNull(desc);
- assertEquals("MANAGED_TABLE1", desc.getPath().getName());
+ assertEquals("MANAGED_TABLE1", new Path(desc.getPath()).getName());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
index cb77936..18f55ff 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java
@@ -107,8 +107,9 @@ public class TestCreateTable extends QueryTestCaseBase {
// checking the existence of the table directory and validating the path
FileSystem fs = testingCluster.getMaster().getStorageManager().getFileSystem();
Path warehouseDir = TajoConf.getWarehouseDir(testingCluster.getConfiguration());
- assertTrue(fs.exists(oldTableDesc.getPath()));
- assertEquals(StorageUtil.concatPath(warehouseDir, databaseName, originalTableName), oldTableDesc.getPath());
+ assertTrue(fs.exists(new Path(oldTableDesc.getPath())));
+ assertEquals(StorageUtil.concatPath(warehouseDir, databaseName, originalTableName),
+ new Path(oldTableDesc.getPath()));
// Rename
client.executeQuery("ALTER TABLE " + CatalogUtil.denormalizeIdentifier(oldFQTableName)
@@ -117,8 +118,8 @@ public class TestCreateTable extends QueryTestCaseBase {
// checking the existence of the new table directory and validating the path
final String newFQTableName = CatalogUtil.buildFQName(databaseName, newTableName);
TableDesc newTableDesc = client.getTableDesc(newFQTableName);
- assertTrue(fs.exists(newTableDesc.getPath()));
- assertEquals(StorageUtil.concatPath(warehouseDir, databaseName, newTableName), newTableDesc.getPath());
+ assertTrue(fs.exists(new Path(newTableDesc.getPath())));
+ assertEquals(StorageUtil.concatPath(warehouseDir, databaseName, newTableName), new Path(newTableDesc.getPath()));
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
index 0d309c7..9c97a55 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
@@ -266,7 +266,7 @@ public class TestInsertQuery extends QueryTestCaseBase {
TableDesc tableDesc = testingCluster.getMaster().getCatalog().getTableDesc(getCurrentDatabase(), tableName);
assertNotNull(tableDesc);
- Path path = tableDesc.getPath();
+ Path path = new Path(tableDesc.getPath());
FileSystem fs = path.getFileSystem(testingCluster.getConfiguration());
FileStatus[] files = fs.listStatus(path);
@@ -451,10 +451,10 @@ public class TestInsertQuery extends QueryTestCaseBase {
}
FileSystem fs = FileSystem.get(testingCluster.getConfiguration());
- assertTrue(fs.exists(desc.getPath()));
+ assertTrue(fs.exists(new Path(desc.getPath())));
CompressionCodecFactory factory = new CompressionCodecFactory(testingCluster.getConfiguration());
- for (FileStatus file : fs.listStatus(desc.getPath())) {
+ for (FileStatus file : fs.listStatus(new Path(desc.getPath()))) {
CompressionCodec codec = factory.getCodec(file.getPath());
assertTrue(codec instanceof DeflateCodec);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
index 3d413b7..c1cec2b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java
@@ -563,7 +563,7 @@ public class TestJoinBroadcast extends QueryTestCaseBase {
appender.flush();
appender.close();
}
- Path dataPath = new Path(table.getPath(), fileIndex + ".csv");
+ Path dataPath = new Path(table.getPath().toString(), fileIndex + ".csv");
fileIndex++;
appender = StorageManager.getStorageManager(conf).getAppender(tableMeta, schema,
dataPath);
@@ -811,16 +811,17 @@ public class TestJoinBroadcast extends QueryTestCaseBase {
private void addEmptyDataFile(String tableName, boolean isPartitioned) throws Exception {
TableDesc table = client.getTableDesc(tableName);
- FileSystem fs = table.getPath().getFileSystem(conf);
+ Path path = new Path(table.getPath());
+ FileSystem fs = path.getFileSystem(conf);
if (isPartitioned) {
- List<Path> partitionPathList = getPartitionPathList(fs, table.getPath());
+ List<Path> partitionPathList = getPartitionPathList(fs, path);
for (Path eachPath: partitionPathList) {
Path dataPath = new Path(eachPath, 0 + "_empty.csv");
OutputStream out = fs.create(dataPath);
out.close();
}
} else {
- Path dataPath = new Path(table.getPath(), 0 + "_empty.csv");
+ Path dataPath = new Path(path, 0 + "_empty.csv");
OutputStream out = fs.create(dataPath);
out.close();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
index 6378814..0e9ec7d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
@@ -202,7 +202,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
private void assertPartitionDirectories(TableDesc desc) throws IOException {
FileSystem fs = FileSystem.get(conf);
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
assertTrue(fs.isDirectory(path));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=17.0")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/key=36.0")));
@@ -309,7 +309,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
FileSystem fs = FileSystem.get(conf);
assertTrue(fs.isDirectory(path));
@@ -373,7 +373,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
res.close();
TableDesc desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
FileSystem fs = FileSystem.get(conf);
assertTrue(fs.isDirectory(path));
@@ -426,7 +426,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
res.close();
desc = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName);
- path = desc.getPath();
+ path = new Path(desc.getPath());
assertTrue(fs.isDirectory(path));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
@@ -455,7 +455,7 @@ public class TestTablePartitions extends QueryTestCaseBase {
"R\n" +
"R\n";
- String tableData = getTableFileContents(desc.getPath());
+ String tableData = getTableFileContents(new Path(desc.getPath()));
assertEquals(expected, tableData);
res = executeString("select * from " + tableName + " where col2 = 2");
@@ -541,10 +541,10 @@ public class TestTablePartitions extends QueryTestCaseBase {
}
FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.exists(desc.getPath()));
+ assertTrue(fs.exists(new Path(desc.getPath())));
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=3")));
@@ -578,10 +578,10 @@ public class TestTablePartitions extends QueryTestCaseBase {
}
FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.exists(desc.getPath()));
+ assertTrue(fs.exists(new Path(desc.getPath())));
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=2")));
@@ -623,10 +623,10 @@ public class TestTablePartitions extends QueryTestCaseBase {
}
FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.exists(desc.getPath()));
+ assertTrue(fs.exists(new Path(desc.getPath())));
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0")));
@@ -706,10 +706,10 @@ public class TestTablePartitions extends QueryTestCaseBase {
}
FileSystem fs = FileSystem.get(conf);
- assertTrue(fs.exists(desc.getPath()));
+ assertTrue(fs.exists(new Path(desc.getPath())));
CompressionCodecFactory factory = new CompressionCodecFactory(conf);
- Path path = desc.getPath();
+ Path path = new Path(desc.getPath());
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1")));
assertTrue(fs.isDirectory(new Path(path.toUri() + "/col1=1/col2=1/col3=17.0")));
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
index 0c92ac1..f491c63 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java
@@ -95,7 +95,7 @@ public class TestResultSet {
stats.setNumBlocks(1000);
stats.setNumShuffleOutputs(100);
desc = new TableDesc(CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"),
- scoreSchema, scoreMeta, p);
+ scoreSchema, scoreMeta, p.toUri());
desc.setStats(stats);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
index e35b80c..186a7f5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
+++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java
@@ -142,7 +142,7 @@ public class TestRangeRetrieverHandler {
TableDesc employee = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, employeeMeta,
- tableDir);
+ tableDir.toUri());
catalog.createTable(employee);
FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employeeMeta, tableDir, Integer.MAX_VALUE);
@@ -265,7 +265,7 @@ public class TestRangeRetrieverHandler {
appender.close();
TableDesc employee = new TableDesc(
- CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, tablePath);
+ CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, tablePath.toUri());
catalog.createTable(employee);
FileFragment[] frags = sm.splitNG(conf, "default.employee", meta, tablePath, Integer.MAX_VALUE);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index e953ccf..3b1b4e3 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -1289,10 +1289,11 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
private void updatePhysicalInfo(TableDesc desc) {
if (desc.getPath() != null) {
try {
- FileSystem fs = desc.getPath().getFileSystem(new Configuration());
- FileStatus status = fs.getFileStatus(desc.getPath());
+ Path path = new Path(desc.getPath());
+ FileSystem fs = path.getFileSystem(new Configuration());
+ FileStatus status = fs.getFileStatus(path);
if (desc.getStats() != null && (status.isDirectory() || status.isFile())) {
- ContentSummary summary = fs.getContentSummary(desc.getPath());
+ ContentSummary summary = fs.getContentSummary(path);
if (summary != null) {
long volume = summary.getLength();
desc.getStats().setNumBytes(volume);
@@ -1676,7 +1677,7 @@ public class LogicalPlanner extends BaseAlgebraVisitor<LogicalPlanner.PlanContex
createTableNode.setExternal(parentTableDesc.isExternal());
if(parentTableDesc.isExternal()) {
- createTableNode.setPath(parentTableDesc.getPath());
+ createTableNode.setPath(new Path(parentTableDesc.getPath()));
}
return createTableNode;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java
index f860f08..df55b63 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java
@@ -46,7 +46,7 @@ public class InsertNode extends StoreTableNode implements Cloneable {
} else {
tableSchema = desc.getSchema();
}
- setPath(desc.getPath());
+ setPath(new Path(desc.getPath()));
setOptions(desc.getMeta().getOptions());
setStorageType(desc.getMeta().getStoreType());
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index 14b24a7..a7d0ce2 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -263,9 +263,9 @@ public class PartitionedTableRewriter implements RewriteRule {
if (indexablePredicateSet.size() > 0) { // There are at least one indexable predicates
return findFilteredPaths(paritionValuesSchema,
- indexablePredicateSet.toArray(new EvalNode[indexablePredicateSet.size()]), table.getPath());
+ indexablePredicateSet.toArray(new EvalNode[indexablePredicateSet.size()]), new Path(table.getPath()));
} else { // otherwise, we will get all partition paths.
- return findFilteredPaths(paritionValuesSchema, null, table.getPath());
+ return findFilteredPaths(paritionValuesSchema, null, new Path(table.getPath()));
}
}
[10/14] tajo git commit: TAJO-1119: JDBC driver should support
TIMESTAMP type. (jaehwa)
Posted by ji...@apache.org.
TAJO-1119: JDBC driver should support TIMESTAMP type. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/7d41c67c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/7d41c67c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/7d41c67c
Branch: refs/heads/index_support
Commit: 7d41c67ca94493e38c67e62663dc097358a18539
Parents: 4637f7f
Author: JaeHwa Jung <bl...@apache.org>
Authored: Thu Nov 27 23:19:31 2014 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Thu Nov 27 23:19:31 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/client/ResultSetUtil.java | 8 +
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 181 ++++++++++++++-----
.../dataset/TestTajoJdbc/table1/table1.tbl | 5 +
.../TestTajoJdbc/create_table_with_date_ddl.sql | 10 +
.../TestTajoJdbc/testSortWithDateTime.result | 7 +
.../org/apache/tajo/jdbc/MetaDataTuple.java | 4 +-
7 files changed, 174 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/7d41c67c/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 8bf0c1c..0d0677c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -77,6 +77,8 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1119: JDBC driver should support TIMESTAMP type. (jaehwa)
+
TAJO-1166: S3 related storage causes compilation error in Hadoop 2.6.0-SNAPSHOT. (jaehwa)
TAJO-1208: Failure of create table using textfile on hivemeta.
http://git-wip-us.apache.org/repos/asf/tajo/blob/7d41c67c/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java b/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java
index 056eb2c..9211a1b 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/ResultSetUtil.java
@@ -75,6 +75,10 @@ public class ResultSetUtil {
return "character";
case DATE:
return "date";
+ case TIMESTAMP:
+ return "timestamp";
+ case TIME:
+ return "time";
case VARCHAR:
return "varchar";
case TEXT:
@@ -103,7 +107,11 @@ public class ResultSetUtil {
case NUMERIC:
return Types.NUMERIC;
case DATE:
+ return Types.DATE;
+ case TIMESTAMP:
return Types.TIMESTAMP;
+ case TIME:
+ return Types.TIME;
case VARCHAR:
return Types.VARCHAR;
case TEXT:
http://git-wip-us.apache.org/repos/asf/tajo/blob/7d41c67c/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
index 08535ef..a004baa 100644
--- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
+++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java
@@ -26,6 +26,7 @@ import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.client.QueryClient;
+import org.apache.tajo.conf.TajoConf;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -33,13 +34,11 @@ import org.junit.experimental.categories.Category;
import java.net.InetSocketAddress;
import java.sql.*;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME;
import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
@Category(IntegrationTest.class)
public class TestTajoJdbc extends QueryTestCaseBase {
@@ -69,13 +68,13 @@ public class TestTajoJdbc extends QueryTestCaseBase {
@Test(expected = SQLException.class)
public void testGetConnection() throws SQLException {
DriverManager.getConnection("jdbc:taju://" + tajoMasterAddress.getHostName() + ":" + tajoMasterAddress.getPort()
- + "/default");
+ + "/default");
}
@Test
public void testStatement() throws Exception {
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- DEFAULT_DATABASE_NAME);
+ DEFAULT_DATABASE_NAME);
Connection conn = DriverManager.getConnection(connUri);
assertTrue(conn.isValid(100));
@@ -85,10 +84,10 @@ public class TestTajoJdbc extends QueryTestCaseBase {
stmt = conn.createStatement();
res = stmt.executeQuery("select l_returnflag, l_linestatus, count(*) as count_order from lineitem " +
- "group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus");
+ "group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus");
try {
- Map<String,Integer> result = Maps.newHashMap();
+ Map<String, Integer> result = Maps.newHashMap();
result.put("NO", 3);
result.put("RF", 2);
@@ -108,10 +107,10 @@ public class TestTajoJdbc extends QueryTestCaseBase {
res.close();
}
} finally {
- if(res != null) {
+ if (res != null) {
res.close();
}
- if(stmt != null) {
+ if (stmt != null) {
stmt.close();
}
}
@@ -120,7 +119,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
@Test
public void testPreparedStatement() throws Exception {
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- TajoConstants.DEFAULT_DATABASE_NAME);
+ TajoConstants.DEFAULT_DATABASE_NAME);
Connection conn = DriverManager.getConnection(connUri);
assertTrue(conn.isValid(100));
@@ -137,7 +136,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
*/
String sql =
- "select l_orderkey, l_quantity, l_returnflag from lineitem where l_quantity > ? and l_returnflag = ?";
+ "select l_orderkey, l_quantity, l_returnflag from lineitem where l_quantity > ? and l_returnflag = ?";
stmt = conn.prepareStatement(sql);
@@ -155,9 +154,9 @@ public class TestTajoJdbc extends QueryTestCaseBase {
try {
int numRows = 0;
String[] resultData = {"136.0N", "238.0N"};
- while(res.next()) {
+ while (res.next()) {
assertEquals(resultData[numRows],
- ("" + res.getObject(1).toString() + res.getObject(2).toString() + res.getObject(3).toString()));
+ ("" + res.getObject(1).toString() + res.getObject(2).toString() + res.getObject(3).toString()));
numRows++;
}
assertEquals(2, numRows);
@@ -179,9 +178,9 @@ public class TestTajoJdbc extends QueryTestCaseBase {
try {
int numRows = 0;
String[] resultData = {"345.0R", "349.0R"};
- while(res.next()) {
+ while (res.next()) {
assertEquals(resultData[numRows],
- ("" + res.getObject(1).toString() + res.getObject(2).toString() + res.getObject(3).toString()));
+ ("" + res.getObject(1).toString() + res.getObject(2).toString() + res.getObject(3).toString()));
numRows++;
}
assertEquals(2, numRows);
@@ -189,10 +188,10 @@ public class TestTajoJdbc extends QueryTestCaseBase {
res.close();
}
} finally {
- if(res != null) {
+ if (res != null) {
res.close();
}
- if(stmt != null) {
+ if (stmt != null) {
stmt.close();
}
}
@@ -201,7 +200,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
@Test
public void testDatabaseMetaDataGetTable() throws Exception {
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- TajoConstants.DEFAULT_DATABASE_NAME);
+ TajoConstants.DEFAULT_DATABASE_NAME);
Connection conn = DriverManager.getConnection(connUri);
assertTrue(conn.isValid(100));
@@ -219,12 +218,12 @@ public class TestTajoJdbc extends QueryTestCaseBase {
Set<String> retrivedViaJavaAPI = new HashSet<String>(client.getTableList("default"));
Set<String> retrievedViaJDBC = new HashSet<String>();
- while(rs.next()) {
+ while (rs.next()) {
retrievedViaJDBC.add(rs.getString("TABLE_NAME"));
}
assertEquals(retrievedViaJDBC, retrivedViaJavaAPI);
} finally {
- if(rs != null) {
+ if (rs != null) {
rs.close();
}
}
@@ -236,7 +235,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
@Test
public void testDatabaseMetaDataGetColumns() throws Exception {
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- TajoConstants.DEFAULT_DATABASE_NAME);
+ TajoConstants.DEFAULT_DATABASE_NAME);
Connection conn = DriverManager.getConnection(connUri);
assertTrue(conn.isValid(100));
@@ -258,7 +257,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
List<Column> columns = tableDesc.getSchema().getColumns();
- while(rs.next()) {
+ while (rs.next()) {
assertEquals(tableName, rs.getString("TABLE_NAME"));
assertEquals(columns.get(numColumns).getSimpleName(), rs.getString("COLUMN_NAME"));
// TODO assert type
@@ -267,7 +266,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
assertEquals(16, numColumns);
} finally {
- if(rs != null) {
+ if (rs != null) {
rs.close();
}
}
@@ -280,24 +279,24 @@ public class TestTajoJdbc extends QueryTestCaseBase {
@Test
public void testMultipleConnections() throws Exception {
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- TajoConstants.DEFAULT_DATABASE_NAME);
+ TajoConstants.DEFAULT_DATABASE_NAME);
Connection[] conns = new Connection[2];
conns[0] = DriverManager.getConnection(connUri);
conns[1] = DriverManager.getConnection(connUri);
try {
- for(int i = 0; i < conns.length; i++) {
+ for (int i = 0; i < conns.length; i++) {
Statement stmt = null;
ResultSet res = null;
try {
stmt = conns[i].createStatement();
res = stmt.executeQuery("select l_returnflag, l_linestatus, count(*) as count_order from lineitem " +
- "group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus");
+ "group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus");
try {
- Map<String,Integer> result = Maps.newHashMap();
+ Map<String, Integer> result = Maps.newHashMap();
result.put("NO", 3);
result.put("RF", 2);
@@ -317,10 +316,10 @@ public class TestTajoJdbc extends QueryTestCaseBase {
res.close();
}
} finally {
- if(res != null) {
+ if (res != null) {
res.close();
}
- if(stmt != null) {
+ if (stmt != null) {
stmt.close();
}
}
@@ -338,24 +337,24 @@ public class TestTajoJdbc extends QueryTestCaseBase {
@Test
public void testMultipleConnectionsSequentialClose() throws Exception {
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- DEFAULT_DATABASE_NAME);
+ DEFAULT_DATABASE_NAME);
Connection[] conns = new Connection[2];
conns[0] = DriverManager.getConnection(connUri);
conns[1] = DriverManager.getConnection(connUri);
try {
- for(int i = 0; i < conns.length; i++) {
+ for (int i = 0; i < conns.length; i++) {
Statement stmt = null;
ResultSet res = null;
try {
stmt = conns[i].createStatement();
res = stmt.executeQuery("select l_returnflag, l_linestatus, count(*) as count_order from lineitem " +
- "group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus");
+ "group by l_returnflag, l_linestatus order by l_returnflag, l_linestatus");
try {
- Map<String,Integer> result = Maps.newHashMap();
+ Map<String, Integer> result = Maps.newHashMap();
result.put("NO", 3);
result.put("RF", 2);
@@ -375,22 +374,22 @@ public class TestTajoJdbc extends QueryTestCaseBase {
res.close();
}
} finally {
- if(res != null) {
+ if (res != null) {
res.close();
}
- if(stmt != null) {
+ if (stmt != null) {
stmt.close();
}
conns[i].close();
}
}
} finally {
- if(!conns[0].isClosed()) {
+ if (!conns[0].isClosed()) {
assertTrue(conns[0].isValid(100));
conns[0].close();
assertFalse(conns[0].isValid(100));
}
- if(!conns[1].isClosed()) {
+ if (!conns[1].isClosed()) {
assertTrue(conns[1].isValid(100));
conns[1].close();
assertFalse(conns[1].isValid(100));
@@ -405,7 +404,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
assertFalse(TajoStatement.isSetVariableQuery("--SET JOIN_TASK_INPUT_SIZE 123"));
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- DEFAULT_DATABASE_NAME);
+ DEFAULT_DATABASE_NAME);
Connection conn = DriverManager.getConnection(connUri);
@@ -419,7 +418,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
assertNotNull(rsmd);
assertEquals(0, rsmd.getColumnCount());
- QueryClient connTajoClient = ((JdbcConnection)stmt.getConnection()).getQueryClient();
+ QueryClient connTajoClient = ((JdbcConnection) stmt.getConnection()).getQueryClient();
Map<String, String> variables = connTajoClient.getAllSessionVariables();
String value = variables.get("JOIN_TASK_INPUT_SIZE");
assertNotNull(value);
@@ -447,7 +446,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
@Test
public void testSetPreparedStatement() throws Exception {
String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
- DEFAULT_DATABASE_NAME);
+ DEFAULT_DATABASE_NAME);
Connection conn = DriverManager.getConnection(connUri);
@@ -461,7 +460,7 @@ public class TestTajoJdbc extends QueryTestCaseBase {
assertNotNull(rsmd);
assertEquals(0, rsmd.getColumnCount());
- QueryClient connTajoClient = ((JdbcConnection)stmt.getConnection()).getQueryClient();
+ QueryClient connTajoClient = ((JdbcConnection) stmt.getConnection()).getQueryClient();
Map<String, String> variables = connTajoClient.getAllSessionVariables();
String value = variables.get("JOIN_TASK_INPUT_SIZE");
assertNotNull(value);
@@ -487,4 +486,102 @@ public class TestTajoJdbc extends QueryTestCaseBase {
}
}
}
+
+ @Test
+ public void testCreateTableWithDateAndTimestamp() throws Exception {
+ String tableName = CatalogUtil.normalizeIdentifier("testCreateTableWithDateAndTimestamp");
+
+ int result;
+ Statement stmt = null;
+ ResultSet res = null;
+
+ try {
+ String connUri = buildConnectionUri(tajoMasterAddress.getHostName(), tajoMasterAddress.getPort(),
+ DEFAULT_DATABASE_NAME);
+ Connection conn = DriverManager.getConnection(connUri);
+ assertTrue(conn.isValid(100));
+
+ stmt = conn.createStatement();
+ result = stmt.executeUpdate("create table " + tableName + " (id int, name text, score double"
+ + ", register_date timestamp, update_date date, send_date time)");
+ assertEquals(result, 1);
+
+ res = stmt.executeQuery("select * from " + tableName);
+ assertFalse(res.next());
+
+ ResultSetMetaData rsmd = res.getMetaData();
+ assertNotNull(rsmd);
+ assertEquals(6, rsmd.getColumnCount());
+
+ assertEquals("id", rsmd.getColumnName(1));
+ assertEquals("name", rsmd.getColumnName(2));
+ assertEquals("score", rsmd.getColumnName(3));
+ assertEquals("register_date", rsmd.getColumnName(4));
+ assertEquals("update_date", rsmd.getColumnName(5));
+ assertEquals("send_date", rsmd.getColumnName(6));
+
+ assertEquals("integer", rsmd.getColumnTypeName(1));
+ assertEquals("varchar", rsmd.getColumnTypeName(2));
+ assertEquals("float8", rsmd.getColumnTypeName(3));
+ assertEquals("timestamp", rsmd.getColumnTypeName(4));
+ assertEquals("date", rsmd.getColumnTypeName(5));
+ assertEquals("time", rsmd.getColumnTypeName(6));
+
+ } finally {
+ cleanupQuery(res);
+ if (stmt != null) {
+ stmt.close();
+ }
+ }
+ }
+
+ @Test
+ public void testSortWithDateTime() throws Exception {
+ Statement stmt = null;
+ ResultSet res = null;
+ int result;
+
+ // skip this test if catalog uses HCatalogStore.
+ // It is because HCatalogStore does not support Time data type.
+ TimeZone oldTimeZone = TajoConf.setCurrentTimeZone(TimeZone.getTimeZone("UTC"));
+ TimeZone systemOldTimeZone = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
+
+ try {
+ if (!testingCluster.isHCatalogStoreRunning()) {
+ executeDDL("create_table_with_date_ddl.sql", "table1");
+
+ String connUri = buildConnectionUri(tajoMasterAddress.getHostName(),
+ tajoMasterAddress.getPort(), "TestTajoJdbc");
+
+ Connection conn = DriverManager.getConnection(connUri);
+ assertTrue(conn.isValid(100));
+
+ stmt = conn.createStatement();
+ res = stmt.executeQuery("select col1, col2, col3 from table1 order by col1, col2, col3");
+
+ ResultSetMetaData rsmd = res.getMetaData();
+ assertNotNull(rsmd);
+ assertEquals(3, rsmd.getColumnCount());
+
+ assertEquals("timestamp", rsmd.getColumnTypeName(1));
+ assertEquals("date", rsmd.getColumnTypeName(2));
+ assertEquals("time", rsmd.getColumnTypeName(3));
+
+ assertResultSet(res);
+
+ result = stmt.executeUpdate("drop table table1");
+ assertEquals(result, 1);
+
+ }
+ } finally {
+ TajoConf.setCurrentTimeZone(oldTimeZone);
+ TimeZone.setDefault(systemOldTimeZone);
+
+ cleanupQuery(res);
+ if (stmt != null) {
+ stmt.close();
+ }
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/7d41c67c/tajo-core/src/test/resources/dataset/TestTajoJdbc/table1/table1.tbl
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/dataset/TestTajoJdbc/table1/table1.tbl b/tajo-core/src/test/resources/dataset/TestTajoJdbc/table1/table1.tbl
new file mode 100644
index 0000000..52fa2fe
--- /dev/null
+++ b/tajo-core/src/test/resources/dataset/TestTajoJdbc/table1/table1.tbl
@@ -0,0 +1,5 @@
+1997-11-09 20:34:56|1996-04-12|15:34:56
+1997-11-09 20:34:56|1996-03-13|19:34:56
+1993-11-09 20:34:56|1997-01-28|08:34:56
+1995-11-09 20:34:56|1994-02-02|17:34:56
+1995-11-09 20:34:56|1993-11-09|20:34:56
http://git-wip-us.apache.org/repos/asf/tajo/blob/7d41c67c/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql b/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql
new file mode 100644
index 0000000..846cbb6
--- /dev/null
+++ b/tajo-core/src/test/resources/queries/TestTajoJdbc/create_table_with_date_ddl.sql
@@ -0,0 +1,10 @@
+-- Sort Table
+-- It is used in TestSortQuery::testSortWithDate
+
+create external table table1 (
+ col1 timestamp,
+ col2 date,
+ col3 time
+) using csv
+with ('csvfile.delimiter'='|', 'csvfile.null'='NULL')
+location ${table.path};
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/7d41c67c/tajo-core/src/test/resources/results/TestTajoJdbc/testSortWithDateTime.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestTajoJdbc/testSortWithDateTime.result b/tajo-core/src/test/resources/results/TestTajoJdbc/testSortWithDateTime.result
new file mode 100644
index 0000000..118909c
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestTajoJdbc/testSortWithDateTime.result
@@ -0,0 +1,7 @@
+col1,col2,col3
+-------------------------------
+1993-11-09 20:34:56,1997-01-28,08:34:56
+1995-11-09 20:34:56,1993-11-09,20:34:56
+1995-11-09 20:34:56,1994-02-02,17:34:56
+1997-11-09 20:34:56,1996-03-13,19:34:56
+1997-11-09 20:34:56,1996-04-12,15:34:56
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/7d41c67c/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java
----------------------------------------------------------------------
diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java
index 6c8ef5d..30fc9ee 100644
--- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java
+++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/MetaDataTuple.java
@@ -1,4 +1,4 @@
-package org.apache.tajo.jdbc; /**
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,6 +16,8 @@ package org.apache.tajo.jdbc; /**
* limitations under the License.
*/
+package org.apache.tajo.jdbc;
+
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.IntervalDatum;
import org.apache.tajo.datum.NullDatum;
[04/14] tajo git commit: TAJO-1166: S3 related storage causes
compilation error in Hadoop 2.6.0-SNAPSHOT. (jaehwa)
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
index 0e3ccad..0cc87fc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerResourceManager.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.QueryId;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.master.TajoMaster;
import org.apache.tajo.master.querymaster.QueryInProgress;
@@ -48,7 +49,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import static org.apache.tajo.ipc.TajoMasterProtocol.*;
@@ -80,7 +80,8 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
private TajoConf systemConf;
- private ConcurrentMap<ContainerIdProto, AllocatedWorkerResource> allocatedResourceMap = Maps.newConcurrentMap();
+ private ConcurrentMap<ContainerProtocol.TajoContainerIdProto, AllocatedWorkerResource> allocatedResourceMap = Maps
+ .newConcurrentMap();
/** It receives status messages from workers and their resources. */
private TajoResourceTracker resourceTracker;
@@ -194,7 +195,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
private WorkerResourceAllocationRequest createQMResourceRequest(QueryId queryId) {
float queryMasterDefaultDiskSlot = masterContext.getConf().getFloatVar(
- TajoConf.ConfVars.TAJO_QUERYMASTER_DISK_SLOT);
+ TajoConf.ConfVars.TAJO_QUERYMASTER_DISK_SLOT);
int queryMasterDefaultMemoryMB = masterContext.getConf().getIntVar(TajoConf.ConfVars.TAJO_QUERYMASTER_MEMORY_MB);
WorkerResourceAllocationRequest.Builder builder = WorkerResourceAllocationRequest.newBuilder();
@@ -235,7 +236,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
return resource;
}
- private void registerQueryMaster(QueryId queryId, ContainerIdProto containerId) {
+ private void registerQueryMaster(QueryId queryId, ContainerProtocol.TajoContainerIdProto containerId) {
rmContext.getQueryMasterContainer().putIfAbsent(queryId, containerId);
}
@@ -256,9 +257,9 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
WorkerResourceAllocationRequest request;
RpcCallback<WorkerResourceAllocationResponse> callBack;
WorkerResourceRequest(
- QueryId queryId,
- boolean queryMasterRequest, WorkerResourceAllocationRequest request,
- RpcCallback<WorkerResourceAllocationResponse> callBack) {
+ QueryId queryId,
+ boolean queryMasterRequest, WorkerResourceAllocationRequest request,
+ RpcCallback<WorkerResourceAllocationResponse> callBack) {
this.queryId = queryId;
this.queryMasterRequest = queryMasterRequest;
this.request = request;
@@ -282,14 +283,14 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
if (LOG.isDebugEnabled()) {
LOG.debug("allocateWorkerResources:" +
- (new QueryId(resourceRequest.request.getQueryId())) +
- ", requiredMemory:" + resourceRequest.request.getMinMemoryMBPerContainer() +
- "~" + resourceRequest.request.getMaxMemoryMBPerContainer() +
- ", requiredContainers:" + resourceRequest.request.getNumContainers() +
- ", requiredDiskSlots:" + resourceRequest.request.getMinDiskSlotPerContainer() +
- "~" + resourceRequest.request.getMaxDiskSlotPerContainer() +
- ", queryMasterRequest=" + resourceRequest.queryMasterRequest +
- ", liveWorkers=" + rmContext.getWorkers().size());
+ (new QueryId(resourceRequest.request.getQueryId())) +
+ ", requiredMemory:" + resourceRequest.request.getMinMemoryMBPerContainer() +
+ "~" + resourceRequest.request.getMaxMemoryMBPerContainer() +
+ ", requiredContainers:" + resourceRequest.request.getNumContainers() +
+ ", requiredDiskSlots:" + resourceRequest.request.getMinDiskSlotPerContainer() +
+ "~" + resourceRequest.request.getMaxDiskSlotPerContainer() +
+ ", queryMasterRequest=" + resourceRequest.queryMasterRequest +
+ ", liveWorkers=" + rmContext.getWorkers().size());
}
// TajoWorkerResourceManager can't return allocated disk slots occasionally.
@@ -300,25 +301,25 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
if(allocatedWorkerResources.size() > 0) {
List<WorkerAllocatedResource> allocatedResources =
- new ArrayList<WorkerAllocatedResource>();
+ new ArrayList<WorkerAllocatedResource>();
for(AllocatedWorkerResource allocatedResource: allocatedWorkerResources) {
NodeId nodeId = NodeId.newInstance(allocatedResource.worker.getConnectionInfo().getHost(),
- allocatedResource.worker.getConnectionInfo().getPeerRpcPort());
+ allocatedResource.worker.getConnectionInfo().getPeerRpcPort());
TajoWorkerContainerId containerId = new TajoWorkerContainerId();
containerId.setApplicationAttemptId(
- ApplicationIdUtils.createApplicationAttemptId(resourceRequest.queryId));
+ ApplicationIdUtils.createApplicationAttemptId(resourceRequest.queryId));
containerId.setId(containerIdSeq.incrementAndGet());
- ContainerIdProto containerIdProto = containerId.getProto();
+ ContainerProtocol.TajoContainerIdProto containerIdProto = containerId.getProto();
allocatedResources.add(WorkerAllocatedResource.newBuilder()
- .setContainerId(containerIdProto)
- .setConnectionInfo(allocatedResource.worker.getConnectionInfo().getProto())
- .setAllocatedMemoryMB(allocatedResource.allocatedMemoryMB)
- .setAllocatedDiskSlots(allocatedResource.allocatedDiskSlots)
- .build());
+ .setContainerId(containerIdProto)
+ .setConnectionInfo(allocatedResource.worker.getConnectionInfo().getProto())
+ .setAllocatedMemoryMB(allocatedResource.allocatedMemoryMB)
+ .setAllocatedDiskSlots(allocatedResource.allocatedDiskSlots)
+ .build());
allocatedResourceMap.putIfAbsent(containerIdProto, allocatedResource);
@@ -358,7 +359,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
int allocatedResources = 0;
TajoMasterProtocol.ResourceRequestPriority resourceRequestPriority
- = resourceRequest.request.getResourceRequestPriority();
+ = resourceRequest.request.getResourceRequestPriority();
if(resourceRequestPriority == TajoMasterProtocol.ResourceRequestPriority.MEMORY) {
synchronized(rmContext) {
@@ -369,7 +370,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
int minMemoryMB = resourceRequest.request.getMinMemoryMBPerContainer();
int maxMemoryMB = resourceRequest.request.getMaxMemoryMBPerContainer();
float diskSlot = Math.max(resourceRequest.request.getMaxDiskSlotPerContainer(),
- resourceRequest.request.getMinDiskSlotPerContainer());
+ resourceRequest.request.getMinDiskSlotPerContainer());
int liveWorkerSize = randomWorkers.size();
Set<Integer> insufficientWorkers = new HashSet<Integer>();
@@ -418,7 +419,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
}
workerResource.allocateResource(allocatedWorkerResource.allocatedDiskSlots,
- allocatedWorkerResource.allocatedMemoryMB);
+ allocatedWorkerResource.allocatedMemoryMB);
selectedWorkers.add(allocatedWorkerResource);
@@ -438,7 +439,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
float minDiskSlots = resourceRequest.request.getMinDiskSlotPerContainer();
float maxDiskSlots = resourceRequest.request.getMaxDiskSlotPerContainer();
int memoryMB = Math.max(resourceRequest.request.getMaxMemoryMBPerContainer(),
- resourceRequest.request.getMinMemoryMBPerContainer());
+ resourceRequest.request.getMinMemoryMBPerContainer());
int liveWorkerSize = randomWorkers.size();
Set<Integer> insufficientWorkers = new HashSet<Integer>();
@@ -487,7 +488,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
allocatedWorkerResource.allocatedMemoryMB = workerResource.getAvailableMemoryMB();
}
workerResource.allocateResource(allocatedWorkerResource.allocatedDiskSlots,
- allocatedWorkerResource.allocatedMemoryMB);
+ allocatedWorkerResource.allocatedMemoryMB);
selectedWorkers.add(allocatedWorkerResource);
@@ -508,7 +509,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
* @param containerId ContainerIdProto to be released
*/
@Override
- public void releaseWorkerResource(ContainerIdProto containerId) {
+ public void releaseWorkerResource(ContainerProtocol.TajoContainerIdProto containerId) {
AllocatedWorkerResource allocated = allocatedResourceMap.get(containerId);
if(allocated != null) {
LOG.info("Release Resource: " + allocated.allocatedDiskSlots + "," + allocated.allocatedMemoryMB);
@@ -530,7 +531,7 @@ public class TajoWorkerResourceManager extends CompositeService implements Worke
LOG.warn("No QueryMaster resource info for " + queryId);
return;
} else {
- ContainerIdProto containerId = rmContext.getQueryMasterContainer().remove(queryId);
+ ContainerProtocol.TajoContainerIdProto containerId = rmContext.getQueryMasterContainer().remove(queryId);
releaseWorkerResource(containerId);
rmContext.getStoppedQueryIds().add(queryId);
LOG.info(String.format("Released QueryMaster (%s) resource." , queryId.toString()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java
index 8e8ac51..9c2b71b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/WorkerResourceManager.java
@@ -20,8 +20,8 @@ package org.apache.tajo.master.rm;
import com.google.protobuf.RpcCallback;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.tajo.QueryId;
+import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.master.querymaster.QueryInProgress;
@@ -63,7 +63,7 @@ public interface WorkerResourceManager extends Service {
*
* @param containerId ContainerIdProto to be released
*/
- public void releaseWorkerResource(ContainerIdProto containerId);
+ public void releaseWorkerResource(ContainerProtocol.TajoContainerIdProto containerId);
public String getSeedQueryId() throws IOException;
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java
index ca71c53..68c57f2 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/AbstractResourceAllocator.java
@@ -20,9 +20,9 @@ package org.apache.tajo.worker;
import com.google.common.collect.Maps;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tajo.master.ContainerProxy;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.container.TajoContainerId;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
@@ -41,29 +41,29 @@ public abstract class AbstractResourceAllocator extends CompositeService impleme
workerInfoMap.putIfAbsent(connectionInfo.getId(), connectionInfo);
}
- private Map<ContainerId, ContainerProxy> containers = Maps.newConcurrentMap();
+ private Map<TajoContainerId, ContainerProxy> containers = Maps.newConcurrentMap();
public AbstractResourceAllocator() {
super(AbstractResourceAllocator.class.getName());
}
- public void addContainer(ContainerId cId, ContainerProxy container) {
+ public void addContainer(TajoContainerId cId, ContainerProxy container) {
containers.put(cId, container);
}
- public void removeContainer(ContainerId cId) {
+ public void removeContainer(TajoContainerId cId) {
containers.remove(cId);
}
- public boolean containsContainer(ContainerId cId) {
+ public boolean containsContainer(TajoContainerId cId) {
return containers.containsKey(cId);
}
- public ContainerProxy getContainer(ContainerId cId) {
+ public ContainerProxy getContainer(TajoContainerId cId) {
return containers.get(cId);
}
- public Map<ContainerId, ContainerProxy> getContainers() {
+ public Map<TajoContainerId, ContainerProxy> getContainers() {
return containers;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java
index 8b9219c..b713e70 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/ResourceAllocator.java
@@ -18,12 +18,12 @@
package org.apache.tajo.worker;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.proto.YarnProtos;
+import org.apache.tajo.ipc.ContainerProtocol;
+import org.apache.tajo.master.container.TajoContainerId;
public interface ResourceAllocator {
public void allocateTaskWorker();
- public ContainerId makeContainerId(YarnProtos.ContainerIdProto containerId);
+ public TajoContainerId makeContainerId(ContainerProtocol.TajoContainerIdProto containerId);
public int calculateNumRequestContainers(TajoWorker.WorkerContext workerContext,
int numTasks, int memoryMBPerTask);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
index 2220089..9345885 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoResourceAllocator.java
@@ -23,28 +23,25 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.*;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.container.TajoContainer;
+import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.event.ContainerAllocationEvent;
import org.apache.tajo.master.event.ContainerAllocatorEventType;
import org.apache.tajo.master.event.SubQueryContainerAllocationEvent;
import org.apache.tajo.master.querymaster.QueryMasterTask;
import org.apache.tajo.master.querymaster.SubQuery;
import org.apache.tajo.master.querymaster.SubQueryState;
-import org.apache.tajo.master.rm.TajoWorkerContainer;
-import org.apache.tajo.master.rm.TajoWorkerContainerId;
-import org.apache.tajo.master.rm.Worker;
-import org.apache.tajo.master.rm.WorkerResource;
+import org.apache.tajo.master.rm.*;
import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.NullCallback;
@@ -72,11 +69,11 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
public TajoResourceAllocator(QueryMasterTask.QueryMasterTaskContext queryTaskContext) {
this.queryTaskContext = queryTaskContext;
executorService = Executors.newFixedThreadPool(
- queryTaskContext.getConf().getIntVar(TajoConf.ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM));
+ queryTaskContext.getConf().getIntVar(TajoConf.ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM));
}
@Override
- public ContainerId makeContainerId(YarnProtos.ContainerIdProto containerIdProto) {
+ public TajoContainerId makeContainerId(ContainerProtocol.TajoContainerIdProto containerIdProto) {
TajoWorkerContainerId containerId = new TajoWorkerContainerId();
ApplicationAttemptId appAttemptId = new ApplicationAttemptIdPBImpl(containerIdProto.getAppAttemptId());
containerId.setApplicationAttemptId(appAttemptId);
@@ -98,7 +95,7 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
int clusterSlots = clusterResource == null ? 0 : clusterResource.getTotalMemoryMB() / memoryMBPerTask;
clusterSlots = Math.max(1, clusterSlots - 1); // reserve query master slot
LOG.info("CalculateNumberRequestContainer - Number of Tasks=" + numTasks +
- ", Number of Cluster Slots=" + clusterSlots);
+ ", Number of Cluster Slots=" + clusterSlots);
return Math.min(numTasks, clusterSlots);
}
@@ -121,7 +118,8 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
executorService.shutdownNow();
- Map<ContainerId, ContainerProxy> containers = queryTaskContext.getResourceAllocator().getContainers();
+ Map<TajoContainerId, ContainerProxy> containers = queryTaskContext.getResourceAllocator()
+ .getContainers();
List<ContainerProxy> list = new ArrayList<ContainerProxy>(containers.values());
for(ContainerProxy eachProxy: list) {
try {
@@ -156,16 +154,17 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
private void launchTaskRunners(LaunchTaskRunnersEvent event) {
// Query in standby mode doesn't need launch Worker.
// But, Assign ExecutionBlock to assigned tajo worker
- for(Container eachContainer: event.getContainers()) {
+ for(TajoContainer eachContainer: event.getContainers()) {
TajoContainerProxy containerProxy = new TajoContainerProxy(queryTaskContext, tajoConf,
- eachContainer, event.getQueryContext(), event.getExecutionBlockId(), event.getPlanJson());
+ eachContainer, event.getQueryContext(), event.getExecutionBlockId(), event.getPlanJson());
executorService.submit(new LaunchRunner(eachContainer.getId(), containerProxy));
}
}
- public void stopExecutionBlock(final ExecutionBlockId executionBlockId, Collection<Container> containers) {
+ public void stopExecutionBlock(final ExecutionBlockId executionBlockId,
+ Collection<TajoContainer> containers) {
Set<NodeId> workers = Sets.newHashSet();
- for (Container container : containers){
+ for (TajoContainer container : containers){
workers.add(container.getNodeId());
}
@@ -196,8 +195,8 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
protected static class LaunchRunner implements Runnable {
private final ContainerProxy proxy;
- private final ContainerId id;
- public LaunchRunner(ContainerId id, ContainerProxy proxy) {
+ private final TajoContainerId id;
+ public LaunchRunner(TajoContainerId id, ContainerProxy proxy) {
this.proxy = proxy;
this.id = id;
}
@@ -210,8 +209,8 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
}
}
- private void stopContainers(Collection<Container> containers) {
- for (Container container : containers) {
+ private void stopContainers(Collection<TajoContainer> containers) {
+ for (TajoContainer container : containers) {
final ContainerProxy proxy = queryTaskContext.getResourceAllocator().getContainer(container.getId());
executorService.submit(new StopContainerRunner(container.getId(), proxy));
}
@@ -219,8 +218,8 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
private static class StopContainerRunner implements Runnable {
private final ContainerProxy proxy;
- private final ContainerId id;
- public StopContainerRunner(ContainerId id, ContainerProxy proxy) {
+ private final TajoContainerId id;
+ public StopContainerRunner(TajoContainerId id, ContainerProxy proxy) {
this.id = id;
this.proxy = proxy;
}
@@ -251,23 +250,23 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
public void run() {
LOG.info("Start TajoWorkerAllocationThread");
CallFuture<TajoMasterProtocol.WorkerResourceAllocationResponse> callBack =
- new CallFuture<TajoMasterProtocol.WorkerResourceAllocationResponse>();
+ new CallFuture<TajoMasterProtocol.WorkerResourceAllocationResponse>();
//TODO consider task's resource usage pattern
int requiredMemoryMB = tajoConf.getIntVar(TajoConf.ConfVars.TASK_DEFAULT_MEMORY);
float requiredDiskSlots = tajoConf.getFloatVar(TajoConf.ConfVars.TASK_DEFAULT_DISK);
TajoMasterProtocol.WorkerResourceAllocationRequest request =
- TajoMasterProtocol.WorkerResourceAllocationRequest.newBuilder()
- .setMinMemoryMBPerContainer(requiredMemoryMB)
- .setMaxMemoryMBPerContainer(requiredMemoryMB)
- .setNumContainers(event.getRequiredNum())
- .setResourceRequestPriority(!event.isLeafQuery() ? TajoMasterProtocol.ResourceRequestPriority.MEMORY
- : TajoMasterProtocol.ResourceRequestPriority.DISK)
- .setMinDiskSlotPerContainer(requiredDiskSlots)
- .setMaxDiskSlotPerContainer(requiredDiskSlots)
- .setQueryId(event.getExecutionBlockId().getQueryId().getProto())
- .build();
+ TajoMasterProtocol.WorkerResourceAllocationRequest.newBuilder()
+ .setMinMemoryMBPerContainer(requiredMemoryMB)
+ .setMaxMemoryMBPerContainer(requiredMemoryMB)
+ .setNumContainers(event.getRequiredNum())
+ .setResourceRequestPriority(!event.isLeafQuery() ? TajoMasterProtocol.ResourceRequestPriority.MEMORY
+ : TajoMasterProtocol.ResourceRequestPriority.DISK)
+ .setMinDiskSlotPerContainer(requiredDiskSlots)
+ .setMaxDiskSlotPerContainer(requiredDiskSlots)
+ .setQueryId(event.getExecutionBlockId().getQueryId().getProto())
+ .build();
RpcConnectionPool connPool = RpcConnectionPool.getPool(queryTaskContext.getConf());
NettyClientBase tmClient = null;
@@ -280,21 +279,21 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
if (tajoConf.getBoolVar(TajoConf.ConfVars.TAJO_MASTER_HA_ENABLE)) {
try {
tmClient = connPool.getConnection(
- queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(),
- TajoMasterProtocol.class, true);
+ queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(),
+ TajoMasterProtocol.class, true);
} catch (Exception e) {
queryTaskContext.getQueryMasterContext().getWorkerContext().
- setWorkerResourceTrackerAddr(HAServiceUtil.getResourceTrackerAddress(tajoConf));
+ setWorkerResourceTrackerAddr(HAServiceUtil.getResourceTrackerAddress(tajoConf));
queryTaskContext.getQueryMasterContext().getWorkerContext().
- setTajoMasterAddress(HAServiceUtil.getMasterUmbilicalAddress(tajoConf));
+ setTajoMasterAddress(HAServiceUtil.getMasterUmbilicalAddress(tajoConf));
tmClient = connPool.getConnection(
- queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(),
- TajoMasterProtocol.class, true);
+ queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(),
+ TajoMasterProtocol.class, true);
}
} else {
tmClient = connPool.getConnection(
- queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(),
- TajoMasterProtocol.class, true);
+ queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(),
+ TajoMasterProtocol.class, true);
}
TajoMasterProtocol.TajoMasterProtocolService masterClientService = tmClient.getStub();
@@ -325,17 +324,17 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
List<TajoMasterProtocol.WorkerAllocatedResource> allocatedResources = response.getWorkerAllocatedResourceList();
ExecutionBlockId executionBlockId = event.getExecutionBlockId();
- List<Container> containers = new ArrayList<Container>();
+ List<TajoContainer> containers = new ArrayList<TajoContainer>();
for(TajoMasterProtocol.WorkerAllocatedResource eachAllocatedResource: allocatedResources) {
TajoWorkerContainer container = new TajoWorkerContainer();
NodeId nodeId = NodeId.newInstance(eachAllocatedResource.getConnectionInfo().getHost(),
- eachAllocatedResource.getConnectionInfo().getPeerRpcPort());
+ eachAllocatedResource.getConnectionInfo().getPeerRpcPort());
TajoWorkerContainerId containerId = new TajoWorkerContainerId();
containerId.setApplicationAttemptId(
- ApplicationIdUtils.createApplicationAttemptId(executionBlockId.getQueryId(),
- eachAllocatedResource.getContainerId().getAppAttemptId().getAttemptId()));
+ ApplicationIdUtils.createApplicationAttemptId(executionBlockId.getQueryId(),
+ eachAllocatedResource.getContainerId().getAppAttemptId().getAttemptId()));
containerId.setId(eachAllocatedResource.getContainerId().getId());
container.setId(containerId);
@@ -347,7 +346,7 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
workerResource.setDiskSlots(eachAllocatedResource.getAllocatedDiskSlots());
Worker worker = new Worker(null, workerResource,
- new WorkerConnectionInfo(eachAllocatedResource.getConnectionInfo()));
+ new WorkerConnectionInfo(eachAllocatedResource.getConnectionInfo()));
container.setWorkerResource(worker);
addWorkerConnectionInfo(worker.getConnectionInfo());
containers.add(container);
@@ -356,8 +355,8 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
SubQueryState state = queryTaskContext.getSubQuery(executionBlockId).getSynchronizedState();
if (!SubQuery.isRunningState(state)) {
try {
- List<ContainerId> containerIds = new ArrayList<ContainerId>();
- for(Container eachContainer: containers) {
+ List<TajoContainerId> containerIds = new ArrayList<TajoContainerId>();
+ for(TajoContainer eachContainer: containers) {
containerIds.add(eachContainer.getId());
}
TajoContainerProxy.releaseWorkerResource(queryTaskContext, executionBlockId, containerIds);
@@ -378,10 +377,10 @@ public class TajoResourceAllocator extends AbstractResourceAllocator {
}
if(event.getRequiredNum() > numAllocatedContainers) {
ContainerAllocationEvent shortRequestEvent = new ContainerAllocationEvent(
- event.getType(), event.getExecutionBlockId(), event.getPriority(),
- event.getResource(),
- event.getRequiredNum() - numAllocatedContainers,
- event.isLeafQuery(), event.getProgress()
+ event.getType(), event.getExecutionBlockId(), event.getPriority(),
+ event.getResource(),
+ event.getRequiredNum() - numAllocatedContainers,
+ event.isLeafQuery(), event.getProgress()
);
queryTaskContext.getEventHandler().handle(shortRequestEvent);
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
index 1910575..4e9860b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunner.java
@@ -24,15 +24,15 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.engine.query.QueryUnitRequestImpl;
import org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService;
+import org.apache.tajo.master.container.TajoContainerId;
+import org.apache.tajo.master.container.impl.pb.TajoContainerIdPBImpl;
+import org.apache.tajo.master.container.TajoConverterUtils;
import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.rpc.NullCallback;
import org.jboss.netty.channel.ConnectTimeoutException;
@@ -53,7 +53,7 @@ public class TaskRunner extends AbstractService {
private volatile boolean stopped = false;
private Path baseDirPath;
- private ContainerId containerId;
+ private TajoContainerId containerId;
// for Fetcher
private ExecutorService fetchLauncher;
@@ -77,7 +77,7 @@ public class TaskRunner extends AbstractService {
this.fetchLauncher = Executors.newFixedThreadPool(
systemConf.getIntVar(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM), fetcherFactory);
try {
- this.containerId = ConverterUtils.toContainerId(containerId);
+ this.containerId = TajoConverterUtils.toTajoContainerId(containerId);
this.executionBlockContext = executionBlockContext;
this.history = executionBlockContext.createTaskRunnerHistory(this);
this.history.setState(getServiceState());
@@ -91,11 +91,11 @@ public class TaskRunner extends AbstractService {
return getId(getContext().getExecutionBlockId(), containerId);
}
- public ContainerId getContainerId(){
+ public TajoContainerId getContainerId(){
return containerId;
}
- public static String getId(ExecutionBlockId executionBlockId, ContainerId containerId) {
+ public static String getId(ExecutionBlockId executionBlockId, TajoContainerId containerId) {
return executionBlockId + "," + containerId;
}
@@ -211,7 +211,7 @@ public class TaskRunner extends AbstractService {
LOG.info("Request GetTask: " + getId());
GetTaskRequestProto request = GetTaskRequestProto.newBuilder()
.setExecutionBlockId(getExecutionBlockId().getProto())
- .setContainerId(((ContainerIdPBImpl) containerId).getProto())
+ .setContainerId(((TajoContainerIdPBImpl) containerId).getProto())
.setWorkerId(getContext().getWorkerContext().getConnectionInfo().getId())
.build();
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
index a8a11c1..364348f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
+++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskRunnerHistory.java
@@ -21,11 +21,11 @@ package org.apache.tajo.worker;
import com.google.common.base.Objects;
import com.google.common.collect.Maps;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.common.ProtoObject;
+import org.apache.tajo.master.container.TajoContainerId;
+import org.apache.tajo.master.container.TajoConverterUtils;
import java.util.Collections;
import java.util.Map;
@@ -39,13 +39,13 @@ import static org.apache.tajo.ipc.TajoWorkerProtocol.TaskRunnerHistoryProto;
public class TaskRunnerHistory implements ProtoObject<TaskRunnerHistoryProto> {
private Service.STATE state;
- private ContainerId containerId;
+ private TajoContainerId containerId;
private long startTime;
private long finishTime;
private ExecutionBlockId executionBlockId;
private Map<QueryUnitAttemptId, TaskHistory> taskHistoryMap = null;
- public TaskRunnerHistory(ContainerId containerId, ExecutionBlockId executionBlockId) {
+ public TaskRunnerHistory(TajoContainerId containerId, ExecutionBlockId executionBlockId) {
init();
this.containerId = containerId;
this.executionBlockId = executionBlockId;
@@ -53,7 +53,7 @@ public class TaskRunnerHistory implements ProtoObject<TaskRunnerHistoryProto> {
public TaskRunnerHistory(TaskRunnerHistoryProto proto) {
this.state = Service.STATE.valueOf(proto.getState());
- this.containerId = ConverterUtils.toContainerId(proto.getContainerId());
+ this.containerId = TajoConverterUtils.toTajoContainerId(proto.getContainerId());
this.startTime = proto.getStartTime();
this.finishTime = proto.getFinishTime();
this.executionBlockId = new ExecutionBlockId(proto.getExecutionBlockId());
@@ -129,11 +129,11 @@ public class TaskRunnerHistory implements ProtoObject<TaskRunnerHistoryProto> {
this.state = state;
}
- public ContainerId getContainerId() {
+ public TajoContainerId getContainerId() {
return containerId;
}
- public void setContainerId(ContainerId containerId) {
+ public void setContainerId(TajoContainerId containerId) {
this.containerId = containerId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/proto/ContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/ContainerProtocol.proto b/tajo-core/src/main/proto/ContainerProtocol.proto
new file mode 100644
index 0000000..df7a450
--- /dev/null
+++ b/tajo-core/src/main/proto/ContainerProtocol.proto
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are public and stable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *stable* .proto interface.
+ */
+
+option java_package = "org.apache.tajo.ipc";
+option java_outer_classname = "ContainerProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+package hadoop.yarn;
+
+import "Security.proto";
+import "yarn_protos.proto";
+
+message TajoContainerIdProto {
+ optional ApplicationIdProto app_id = 1;
+ optional ApplicationAttemptIdProto app_attempt_id = 2;
+ optional int32 id = 3;
+}
+
+message TajoContainerProto {
+ optional TajoContainerIdProto id = 1;
+ optional NodeIdProto nodeId = 2;
+ optional string node_http_address = 3;
+ optional ResourceProto resource = 4;
+ optional PriorityProto priority = 5;
+ optional hadoop.common.TokenProto container_token = 6;
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/proto/QueryMasterProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/QueryMasterProtocol.proto b/tajo-core/src/main/proto/QueryMasterProtocol.proto
index 06d2a42..494d296 100644
--- a/tajo-core/src/main/proto/QueryMasterProtocol.proto
+++ b/tajo-core/src/main/proto/QueryMasterProtocol.proto
@@ -27,6 +27,9 @@ import "TajoIdProtos.proto";
import "CatalogProtos.proto";
import "PrimitiveProtos.proto";
import "TajoWorkerProtocol.proto";
+import "ContainerProtocol.proto";
+
+package hadoop.yarn;
service QueryMasterProtocolService {
//from Worker
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/ResourceTrackerProtocol.proto b/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
index b117cac..b2db46a 100644
--- a/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
+++ b/tajo-core/src/main/proto/ResourceTrackerProtocol.proto
@@ -23,8 +23,11 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
import "TajoMasterProtocol.proto";
+import "ContainerProtocol.proto";
import "tajo_protos.proto";
+package hadoop.yarn;
+
message NodeHeartbeat {
required WorkerConnectionInfoProto connectionInfo = 1;
optional ServerStatusProto serverStatus = 2;
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/proto/TajoMasterProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/TajoMasterProtocol.proto b/tajo-core/src/main/proto/TajoMasterProtocol.proto
index 7283543..e5eab4f 100644
--- a/tajo-core/src/main/proto/TajoMasterProtocol.proto
+++ b/tajo-core/src/main/proto/TajoMasterProtocol.proto
@@ -28,6 +28,9 @@ import "tajo_protos.proto";
import "TajoIdProtos.proto";
import "CatalogProtos.proto";
import "PrimitiveProtos.proto";
+import "ContainerProtocol.proto";
+
+package hadoop.yarn;
message ServerStatusProto {
message System {
@@ -119,11 +122,11 @@ message WorkerResourcesRequest {
message WorkerResourceReleaseRequest {
required ExecutionBlockIdProto executionBlockId = 1;
- repeated hadoop.yarn.ContainerIdProto containerIds = 2;
+ repeated TajoContainerIdProto containerIds = 2;
}
message WorkerAllocatedResource {
- required hadoop.yarn.ContainerIdProto containerId = 1;
+ required TajoContainerIdProto containerId = 1;
required WorkerConnectionInfoProto connectionInfo = 2;
required int32 allocatedMemoryMB = 3;
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/proto/TajoWorkerProtocol.proto
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
index e515438..989b0e3 100644
--- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto
+++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto
@@ -28,6 +28,9 @@ import "TajoIdProtos.proto";
import "CatalogProtos.proto";
import "PrimitiveProtos.proto";
import "Plan.proto";
+import "ContainerProtocol.proto";
+
+package hadoop.yarn;
message SessionProto {
required string session_id = 1;
@@ -170,7 +173,7 @@ message QueryExecutionRequestProto {
message GetTaskRequestProto {
required int32 workerId = 1;
- required hadoop.yarn.ContainerIdProto containerId = 2;
+ required TajoContainerIdProto containerId = 2;
required ExecutionBlockIdProto executionBlockId = 3;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java b/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java
index 0423894..b8fbd67 100644
--- a/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java
+++ b/tajo-core/src/test/java/org/apache/tajo/master/rm/TestTajoResourceManager.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.tajo.QueryId;
import org.apache.tajo.QueryIdFactory;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.TajoMasterProtocol.*;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.rpc.NullCallback;
@@ -150,7 +151,8 @@ public class TestTajoResourceManager {
.build();
final CountDownLatch barrier = new CountDownLatch(1);
- final List<YarnProtos.ContainerIdProto> containerIds = new ArrayList<YarnProtos.ContainerIdProto>();
+ final List<ContainerProtocol.TajoContainerIdProto> containerIds = new
+ ArrayList<ContainerProtocol.TajoContainerIdProto>();
RpcCallback<WorkerResourceAllocationResponse> callBack = new RpcCallback<WorkerResourceAllocationResponse>() {
@@ -190,7 +192,7 @@ public class TestTajoResourceManager {
containerIds.add(eachResource.getContainerId());
}
- for(YarnProtos.ContainerIdProto eachContainerId: containerIds) {
+ for(ContainerProtocol.TajoContainerIdProto eachContainerId: containerIds) {
tajoWorkerResourceManager.releaseWorkerResource(eachContainerId);
}
@@ -318,7 +320,8 @@ public class TestTajoResourceManager {
.build();
final CountDownLatch barrier = new CountDownLatch(1);
- final List<YarnProtos.ContainerIdProto> containerIds = new ArrayList<YarnProtos.ContainerIdProto>();
+ final List<ContainerProtocol.TajoContainerIdProto> containerIds = new
+ ArrayList<ContainerProtocol.TajoContainerIdProto>();
RpcCallback<WorkerResourceAllocationResponse> callBack = new RpcCallback<WorkerResourceAllocationResponse>() {
@@ -356,7 +359,7 @@ public class TestTajoResourceManager {
assertEquals(numWorkers * 3, response.getWorkerAllocatedResourceList().size());
- for(YarnProtos.ContainerIdProto eachContainerId: containerIds) {
+ for(ContainerProtocol.TajoContainerIdProto eachContainerId: containerIds) {
tajoWorkerResourceManager.releaseWorkerResource(eachContainerId);
}
@@ -399,7 +402,8 @@ public class TestTajoResourceManager {
.build();
final CountDownLatch barrier = new CountDownLatch(1);
- final List<YarnProtos.ContainerIdProto> containerIds = new ArrayList<YarnProtos.ContainerIdProto>();
+ final List<ContainerProtocol.TajoContainerIdProto> containerIds = new
+ ArrayList<ContainerProtocol.TajoContainerIdProto>();
RpcCallback<WorkerResourceAllocationResponse> callBack = new RpcCallback<WorkerResourceAllocationResponse>() {
@@ -431,7 +435,7 @@ public class TestTajoResourceManager {
assertEquals(0, totalUsedDisks, 0);
- for(YarnProtos.ContainerIdProto eachContainerId: containerIds) {
+ for(ContainerProtocol.TajoContainerIdProto eachContainerId: containerIds) {
tajoWorkerResourceManager.releaseWorkerResource(eachContainerId);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
index 87b4197..220eb6c 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java
@@ -567,6 +567,7 @@ public class StorageManager {
for (Path p : inputs) {
FileSystem fs = p.getFileSystem(conf);
+
ArrayList<FileStatus> files = Lists.newArrayList();
if (fs.isFile(p)) {
files.addAll(Lists.newArrayList(fs.getFileStatus(p)));
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-storage/src/test/java/org/apache/tajo/storage/TestFileSystems.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestFileSystems.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestFileSystems.java
index a355a94..bec0daf 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestFileSystems.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestFileSystems.java
@@ -1,129 +1,138 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3.S3FileSystem;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
-import org.apache.tajo.common.TajoDataTypes.Type;
-import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.DatumFactory;
-import org.apache.tajo.storage.fragment.FileFragment;
-import org.apache.tajo.storage.s3.InMemoryFileSystemStore;
-import org.apache.tajo.storage.s3.SmallBlockS3FileSystem;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-@RunWith(Parameterized.class)
-public class TestFileSystems {
-
- protected byte[] data = null;
-
- private static String TEST_PATH = "target/test-data/TestFileSystem";
- private TajoConf conf = null;
- private StorageManager sm = null;
- private FileSystem fs = null;
- Path testDir;
-
- public TestFileSystems(FileSystem fs) throws IOException {
- conf = new TajoConf();
-
- if(fs instanceof S3FileSystem){
- conf.set(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, "10");
- fs.initialize(URI.create(fs.getScheme() + ":///"), conf);
- }
- this.fs = fs;
- sm = StorageManager.getStorageManager(conf);
- testDir = getTestDir(this.fs, TEST_PATH);
- }
-
- public Path getTestDir(FileSystem fs, String dir) throws IOException {
- Path path = new Path(dir);
- if(fs.exists(path))
- fs.delete(path, true);
-
- fs.mkdirs(path);
-
- return fs.makeQualified(path);
- }
-
- @Parameterized.Parameters
- public static Collection<Object[]> generateParameters() {
- return Arrays.asList(new Object[][] {
- {new SmallBlockS3FileSystem(new InMemoryFileSystemStore())},
- });
- }
-
- @Test
- public void testBlockSplit() throws IOException {
-
- Schema schema = new Schema();
- schema.addColumn("id", Type.INT4);
- schema.addColumn("age", Type.INT4);
- schema.addColumn("name", Type.TEXT);
-
- TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
-
- Tuple[] tuples = new Tuple[4];
- for (int i = 0; i < tuples.length; i++) {
- tuples[i] = new VTuple(3);
- tuples[i]
- .put(new Datum[] { DatumFactory.createInt4(i),
- DatumFactory.createInt4(i + 32),
- DatumFactory.createText("name" + i) });
- }
-
- Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender",
- "table.csv");
- fs.mkdirs(path.getParent());
-
- Appender appender = sm.getAppender(meta, schema, path);
- appender.init();
- for (Tuple t : tuples) {
- appender.addTuple(t);
- }
- appender.close();
- FileStatus fileStatus = fs.getFileStatus(path);
-
- List<FileFragment> splits = sm.getSplits("table", meta, schema, path);
- int splitSize = (int) Math.ceil(fileStatus.getLen() / (double) fileStatus.getBlockSize());
- assertEquals(splitSize, splits.size());
-
- for (FileFragment fragment : splits) {
- assertTrue(fragment.getEndKey() <= fileStatus.getBlockSize());
- }
- }
-}
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.??See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.??The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.??You may obtain a copy of the License at
+ *
+ *?????http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.storage.fragment.FileFragment;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(Parameterized.class)
+public class TestFileSystems {
+
+ private static String TEST_PATH = "target/test-data/TestFileSystem";
+ private Configuration conf;
+ private StorageManager sm;
+ private FileSystem fs;
+ private Path testDir;
+
+ public TestFileSystems(FileSystem fs) throws IOException {
+ this.fs = fs;
+ this.conf = fs.getConf();
+ this.testDir = getTestDir(this.fs, TEST_PATH);
+ this.sm = StorageManager.getStorageManager(new TajoConf(this.conf));
+ }
+
+ public Path getTestDir(FileSystem fs, String dir) throws IOException {
+ Path path = new Path(dir);
+ if (fs.exists(path))
+ fs.delete(path, true);
+
+ fs.mkdirs(path);
+
+ return fs.makeQualified(path);
+ }
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> generateParameters() throws IOException {
+ return Arrays.asList(new Object[][]{
+ {FileSystem.getLocal(new TajoConf())},
+ });
+ }
+
+ @Before
+ public void setup() throws IOException {
+ if (!(fs instanceof LocalFileSystem)) {
+ conf.set("fs.local.block.size", "10");
+ fs.initialize(URI.create(fs.getScheme() + ":///"), conf);
+ fs.setConf(conf);
+ }
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (!(fs instanceof LocalFileSystem)) {
+ fs.setConf(new TajoConf());
+ }
+ }
+
+ @Test
+ public void testBlockSplit() throws IOException {
+
+ Schema schema = new Schema();
+ schema.addColumn("id", Type.INT4);
+ schema.addColumn("age", Type.INT4);
+ schema.addColumn("name", Type.TEXT);
+
+ TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
+
+ Tuple[] tuples = new Tuple[4];
+ for (int i = 0; i < tuples.length; i++) {
+ tuples[i] = new VTuple(3);
+ tuples[i]
+ .put(new Datum[]{DatumFactory.createInt4(i),
+ DatumFactory.createInt4(i + 32),
+ DatumFactory.createText("name" + i)});
+ }
+
+ Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender",
+ "table.csv");
+ fs.mkdirs(path.getParent());
+
+ Appender appender = sm.getAppender(meta, schema, path);
+ appender.init();
+ for (Tuple t : tuples) {
+ appender.addTuple(t);
+ }
+ appender.close();
+ FileStatus fileStatus = fs.getFileStatus(path);
+
+ List<FileFragment> splits = sm.getSplits("table", meta, schema, path);
+ int splitSize = (int) Math.ceil(fileStatus.getLen() / (double) fileStatus.getBlockSize());
+ assertEquals(splitSize, splits.size());
+
+ for (FileFragment fragment : splits) {
+ assertTrue(fragment.getEndKey() <= fileStatus.getBlockSize());
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-storage/src/test/java/org/apache/tajo/storage/s3/INode.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/INode.java b/tajo-storage/src/test/java/org/apache/tajo/storage/s3/INode.java
deleted file mode 100644
index 7b09937..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/INode.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.s3;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.s3.Block;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.*;
-
-/**
- * Holds file metadata including type (regular file, or directory),
- * and the list of blocks that are pointers to the data.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class INode {
-
- enum FileType {
- DIRECTORY, FILE
- }
-
- public static final FileType[] FILE_TYPES = {
- FileType.DIRECTORY,
- FileType.FILE
- };
-
- public static final INode DIRECTORY_INODE = new INode(FileType.DIRECTORY, null);
-
- private FileType fileType;
- private Block[] blocks;
-
- public INode(FileType fileType, Block[] blocks) {
- this.fileType = fileType;
- if (isDirectory() && blocks != null) {
- throw new IllegalArgumentException("A directory cannot contain blocks.");
- }
- this.blocks = blocks;
- }
-
- public Block[] getBlocks() {
- return blocks;
- }
-
- public FileType getFileType() {
- return fileType;
- }
-
- public boolean isDirectory() {
- return fileType == FileType.DIRECTORY;
- }
-
- public boolean isFile() {
- return fileType == FileType.FILE;
- }
-
- public long getSerializedLength() {
- return 1L + (blocks == null ? 0 : 4 + blocks.length * 16);
- }
-
-
- public InputStream serialize() throws IOException {
- ByteArrayOutputStream bytes = new ByteArrayOutputStream();
- DataOutputStream out = new DataOutputStream(bytes);
- try {
- out.writeByte(fileType.ordinal());
- if (isFile()) {
- out.writeInt(blocks.length);
- for (int i = 0; i < blocks.length; i++) {
- out.writeLong(blocks[i].getId());
- out.writeLong(blocks[i].getLength());
- }
- }
- out.close();
- out = null;
- } finally {
- IOUtils.closeStream(out);
- }
- return new ByteArrayInputStream(bytes.toByteArray());
- }
-
- public static INode deserialize(InputStream in) throws IOException {
- if (in == null) {
- return null;
- }
- DataInputStream dataIn = new DataInputStream(in);
- FileType fileType = INode.FILE_TYPES[dataIn.readByte()];
- switch (fileType) {
- case DIRECTORY:
- in.close();
- return INode.DIRECTORY_INODE;
- case FILE:
- int numBlocks = dataIn.readInt();
- Block[] blocks = new Block[numBlocks];
- for (int i = 0; i < numBlocks; i++) {
- long id = dataIn.readLong();
- long length = dataIn.readLong();
- blocks[i] = new Block(id, length);
- }
- in.close();
- return new INode(fileType, blocks);
- default:
- throw new IllegalArgumentException("Cannot deserialize inode.");
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-storage/src/test/java/org/apache/tajo/storage/s3/InMemoryFileSystemStore.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/InMemoryFileSystemStore.java b/tajo-storage/src/test/java/org/apache/tajo/storage/s3/InMemoryFileSystemStore.java
deleted file mode 100644
index 40decc2..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/InMemoryFileSystemStore.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.s3;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3.Block;
-import org.apache.hadoop.fs.s3.FileSystemStore;
-import org.apache.hadoop.fs.s3.INode;
-import org.apache.tajo.common.exception.NotImplementedException;
-
-import java.io.*;
-import java.net.URI;
-import java.util.*;
-
-/**
- * A stub implementation of {@link FileSystemStore} for testing
- * {@link S3FileSystem} without actually connecting to S3.
- */
-public class InMemoryFileSystemStore implements FileSystemStore {
-
- private Configuration conf;
- private SortedMap<Path, INode> inodes = new TreeMap<Path, INode>();
- private Map<Long, byte[]> blocks = new HashMap<Long, byte[]>();
-
- @Override
- public void initialize(URI uri, Configuration conf) {
- this.conf = conf;
- }
-
- @Override
- public String getVersion() throws IOException {
- return "0";
- }
-
- @Override
- public void deleteINode(Path path) throws IOException {
- inodes.remove(normalize(path));
- }
-
- @Override
- public void deleteBlock(Block block) throws IOException {
- blocks.remove(block.getId());
- }
-
- @Override
- public boolean inodeExists(Path path) throws IOException {
- return inodes.containsKey(normalize(path));
- }
-
- @Override
- public boolean blockExists(long blockId) throws IOException {
- return blocks.containsKey(blockId);
- }
-
- @Override
- public INode retrieveINode(Path path) throws IOException {
- return inodes.get(normalize(path));
- }
-
- @Override
- public File retrieveBlock(Block block, long byteRangeStart) throws IOException {
- byte[] data = blocks.get(block.getId());
- File file = createTempFile();
- BufferedOutputStream out = null;
- try {
- out = new BufferedOutputStream(new FileOutputStream(file));
- out.write(data, (int) byteRangeStart, data.length - (int) byteRangeStart);
- } finally {
- if (out != null) {
- out.close();
- }
- }
- return file;
- }
-
- private File createTempFile() throws IOException {
- File dir = new File(conf.get("fs.s3.buffer.dir"));
- if (!dir.exists() && !dir.mkdirs()) {
- throw new IOException("Cannot create S3 buffer directory: " + dir);
- }
- File result = File.createTempFile("test-", ".tmp", dir);
- result.deleteOnExit();
- return result;
- }
-
- @Override
- public Set<Path> listSubPaths(Path path) throws IOException {
- Path normalizedPath = normalize(path);
- // This is inefficient but more than adequate for testing purposes.
- Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(normalizedPath).keySet()) {
- if (normalizedPath.equals(p.getParent())) {
- subPaths.add(p);
- }
- }
- return subPaths;
- }
-
- @Override
- public Set<Path> listDeepSubPaths(Path path) throws IOException {
- Path normalizedPath = normalize(path);
- String pathString = normalizedPath.toUri().getPath();
- if (!pathString.endsWith("/")) {
- pathString += "/";
- }
- // This is inefficient but more than adequate for testing purposes.
- Set<Path> subPaths = new LinkedHashSet<Path>();
- for (Path p : inodes.tailMap(normalizedPath).keySet()) {
- if (p.toUri().getPath().startsWith(pathString)) {
- subPaths.add(p);
- }
- }
- return subPaths;
- }
-
- @Override
- public void storeINode(Path path, INode inode) throws IOException {
- inodes.put(normalize(path), inode);
- }
-
- @Override
- public void storeBlock(Block block, File file) throws IOException {
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- byte[] buf = new byte[8192];
- int numRead;
- BufferedInputStream in = null;
- try {
- in = new BufferedInputStream(new FileInputStream(file));
- while ((numRead = in.read(buf)) >= 0) {
- out.write(buf, 0, numRead);
- }
- } finally {
- if (in != null) {
- in.close();
- }
- }
- blocks.put(block.getId(), out.toByteArray());
- }
-
- private Path normalize(Path path) {
- if (!path.isAbsolute()) {
- throw new IllegalArgumentException("Path must be absolute: " + path);
- }
- return new Path(path.toUri().getPath());
- }
-
- @Override
- public void purge() throws IOException {
- inodes.clear();
- blocks.clear();
- }
-
- @Override
- public void dump() throws IOException {
- throw new NotImplementedException();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-storage/src/test/java/org/apache/tajo/storage/s3/S3OutputStream.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/S3OutputStream.java b/tajo-storage/src/test/java/org/apache/tajo/storage/s3/S3OutputStream.java
deleted file mode 100644
index d4034b9..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/S3OutputStream.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.s3;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.s3.Block;
-import org.apache.hadoop.fs.s3.FileSystemStore;
-import org.apache.hadoop.fs.s3.INode;
-import org.apache.hadoop.util.Progressable;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-class S3OutputStream extends OutputStream {
-
- private Configuration conf;
-
- private int bufferSize;
-
- private FileSystemStore store;
-
- private Path path;
-
- private long blockSize;
-
- private File backupFile;
-
- private OutputStream backupStream;
-
- private Random r = new Random();
-
- private boolean closed;
-
- private int pos = 0;
-
- private long filePos = 0;
-
- private int bytesWrittenToBlock = 0;
-
- private byte[] outBuf;
-
- private List<Block> blocks = new ArrayList<Block>();
-
- private Block nextBlock;
-
- private static final Log LOG =
- LogFactory.getLog(S3OutputStream.class.getName());
-
-
- public S3OutputStream(Configuration conf, FileSystemStore store,
- Path path, long blockSize, Progressable progress,
- int buffersize) throws IOException {
-
- this.conf = conf;
- this.store = store;
- this.path = path;
- this.blockSize = blockSize;
- this.backupFile = newBackupFile();
- this.backupStream = new FileOutputStream(backupFile);
- this.bufferSize = buffersize;
- this.outBuf = new byte[bufferSize];
-
- }
-
- private File newBackupFile() throws IOException {
- File dir = new File(conf.get("fs.s3.buffer.dir"));
- if (!dir.exists() && !dir.mkdirs()) {
- throw new IOException("Cannot create S3 buffer directory: " + dir);
- }
- File result = File.createTempFile("output-", ".tmp", dir);
- result.deleteOnExit();
- return result;
- }
-
- public long getPos() throws IOException {
- return filePos;
- }
-
- @Override
- public synchronized void write(int b) throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
-
- if ((bytesWrittenToBlock + pos == blockSize) || (pos >= bufferSize)) {
- flush();
- }
- outBuf[pos++] = (byte) b;
- filePos++;
- }
-
- @Override
- public synchronized void write(byte b[], int off, int len) throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
- while (len > 0) {
- int remaining = bufferSize - pos;
- int toWrite = Math.min(remaining, len);
- System.arraycopy(b, off, outBuf, pos, toWrite);
- pos += toWrite;
- off += toWrite;
- len -= toWrite;
- filePos += toWrite;
-
- if ((bytesWrittenToBlock + pos >= blockSize) || (pos == bufferSize)) {
- flush();
- }
- }
- }
-
- @Override
- public synchronized void flush() throws IOException {
- if (closed) {
- throw new IOException("Stream closed");
- }
-
- if (bytesWrittenToBlock + pos >= blockSize) {
- flushData((int) blockSize - bytesWrittenToBlock);
- }
- if (bytesWrittenToBlock == blockSize) {
- endBlock();
- }
- flushData(pos);
- }
-
- private synchronized void flushData(int maxPos) throws IOException {
- int workingPos = Math.min(pos, maxPos);
-
- if (workingPos > 0) {
- //
- // To the local block backup, write just the bytes
- //
- backupStream.write(outBuf, 0, workingPos);
-
- //
- // Track position
- //
- bytesWrittenToBlock += workingPos;
- System.arraycopy(outBuf, workingPos, outBuf, 0, pos - workingPos);
- pos -= workingPos;
- }
- }
-
- private synchronized void endBlock() throws IOException {
- //
- // Done with local copy
- //
- backupStream.close();
-
- //
- // Send it to S3
- //
- // TODO: Use passed in Progressable to report progress.
- nextBlockOutputStream();
- store.storeBlock(nextBlock, backupFile);
- Block[] arr = new Block[blocks.size()];
- arr = blocks.toArray(arr);
- store.storeINode(path, new INode(INode.FILE_TYPES[1], arr));
-
- //
- // Delete local backup, start new one
- //
- boolean b = backupFile.delete();
- if (!b) {
- LOG.warn("Ignoring failed delete");
- }
- backupFile = newBackupFile();
- backupStream = new FileOutputStream(backupFile);
- bytesWrittenToBlock = 0;
- }
-
- private synchronized void nextBlockOutputStream() throws IOException {
- long blockId = r.nextLong();
- while (store.blockExists(blockId)) {
- blockId = r.nextLong();
- }
- nextBlock = new Block(blockId, bytesWrittenToBlock);
- blocks.add(nextBlock);
- bytesWrittenToBlock = 0;
- }
-
-
- @Override
- public synchronized void close() throws IOException {
- if (closed) {
- return;
- }
-
- flush();
- if (filePos == 0 || bytesWrittenToBlock != 0) {
- endBlock();
- }
-
- backupStream.close();
- boolean b = backupFile.delete();
- if (!b) {
- LOG.warn("Ignoring failed delete");
- }
-
- super.close();
-
- closed = true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-storage/src/test/java/org/apache/tajo/storage/s3/SmallBlockS3FileSystem.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/SmallBlockS3FileSystem.java b/tajo-storage/src/test/java/org/apache/tajo/storage/s3/SmallBlockS3FileSystem.java
deleted file mode 100644
index fc1c908..0000000
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/s3/SmallBlockS3FileSystem.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage.s3;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.s3.Block;
-import org.apache.hadoop.fs.s3.FileSystemStore;
-import org.apache.hadoop.fs.s3.INode;
-import org.apache.hadoop.fs.s3.S3FileSystem;
-import org.apache.hadoop.util.Progressable;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.List;
-
-public class SmallBlockS3FileSystem extends S3FileSystem {
-
- private URI uri;
-
- private FileSystemStore store;
-
- private Path workingDir;
-
- static class Holder {
- private static InMemoryFileSystemStore s;
-
- public synchronized static FileSystemStore get() {
- if(s != null) {
- return s;
- }
- s = new InMemoryFileSystemStore();
- return s;
- }
-
- public synchronized static void set(InMemoryFileSystemStore inMemoryFileSystemStore) {
- s = inMemoryFileSystemStore;
- }
- }
-
- public SmallBlockS3FileSystem() {
- }
-
-
- public SmallBlockS3FileSystem(
- InMemoryFileSystemStore inMemoryFileSystemStore) {
- Holder.set(inMemoryFileSystemStore);
- this.store = inMemoryFileSystemStore;
- }
-
- @Override
- public URI getUri() {
- return uri;
- }
- @Override
- public long getDefaultBlockSize() {
- return 10;
- }
-
- @Override
- public void initialize(URI uri, Configuration conf) throws IOException {
- if (store == null) {
- store = Holder.get();
- }
- store.initialize(uri, conf);
- setConf(conf);
- this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
- this.workingDir =
- new Path("/user", System.getProperty("user.name")).makeQualified(this);
- }
- @Override
- public boolean isFile(Path path) throws IOException {
- INode inode = store.retrieveINode(makeAbsolute(path));
- if (inode == null) {
- return false;
- }
- return inode.isFile();
- }
-
- private INode checkFile(Path path) throws IOException {
- INode inode = store.retrieveINode(makeAbsolute(path));
- if (inode == null) {
- throw new IOException("No such file.");
- }
- if (inode.isDirectory()) {
- throw new IOException("Path " + path + " is a directory.");
- }
- return inode;
- }
-
- @Override
- public FileStatus[] listStatus(Path f) throws IOException {
- Path absolutePath = makeAbsolute(f);
- INode inode = store.retrieveINode(absolutePath);
- if (inode == null) {
- throw new FileNotFoundException("File " + f + " does not exist.");
- }
- if (inode.isFile()) {
- return new FileStatus[] {
- new S3FileStatus(f.makeQualified(this), inode)
- };
- }
- ArrayList<FileStatus> ret = new ArrayList<FileStatus>();
- for (Path p : store.listSubPaths(absolutePath)) {
- ret.add(getFileStatus(p.makeQualified(this)));
- }
- return ret.toArray(new FileStatus[0]);
- }
- @Override
- public FSDataOutputStream create(Path file, FsPermission permission,
- boolean overwrite, int bufferSize,
- short replication, long blockSize, Progressable progress)
- throws IOException {
-
- INode inode = store.retrieveINode(makeAbsolute(file));
- if (inode != null) {
- if (overwrite) {
- delete(file, true);
- } else {
- throw new IOException("File already exists: " + file);
- }
- } else {
- Path parent = file.getParent();
- if (parent != null) {
- if (!mkdirs(parent)) {
- throw new IOException("Mkdirs failed to create " + parent.toString());
- }
- }
- }
- return new FSDataOutputStream
- (new S3OutputStream(getConf(), store, makeAbsolute(file),
- blockSize, progress, bufferSize),
- statistics);
- }
- @Override
- public boolean mkdirs(Path path, FsPermission permission) throws IOException {
- Path absolutePath = makeAbsolute(path);
- List<Path> paths = new ArrayList<Path>();
- do {
- paths.add(0, absolutePath);
- absolutePath = absolutePath.getParent();
- } while (absolutePath != null);
-
- boolean result = true;
- for (Path p : paths) {
- result &= mkdir(p);
- }
- return result;
- }
-
- @Override
- public Path getWorkingDirectory() {
- return workingDir;
- }
-
- @Override
- public boolean rename(Path src, Path dst) throws IOException {
- Path absoluteSrc = makeAbsolute(src);
- INode srcINode = store.retrieveINode(absoluteSrc);
- if (srcINode == null) {
- // src path doesn't exist
- return false;
- }
- Path absoluteDst = makeAbsolute(dst);
- INode dstINode = store.retrieveINode(absoluteDst);
- if (dstINode != null && dstINode.isDirectory()) {
- absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
- dstINode = store.retrieveINode(absoluteDst);
- }
- if (dstINode != null) {
- // dst path already exists - can't overwrite
- return false;
- }
- Path dstParent = absoluteDst.getParent();
- if (dstParent != null) {
- INode dstParentINode = store.retrieveINode(dstParent);
- if (dstParentINode == null || dstParentINode.isFile()) {
- // dst parent doesn't exist or is a file
- return false;
- }
- }
- return renameRecursive(absoluteSrc, absoluteDst);
- }
-
- private boolean renameRecursive(Path src, Path dst) throws IOException {
- INode srcINode = store.retrieveINode(src);
- store.storeINode(dst, srcINode);
- store.deleteINode(src);
- if (srcINode.isDirectory()) {
- for (Path oldSrc : store.listDeepSubPaths(src)) {
- INode inode = store.retrieveINode(oldSrc);
- if (inode == null) {
- return false;
- }
- String oldSrcPath = oldSrc.toUri().getPath();
- String srcPath = src.toUri().getPath();
- String dstPath = dst.toUri().getPath();
- Path newDst = new Path(oldSrcPath.replaceFirst(srcPath, dstPath));
- store.storeINode(newDst, inode);
- store.deleteINode(oldSrc);
- }
- }
- return true;
- }
-
- @Override
- public boolean delete(Path path, boolean recursive) throws IOException {
- Path absolutePath = makeAbsolute(path);
- INode inode = store.retrieveINode(absolutePath);
- if (inode == null) {
- return false;
- }
- if (inode.isFile()) {
- store.deleteINode(absolutePath);
- for (Block block: inode.getBlocks()) {
- store.deleteBlock(block);
- }
- } else {
- FileStatus[] contents = null;
- try {
- contents = listStatus(absolutePath);
- } catch(FileNotFoundException fnfe) {
- return false;
- }
-
- if ((contents.length !=0) && (!recursive)) {
- throw new IOException("Directory " + path.toString()
- + " is not empty.");
- }
- for (FileStatus p:contents) {
- if (!delete(p.getPath(), recursive)) {
- return false;
- }
- }
- store.deleteINode(absolutePath);
- }
- return true;
- }
-
- /**
- * FileStatus for S3 file systems.
- */
- @Override
- public FileStatus getFileStatus(Path f) throws IOException {
- INode inode = store.retrieveINode(makeAbsolute(f));
- if (inode == null) {
- throw new FileNotFoundException(f + ": No such file or directory.");
- }
- return new S3FileStatus(f.makeQualified(this), inode);
- }
- private boolean mkdir(Path path) throws IOException {
- Path absolutePath = makeAbsolute(path);
- INode inode = store.retrieveINode(absolutePath);
- if (inode == null) {
- store.storeINode(absolutePath, INode.DIRECTORY_INODE);
- } else if (inode.isFile()) {
- throw new IOException(String.format(
- "Can't make directory for path %s since it is a file.",
- absolutePath));
- }
- return true;
- }
- private Path makeAbsolute(Path path) {
- if (path.isAbsolute()) {
- return path;
- }
- return new Path(workingDir, path);
- }
-
- private static class S3FileStatus extends FileStatus {
-
- S3FileStatus(Path f, INode inode) throws IOException {
- super(findLength(inode), inode.isDirectory(), 1,
- findBlocksize(inode), 0, f);
- }
-
- private static long findLength(INode inode) {
- if (!inode.isDirectory()) {
- long length = 0L;
- for (Block block : inode.getBlocks()) {
- length += block.getLength();
- }
- return length;
- }
- return 0;
- }
-
- private static long findBlocksize(INode inode) {
- final Block[] ret = inode.getBlocks();
- return ret == null ? 0L : ret[0].getLength();
- }
- }
-}
\ No newline at end of file
[08/14] tajo git commit: TAJO-1204: Remove unused ServerName class.
(DaeMyung Kang via jaehwa)
Posted by ji...@apache.org.
TAJO-1204: Remove unused ServerName class. (DaeMyung Kang via jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f3291621
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f3291621
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f3291621
Branch: refs/heads/index_support
Commit: f3291621c31bbc24aff47dbc363ddfe2a1a9b21c
Parents: 72dd29c
Author: JaeHwa Jung <bl...@apache.org>
Authored: Thu Nov 27 22:28:06 2014 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Thu Nov 27 22:28:06 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../apache/tajo/master/cluster/ServerName.java | 123 -------------------
.../org/apache/tajo/cluster/TestServerName.java | 102 ---------------
3 files changed, 2 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/f3291621/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6f38f65..9b15898 100644
--- a/CHANGES
+++ b/CHANGES
@@ -13,6 +13,8 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1204: Remove unused ServerName class. (DaeMyung Kang via jaehwa)
+
TAJO-1053: ADD PARTITIONS for HCatalogStore. (jaehwa)
TAJO-1195: Remove unused CachedDNSResolver Class. (DaeMyung Kang via jaehwa)
http://git-wip-us.apache.org/repos/asf/tajo/blob/f3291621/tajo-core/src/main/java/org/apache/tajo/master/cluster/ServerName.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/cluster/ServerName.java b/tajo-core/src/main/java/org/apache/tajo/master/cluster/ServerName.java
deleted file mode 100644
index 028af65..0000000
--- a/tajo-core/src/main/java/org/apache/tajo/master/cluster/ServerName.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.master.cluster;
-
-public class ServerName implements Comparable<ServerName> {
- /**
- * This character is used as separator between server hostname and port.
- */
- public static final String SERVERNAME_SEPARATOR = ":";
-
- private final String serverName;
- private final String hostname;
- private final int port;
-
-
- public ServerName(final String hostname, final int port) {
- this.hostname = hostname;
- this.port = port;
- this.serverName = getServerName(hostname, port);
- }
-
- public ServerName(final String serverName) {
- this(parseHostname(serverName), parsePort(serverName));
- }
-
- public static ServerName create(final String serverName) {
- return new ServerName(serverName);
- }
-
- public static ServerName createWithDefaultPort(final String serverName,
- final int defaultPort) {
- if (serverName == null || serverName.length() <= 0) {
- throw new IllegalArgumentException("Passed hostname is null or empty ("
- + serverName + ")");
- }
- int index = serverName.indexOf(SERVERNAME_SEPARATOR);
- if (index == -1) {
- return new ServerName(parseHostname(serverName), defaultPort);
- } else {
- return new ServerName(parseHostname(serverName), parsePort(serverName));
- }
- }
-
- public static String parseHostname(final String serverName) {
- if (serverName == null || serverName.length() <= 0) {
- throw new IllegalArgumentException("Passed hostname is null or empty ("
- + serverName + ")");
- }
- int index = serverName.indexOf(SERVERNAME_SEPARATOR);
- if (index == -1) { // if a port is missing, the index will be set to -1.
- throw new IllegalArgumentException("Passed port is missing ("
- + serverName + ")");
- }
- return serverName.substring(0, index);
- }
-
- public static int parsePort(final String serverName) {
- String [] split = serverName.split(SERVERNAME_SEPARATOR);
- return Integer.parseInt(split[1]);
- }
-
- @Override
- public String toString() {
- return getServerName();
- }
-
- public String getServerName() {
- return serverName;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public int getPort() {
- return port;
- }
-
- public static String getServerName(String hostName, int port) {
- final StringBuilder name = new StringBuilder(hostName.length() + 4);
- name.append(hostName);
- name.append(SERVERNAME_SEPARATOR);
- name.append(port);
- return name.toString();
- }
-
- @Override
- public int compareTo(ServerName other) {
- int compare = this.getHostname().toLowerCase().
- compareTo(other.getHostname().toLowerCase());
- if (compare != 0) return compare;
- return this.getPort() - other.getPort();
- }
-
- @Override
- public int hashCode() {
- return getServerName().hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null) return false;
- if (!(o instanceof ServerName)) return false;
- return this.compareTo((ServerName)o) == 0;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/f3291621/tajo-core/src/test/java/org/apache/tajo/cluster/TestServerName.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cluster/TestServerName.java b/tajo-core/src/test/java/org/apache/tajo/cluster/TestServerName.java
deleted file mode 100644
index 513187d..0000000
--- a/tajo-core/src/test/java/org/apache/tajo/cluster/TestServerName.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.cluster;
-
-import org.junit.Test;
-import org.apache.tajo.master.cluster.ServerName;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TestServerName {
-
- @Test
- public void testServerNameStringInt() {
- ServerName server = new ServerName("ex1.com",50030);
- assertEquals("ex1.com", server.getHostname());
- assertEquals(50030, server.getPort());
- }
-
- @Test
- public void testServerNameString() {
- ServerName server = new ServerName("ex1.com:50030");
- assertEquals("ex1.com", server.getHostname());
- assertEquals(50030, server.getPort());
- }
-
- @Test
- public void testParseHostname() {
- assertEquals("ex1.com",ServerName.parseHostname("ex1.com:50030"));
- }
-
- @Test
- public void testParsePort() {
- assertEquals(50030,ServerName.parsePort("ex1.com:50030"));
- }
-
- @Test
- public void testToString() {
- ServerName server = new ServerName("ex1.com",50030);
- assertEquals("ex1.com:50030", server.toString());
- }
-
- @Test
- public void testGetServerName() {
- ServerName server = new ServerName("ex1.com",50030);
- assertEquals("ex1.com:50030", server.getServerName());
- }
-
- @Test
- public void testGetHostname() {
- ServerName server = new ServerName("ex1.com",50030);
- assertEquals("ex1.com", server.getHostname());
- }
-
- @Test
- public void testGetPort() {
- ServerName server = new ServerName("ex1.com",50030);
- assertEquals(50030, server.getPort());
- }
-
- @Test
- public void testGetServerNameStringInt() {
- assertEquals("ex2.com:50030",ServerName.getServerName("ex2.com", 50030));
- }
-
- @Test
- public void testCompareTo() {
- ServerName s1 = new ServerName("ex1.com:50030");
- ServerName s2 = new ServerName("ex1.com:60030");
-
- assertTrue(s1.compareTo(s2) < 0);
- assertTrue(s2.compareTo(s1) > 0);
-
- ServerName s3 = new ServerName("ex1.com:50030");
- assertTrue(s1.compareTo(s3) == 0);
-
- ServerName s4 = new ServerName("ex2.com:50030");
- assertTrue(s1.compareTo(s4) < 0);
- assertTrue(s4.compareTo(s1) > 0);
- }
-
- @Test (expected = IllegalArgumentException.class)
- public void testException() {
- new ServerName("ex1.com");
- }
-}
[07/14] tajo git commit: TAJO-1209: Pluggable line (de)serializer for
DelimitedTextFile.
Posted by ji...@apache.org.
TAJO-1209: Pluggable line (de)serializer for DelimitedTextFile.
Closes #1209
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/72dd29c5
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/72dd29c5
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/72dd29c5
Branch: refs/heads/index_support
Commit: 72dd29c520981a3ffaac2150ee7306ca41192893
Parents: 3ae44b1
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Nov 27 19:46:58 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Nov 27 19:46:58 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../apache/tajo/storage/StorageConstants.java | 3 +-
.../org/apache/tajo/util/ReflectionUtil.java | 4 +-
.../tajo/storage/text/CSVLineDeserializer.java | 96 +++++++++++
.../apache/tajo/storage/text/CSVLineSerDe.java | 45 +++++
.../tajo/storage/text/CSVLineSerializer.java | 68 ++++++++
.../tajo/storage/text/DelimitedTextFile.java | 163 +++++++------------
.../tajo/storage/text/TextLineDeserializer.java | 60 +++++++
.../apache/tajo/storage/text/TextLineSerDe.java | 65 ++++++++
.../tajo/storage/text/TextLineSerializer.java | 45 +++++
10 files changed, 449 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index f2141a6..6f38f65 100644
--- a/CHANGES
+++ b/CHANGES
@@ -67,6 +67,9 @@ Release 0.9.1 - unreleased
TAJO-1188: Fix testcase testTimestampConstructor in TestTimestampDatum.
(DaeMyung Kang via hyunsik)
+ TAJO-1209: Pluggable line (de)serializer for DelimitedTextFile.
+ (hyunsik)
+
BUG FIXES
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
index 11ac9b7..3065d31 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
@@ -32,7 +32,8 @@ public class StorageConstants {
public static final String TEXT_DELIMITER = "text.delimiter";
public static final String TEXT_NULL = "text.null";
- public static final String TEXTFILE_SERDE = "textfile.serde";
+ public static final String TEXT_SERDE_CLASS = "text.serde.class";
+ public static final String DEFAULT_TEXT_SERDE_CLASS = "org.apache.tajo.storage.text.CSVLineSerde";
@Deprecated
public static final String SEQUENCEFILE_DELIMITER = "sequencefile.delimiter";
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java
index 410815f..eccc61f 100644
--- a/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java
+++ b/tajo-common/src/main/java/org/apache/tajo/util/ReflectionUtil.java
@@ -32,8 +32,8 @@ public class ReflectionUtil {
private static final Map<Class<?>, Constructor<?>> CONSTRUCTOR_CACHE =
new ConcurrentHashMap<Class<?>, Constructor<?>>();
- public static Object newInstance(Class<?> clazz)
- throws InstantiationException, IllegalAccessException {
+ public static Object newInstance(Class<?> clazz)
+ throws InstantiationException, IllegalAccessException {
return clazz.newInstance();
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
new file mode 100644
index 0000000..f580da1
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.text;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.storage.FieldSerializerDeserializer;
+import org.apache.tajo.storage.Tuple;
+
+import java.io.IOException;
+
+public class CSVLineDeserializer extends TextLineDeserializer {
+ private FieldSplitProcessor processor;
+ private FieldSerializerDeserializer fieldSerDer;
+ private ByteBuf nullChars;
+
+ public CSVLineDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
+ super(schema, meta, targetColumnIndexes);
+ }
+
+ @Override
+ public void init() {
+ this.processor = new FieldSplitProcessor(CSVLineSerDe.getFieldDelimiter(meta));
+
+ if (nullChars != null) {
+ nullChars.release();
+ }
+ nullChars = TextLineSerDe.getNullChars(meta);
+
+ fieldSerDer = new TextFieldSerializerDeserializer();
+ }
+
+ public void deserialize(final ByteBuf lineBuf, Tuple tuple) throws IOException {
+ int[] projection = targetColumnIndexes;
+ if (lineBuf == null || targetColumnIndexes == null || targetColumnIndexes.length == 0) {
+ return;
+ }
+
+ final int rowLength = lineBuf.readableBytes();
+ int start = 0, fieldLength = 0, end = 0;
+
+ //Projection
+ int currentTarget = 0;
+ int currentIndex = 0;
+
+ while (end != -1) {
+ end = lineBuf.forEachByte(start, rowLength - start, processor);
+
+ if (end < 0) {
+ fieldLength = rowLength - start;
+ } else {
+ fieldLength = end - start;
+ }
+
+ if (projection.length > currentTarget && currentIndex == projection[currentTarget]) {
+ lineBuf.setIndex(start, start + fieldLength);
+ Datum datum = fieldSerDer.deserialize(lineBuf, schema.getColumn(currentIndex), currentIndex, nullChars);
+ tuple.put(currentIndex, datum);
+ currentTarget++;
+ }
+
+ if (projection.length == currentTarget) {
+ break;
+ }
+
+ start = end + 1;
+ currentIndex++;
+ }
+ }
+
+ @Override
+ public void release() {
+ if (nullChars != null) {
+ nullChars.release();
+ nullChars = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
new file mode 100644
index 0000000..e2686a6
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.text;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.storage.StorageConstants;
+
+public class CSVLineSerDe extends TextLineSerDe {
+
+ public CSVLineSerDe() {
+ }
+
+ @Override
+ public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
+ return new CSVLineDeserializer(schema, meta, targetColumnIndexes);
+ }
+
+ @Override
+ public TextLineSerializer createSerializer(Schema schema, TableMeta meta) {
+ return new CSVLineSerializer(schema, meta);
+ }
+
+ public static char getFieldDelimiter(TableMeta meta) {
+ return StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.TEXT_DELIMITER,
+ StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
new file mode 100644
index 0000000..684519c
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.text;
+
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.storage.FieldSerializerDeserializer;
+import org.apache.tajo.storage.Tuple;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class CSVLineSerializer extends TextLineSerializer {
+ private FieldSerializerDeserializer serde;
+
+ private byte [] nullChars;
+ private char delimiter;
+
+ public CSVLineSerializer(Schema schema, TableMeta meta) {
+ super(schema, meta);
+ }
+
+ @Override
+ public void init() {
+ nullChars = TextLineSerDe.getNullCharsAsBytes(meta);
+ delimiter = CSVLineSerDe.getFieldDelimiter(meta);
+
+ serde = new TextFieldSerializerDeserializer();
+ }
+
+ @Override
+ public int serialize(OutputStream out, Tuple input) throws IOException {
+ int rowBytes = 0;
+
+ for (int i = 0; i < schema.size(); i++) {
+ Datum datum = input.get(i);
+ rowBytes += serde.serialize(out, datum, schema.getColumn(i), i, nullChars);
+
+ if (schema.size() - 1 > i) {
+ out.write((byte) delimiter);
+ rowBytes += 1;
+ }
+ }
+
+ return rowBytes;
+ }
+
+ @Override
+ public void release() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
index 68d89e7..d15f394 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
@@ -19,8 +19,6 @@
package org.apache.tajo.storage.text;
import io.netty.buffer.ByteBuf;
-import org.apache.commons.lang.StringEscapeUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -35,19 +33,20 @@ import org.apache.hadoop.io.compress.Compressor;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
-import org.apache.tajo.datum.Datum;
-import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.*;
import org.apache.tajo.storage.compress.CodecPool;
import org.apache.tajo.storage.exception.AlreadyExistsStorageException;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.rcfile.NonSyncByteArrayOutputStream;
+import org.apache.tajo.util.ReflectionUtil;
import java.io.BufferedOutputStream;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
public class DelimitedTextFile {
@@ -56,15 +55,48 @@ public class DelimitedTextFile {
private static final Log LOG = LogFactory.getLog(DelimitedTextFile.class);
+ /** it caches line serde classes. */
+ private static final Map<String, Class<? extends TextLineSerDe>> serdeClassCache =
+ new ConcurrentHashMap<String, Class<? extends TextLineSerDe>>();
+
+ /**
+ * By default, DelimitedTextFileScanner uses CSVLineSerder. If a table property 'text.serde.class' is given,
+ * it will use the specified serder class.
+ *
+ * @return TextLineSerder
+ */
+ public static TextLineSerDe getLineSerde(TableMeta meta) {
+ TextLineSerDe lineSerder;
+
+ String serDeClassName;
+
+ // if there is no given serde class, it will use CSV line serder.
+ serDeClassName = meta.getOption(StorageConstants.TEXT_SERDE_CLASS, StorageConstants.DEFAULT_TEXT_SERDE_CLASS);
+
+ try {
+ Class<? extends TextLineSerDe> serdeClass;
+
+ if (serdeClassCache.containsKey(serDeClassName)) {
+ serdeClass = serdeClassCache.get(serDeClassName);
+ } else {
+ serdeClass = (Class<? extends TextLineSerDe>) Class.forName(CSVLineSerDe.class.getName());
+ serdeClassCache.put(serDeClassName, serdeClass);
+ }
+ lineSerder = (TextLineSerDe) ReflectionUtil.newInstance(serdeClass);
+ } catch (Throwable e) {
+ throw new RuntimeException("TextLineSerde class cannot be initialized");
+ }
+
+ return lineSerder;
+ }
+
public static class DelimitedTextFileAppender extends FileAppender {
private final TableMeta meta;
private final Schema schema;
- private final int columnNum;
private final FileSystem fs;
private FSDataOutputStream fos;
private DataOutputStream outputStream;
private CompressionOutputStream deflateFilter;
- private char delimiter;
private TableStatistics stats = null;
private Compressor compressor;
private CompressionCodecFactory codecFactory;
@@ -76,7 +108,7 @@ public class DelimitedTextFile {
private long pos = 0;
private NonSyncByteArrayOutputStream os;
- private FieldSerializerDeserializer serde;
+ private TextLineSerializer serializer;
public DelimitedTextFileAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path)
throws IOException {
@@ -84,17 +116,10 @@ public class DelimitedTextFile {
this.fs = path.getFileSystem(conf);
this.meta = meta;
this.schema = schema;
- this.delimiter = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.TEXT_DELIMITER,
- StorageConstants.DEFAULT_FIELD_DELIMITER)).charAt(0);
- this.columnNum = schema.size();
-
- String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(StorageConstants.TEXT_NULL,
- NullDatum.DEFAULT_TEXT));
- if (StringUtils.isEmpty(nullCharacters)) {
- nullChars = NullDatum.get().asTextBytes();
- } else {
- nullChars = nullCharacters.getBytes();
- }
+ }
+
+ public TextLineSerDe getLineSerde() {
+ return DelimitedTextFile.getLineSerde(meta);
}
@Override
@@ -133,7 +158,8 @@ public class DelimitedTextFile {
this.stats = new TableStatistics(this.schema);
}
- serde = new TextFieldSerializerDeserializer();
+ serializer = getLineSerde().createSerializer(schema, meta);
+ serializer.init();
if (os == null) {
os = new NonSyncByteArrayOutputStream(BUFFER_SIZE);
@@ -145,26 +171,20 @@ public class DelimitedTextFile {
super.init();
}
-
@Override
public void addTuple(Tuple tuple) throws IOException {
- Datum datum;
- int rowBytes = 0;
+ // write
+ int rowBytes = serializer.serialize(os, tuple);
- for (int i = 0; i < columnNum; i++) {
- datum = tuple.get(i);
- rowBytes += serde.serialize(os, datum, schema.getColumn(i), i, nullChars);
-
- if (columnNum - 1 > i) {
- os.write((byte) delimiter);
- rowBytes += 1;
- }
- }
+ // new line
os.write(LF);
rowBytes += 1;
+ // update positions
pos += rowBytes;
bufferedBytes += rowBytes;
+
+ // refill buffer if necessary
if (bufferedBytes > BUFFER_SIZE) {
flushBuffer();
}
@@ -197,6 +217,8 @@ public class DelimitedTextFile {
public void close() throws IOException {
try {
+ serializer.release();
+
if(outputStream != null){
flush();
}
@@ -241,18 +263,15 @@ public class DelimitedTextFile {
}
public static class DelimitedTextFileScanner extends FileScanner {
-
private boolean splittable = false;
private final long startOffset;
- private final long endOffset;
+ private final long endOffset;
private int recordCount = 0;
private int[] targetColumnIndexes;
- private ByteBuf nullChars;
- private FieldSerializerDeserializer serde;
private DelimitedLineReader reader;
- private FieldSplitProcessor processor;
+ private TextLineDeserializer deserializer;
public DelimitedTextFileScanner(Configuration conf, final Schema schema, final TableMeta meta,
final FileFragment fragment)
@@ -265,30 +284,14 @@ public class DelimitedTextFile {
startOffset = fragment.getStartKey();
endOffset = startOffset + fragment.getEndKey();
+ }
- //Delimiter
- String delim = meta.getOption(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
- this.processor = new FieldSplitProcessor(StringEscapeUtils.unescapeJava(delim).charAt(0));
+ public TextLineSerDe getLineSerde() {
+ return DelimitedTextFile.getLineSerde(meta);
}
@Override
public void init() throws IOException {
- if (nullChars != null) {
- nullChars.release();
- }
-
- String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.TEXT_NULL,
- NullDatum.DEFAULT_TEXT));
- byte[] bytes;
- if (StringUtils.isEmpty(nullCharacters)) {
- bytes = NullDatum.get().asTextBytes();
- } else {
- bytes = nullCharacters.getBytes();
- }
-
- nullChars = BufferPool.directBuffer(bytes.length, bytes.length);
- nullChars.writeBytes(bytes);
-
if (reader != null) {
reader.close();
}
@@ -305,8 +308,6 @@ public class DelimitedTextFile {
targetColumnIndexes[i] = schema.getColumnId(targets[i].getQualifiedName());
}
- serde = new TextFieldSerializerDeserializer();
-
super.init();
Arrays.sort(targetColumnIndexes);
if (LOG.isDebugEnabled()) {
@@ -316,6 +317,9 @@ public class DelimitedTextFile {
if (startOffset > 0) {
reader.readLine(); // skip first line;
}
+
+ deserializer = getLineSerde().createDeserializer(schema, meta, targetColumnIndexes);
+ deserializer.init();
}
public ByteBuf readLine() throws IOException {
@@ -362,7 +366,7 @@ public class DelimitedTextFile {
}
VTuple tuple = new VTuple(schema.size());
- fillTuple(schema, tuple, buf, targetColumnIndexes);
+ deserializer.deserialize(buf, tuple);
return tuple;
} catch (Throwable t) {
LOG.error("Tuple list current index: " + recordCount + " file offset:" + reader.getCompressedPosition(), t);
@@ -370,44 +374,6 @@ public class DelimitedTextFile {
}
}
- private void fillTuple(Schema schema, Tuple dst, ByteBuf lineBuf, int[] target) throws IOException {
- int[] projection = target;
- if (lineBuf == null || target == null || target.length == 0) {
- return;
- }
-
- final int rowLength = lineBuf.readableBytes();
- int start = 0, fieldLength = 0, end = 0;
-
- //Projection
- int currentTarget = 0;
- int currentIndex = 0;
-
- while (end != -1) {
- end = lineBuf.forEachByte(start, rowLength - start, processor);
-
- if (end < 0) {
- fieldLength = rowLength - start;
- } else {
- fieldLength = end - start;
- }
-
- if (projection.length > currentTarget && currentIndex == projection[currentTarget]) {
- lineBuf.setIndex(start, start + fieldLength);
- Datum datum = serde.deserialize(lineBuf, schema.getColumn(currentIndex), currentIndex, nullChars);
- dst.put(currentIndex, datum);
- currentTarget++;
- }
-
- if (projection.length == currentTarget) {
- break;
- }
-
- start = end + 1;
- currentIndex++;
- }
- }
-
@Override
public void reset() throws IOException {
init();
@@ -416,10 +382,7 @@ public class DelimitedTextFile {
@Override
public void close() throws IOException {
try {
- if (nullChars != null) {
- nullChars.release();
- nullChars = null;
- }
+ deserializer.release();
if (tableStats != null && reader != null) {
tableStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead)
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
new file mode 100644
index 0000000..645d118
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.text;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.storage.Tuple;
+
+import java.io.IOException;
+
+/**
+ * Reads a text line and fills a Tuple with values
+ */
+public abstract class TextLineDeserializer {
+ protected Schema schema;
+ protected TableMeta meta;
+ protected int [] targetColumnIndexes;
+
+ public TextLineDeserializer(Schema schema, TableMeta meta, int [] targetColumnIndexes) {
+ this.schema = schema;
+ this.meta = meta;
+ this.targetColumnIndexes = targetColumnIndexes;
+ }
+
+ /**
+ * Initialize SerDe
+ */
+ public abstract void init();
+
+ /**
+ * It fills a tuple with a read fields in a given line.
+ *
+ * @param buf Read line
+ * @param tuple Tuple to be filled with read fields
+ * @throws java.io.IOException
+ */
+ public abstract void deserialize(final ByteBuf buf, Tuple tuple) throws IOException;
+
+ /**
+ * Release external resources
+ */
+ public abstract void release();
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
new file mode 100644
index 0000000..e81e289
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerDe.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.text;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.storage.BufferPool;
+import org.apache.tajo.storage.StorageConstants;
+
+/**
+ * Pluggable Text Line SerDe class
+ */
+public abstract class TextLineSerDe {
+
+ public TextLineSerDe() {
+ }
+
+ public abstract TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int [] targetColumnIndexes);
+
+ public abstract TextLineSerializer createSerializer(Schema schema, TableMeta meta);
+
+ public static ByteBuf getNullChars(TableMeta meta) {
+ byte[] nullCharByteArray = getNullCharsAsBytes(meta);
+
+ ByteBuf nullChars = BufferPool.directBuffer(nullCharByteArray.length, nullCharByteArray.length);
+ nullChars.writeBytes(nullCharByteArray);
+
+ return nullChars;
+ }
+
+ public static byte [] getNullCharsAsBytes(TableMeta meta) {
+ byte [] nullChars;
+
+ String nullCharacters = StringEscapeUtils.unescapeJava(meta.getOption(StorageConstants.TEXT_NULL,
+ NullDatum.DEFAULT_TEXT));
+ if (StringUtils.isEmpty(nullCharacters)) {
+ nullChars = NullDatum.get().asTextBytes();
+ } else {
+ nullChars = nullCharacters.getBytes();
+ }
+
+ return nullChars;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/72dd29c5/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerializer.java
new file mode 100644
index 0000000..0c2761f
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineSerializer.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.text;
+
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.storage.Tuple;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Write a Tuple into single text formatted line
+ */
+public abstract class TextLineSerializer {
+ protected Schema schema;
+ protected TableMeta meta;
+
+ public TextLineSerializer(Schema schema, TableMeta meta) {
+ this.schema = schema;
+ this.meta = meta;
+ }
+
+ public abstract void init();
+
+ public abstract int serialize(OutputStream out, Tuple input) throws IOException;
+
+ public abstract void release();
+}
[09/14] tajo git commit: TAJO-1165: Needs to show error messages on
query_executor.jsp. (Jihun Kang via jaehwa)
Posted by ji...@apache.org.
TAJO-1165: Needs to show error messages on query_executor.jsp. (Jihun Kang via jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/4637f7fd
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/4637f7fd
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/4637f7fd
Branch: refs/heads/index_support
Commit: 4637f7fd126c46f0e6a4bcdf2d816511b690552b
Parents: f329162
Author: JaeHwa Jung <bl...@apache.org>
Authored: Thu Nov 27 22:34:53 2014 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Thu Nov 27 22:34:53 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
.../apache/tajo/webapp/QueryExecutorServlet.java | 16 ++++++++++++++++
.../main/resources/webapps/admin/query_executor.jsp | 4 ++--
3 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/4637f7fd/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 9b15898..8bf0c1c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -13,6 +13,8 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1165: Needs to show error messages on query_executor.jsp. (Jihun Kang via jaehwa)
+
TAJO-1204: Remove unused ServerName class. (DaeMyung Kang via jaehwa)
TAJO-1053: ADD PARTITIONS for HCatalogStore. (jaehwa)
http://git-wip-us.apache.org/repos/asf/tajo/blob/4637f7fd/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
index 0075b04..da25fe6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
+++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
@@ -316,6 +316,22 @@ public class QueryExecutorServlet extends HttpServlet {
progress.set(100);
}
+ } else if (response.getResultCode() == ClientProtos.ResultCode.ERROR) {
+ if (response.hasErrorMessage()) {
+ StringBuffer errorMessage = new StringBuffer(response.getErrorMessage());
+ String modifiedMessage;
+
+ if (errorMessage.length() > 200) {
+ modifiedMessage = errorMessage.substring(0, 200);
+ } else {
+ modifiedMessage = errorMessage.toString();
+ }
+
+ String lineSeparator = System.getProperty("line.separator");
+ modifiedMessage = modifiedMessage.replaceAll(lineSeparator, "<br/>");
+
+ error = new Exception(modifiedMessage);
+ }
}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/tajo/blob/4637f7fd/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
index bbd1820..9ff6625 100644
--- a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
@@ -102,7 +102,7 @@ function runQuery() {
var resultJson = $.parseJSON(msg);
if(resultJson.success == "false") {
clearTimer();
- alert(resultJson.errorMessage);
+ alert("query execution failed.");
return;
}
queryRunnerId = resultJson.queryRunnerId;
@@ -116,7 +116,7 @@ function runQuery() {
var resultJson = $.parseJSON(msg);
if(resultJson.success == "false") {
clearTimer();
- alert(resultJson.errorMessage);
+ alert("query execution failed.");
$("#queryStatus").html(getQueryStatusHtml(resultJson));
return;
}
[14/14] tajo git commit: Merge branch 'master' of
http://git-wip-us.apache.org/repos/asf/tajo into index_support
Posted by ji...@apache.org.
Merge branch 'master' of http://git-wip-us.apache.org/repos/asf/tajo into index_support
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/708453cd
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/708453cd
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/708453cd
Branch: refs/heads/index_support
Commit: 708453cd8d65880ad081e75c511187d9354d4415
Parents: 1e9c38d b4adc18
Author: Jihoon Son <ji...@apache.org>
Authored: Tue Dec 2 12:56:06 2014 +0900
Committer: Jihoon Son <ji...@apache.org>
Committed: Tue Dec 2 12:56:06 2014 +0900
----------------------------------------------------------------------
CHANGES | 26 ++
.../org/apache/tajo/catalog/CatalogUtil.java | 11 +-
.../org/apache/tajo/catalog/SchemaUtil.java | 8 +
.../java/org/apache/tajo/catalog/TableDesc.java | 21 +-
.../src/main/proto/CatalogProtos.proto | 1 +
.../org/apache/tajo/catalog/TestTableDesc.java | 10 +-
.../tajo/catalog/store/HCatalogStore.java | 11 +-
.../tajo/catalog/store/TestHCatalogStore.java | 22 +-
.../org/apache/tajo/catalog/TestCatalog.java | 16 +-
.../org/apache/tajo/client/ResultSetUtil.java | 8 +
.../org/apache/tajo/jdbc/TajoResultSet.java | 4 +-
.../java/org/apache/tajo/conf/TajoConf.java | 12 +-
.../apache/tajo/storage/StorageConstants.java | 3 +-
.../java/org/apache/tajo/storage/VTuple.java | 4 +-
.../org/apache/tajo/util/ReflectionUtil.java | 4 +-
.../apache/tajo/datum/TestTimestampDatum.java | 3 +-
tajo-core/pom.xml | 1 +
.../engine/planner/global/GlobalPlanner.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 7 +-
.../org/apache/tajo/master/ContainerProxy.java | 10 +-
.../tajo/master/DefaultTaskScheduler.java | 18 +-
.../org/apache/tajo/master/GlobalEngine.java | 17 +-
.../tajo/master/LaunchTaskRunnersEvent.java | 5 +-
.../apache/tajo/master/LazyTaskScheduler.java | 9 +-
.../apache/tajo/master/TajoContainerProxy.java | 22 +-
.../java/org/apache/tajo/master/TajoMaster.java | 2 +-
.../apache/tajo/master/TajoMasterService.java | 5 +-
.../tajo/master/TaskRunnerGroupEvent.java | 8 +-
.../apache/tajo/master/cluster/ServerName.java | 123 --------
.../tajo/master/container/TajoContainer.java | 173 ++++++++++
.../tajo/master/container/TajoContainerId.java | 172 ++++++++++
.../master/container/TajoConverterUtils.java | 263 ++++++++++++++++
.../master/container/TajoRecordFactory.java | 31 ++
.../container/TajoRecordFactoryPBImpl.java | 104 ++++++
.../container/TajoRecordFactoryProvider.java | 70 +++++
.../tajo/master/container/TajoRecords.java | 39 +++
.../impl/pb/TajoContainerIdPBImpl.java | 100 ++++++
.../tajo/master/event/LocalTaskEvent.java | 9 +-
.../event/QueryUnitAttemptScheduleEvent.java | 10 +-
.../event/SubQueryContainerAllocationEvent.java | 8 +-
.../master/event/TaskAttemptAssignedEvent.java | 8 +-
.../tajo/master/event/TaskRequestEvent.java | 8 +-
.../apache/tajo/master/querymaster/Query.java | 80 +++--
.../master/querymaster/QueryInProgress.java | 3 +-
.../querymaster/QueryMasterManagerService.java | 4 +-
.../master/querymaster/QueryMasterTask.java | 23 +-
.../master/querymaster/QueryUnitAttempt.java | 6 +-
.../tajo/master/querymaster/Repartitioner.java | 6 +-
.../tajo/master/querymaster/SubQuery.java | 19 +-
.../apache/tajo/master/rm/TajoRMContext.java | 8 +-
.../tajo/master/rm/TajoWorkerContainer.java | 15 +-
.../tajo/master/rm/TajoWorkerContainerId.java | 53 ++--
.../master/rm/TajoWorkerResourceManager.java | 63 ++--
.../tajo/master/rm/WorkerResourceManager.java | 4 +-
.../tajo/webapp/QueryExecutorServlet.java | 16 +
.../tajo/worker/AbstractResourceAllocator.java | 14 +-
.../apache/tajo/worker/ResourceAllocator.java | 6 +-
.../tajo/worker/TajoResourceAllocator.java | 101 +++---
.../java/org/apache/tajo/worker/TaskRunner.java | 16 +-
.../apache/tajo/worker/TaskRunnerHistory.java | 14 +-
.../src/main/proto/ContainerProtocol.proto | 48 +++
.../src/main/proto/QueryMasterProtocol.proto | 3 +
.../main/proto/ResourceTrackerProtocol.proto | 3 +
.../src/main/proto/TajoMasterProtocol.proto | 7 +-
.../src/main/proto/TajoWorkerProtocol.proto | 5 +-
.../src/main/resources/webapps/admin/index.jsp | 2 +-
.../resources/webapps/admin/query_executor.jsp | 4 +-
.../apache/tajo/LocalTajoTestingUtility.java | 2 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 4 +-
.../apache/tajo/cli/tools/TestDDLBuilder.java | 8 +-
.../org/apache/tajo/client/TestTajoClient.java | 18 +-
.../org/apache/tajo/cluster/TestServerName.java | 102 ------
.../apache/tajo/engine/eval/ExprTestBase.java | 2 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 2 +-
.../engine/planner/TestLogicalOptimizer.java | 6 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 6 +-
.../tajo/engine/planner/TestPlannerUtil.java | 8 +-
.../planner/physical/TestBNLJoinExec.java | 14 +-
.../planner/physical/TestBSTIndexExec.java | 2 +-
.../planner/physical/TestExternalSortExec.java | 6 +-
.../physical/TestFullOuterHashJoinExec.java | 23 +-
.../physical/TestFullOuterMergeJoinExec.java | 36 ++-
.../planner/physical/TestHashAntiJoinExec.java | 8 +-
.../planner/physical/TestHashJoinExec.java | 14 +-
.../planner/physical/TestHashSemiJoinExec.java | 18 +-
.../physical/TestLeftOuterHashJoinExec.java | 38 +--
.../physical/TestLeftOuterNLJoinExec.java | 20 +-
.../planner/physical/TestMergeJoinExec.java | 6 +-
.../engine/planner/physical/TestNLJoinExec.java | 16 +-
.../planner/physical/TestPhysicalPlanner.java | 65 ++--
.../physical/TestProgressExternalSortExec.java | 6 +-
.../physical/TestRightOuterHashJoinExec.java | 23 +-
.../physical/TestRightOuterMergeJoinExec.java | 26 +-
.../engine/planner/physical/TestSortExec.java | 2 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 9 +-
.../tajo/engine/query/TestCreateTable.java | 9 +-
.../tajo/engine/query/TestInsertQuery.java | 20 +-
.../tajo/engine/query/TestJoinBroadcast.java | 9 +-
.../tajo/engine/query/TestTablePartitions.java | 26 +-
.../org/apache/tajo/jdbc/TestResultSet.java | 2 +-
.../java/org/apache/tajo/jdbc/TestTajoJdbc.java | 181 ++++++++---
.../tajo/master/rm/TestTajoResourceManager.java | 16 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 4 +-
.../dataset/TestTajoJdbc/table1/table1.tbl | 5 +
.../TestTajoJdbc/create_table_with_date_ddl.sql | 10 +
.../TestTajoJdbc/testSortWithDateTime.result | 7 +
.../src/main/sphinx/hcatalog_integration.rst | 13 +
.../org/apache/tajo/jdbc/MetaDataTuple.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 9 +-
.../apache/tajo/plan/logical/InsertNode.java | 2 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
tajo-storage/pom.xml | 7 +-
.../org/apache/tajo/storage/StorageManager.java | 1 +
.../tajo/storage/json/JsonLineDeserializer.java | 220 +++++++++++++
.../apache/tajo/storage/json/JsonLineSerDe.java | 37 +++
.../tajo/storage/json/JsonLineSerializer.java | 134 ++++++++
.../tajo/storage/text/ByteBufLineReader.java | 22 +-
.../tajo/storage/text/CSVLineDeserializer.java | 96 ++++++
.../apache/tajo/storage/text/CSVLineSerDe.java | 41 +++
.../tajo/storage/text/CSVLineSerializer.java | 71 +++++
.../tajo/storage/text/DelimitedTextFile.java | 163 ++++------
.../text/TextFieldSerializerDeserializer.java | 2 +-
.../tajo/storage/text/TextLineDeserializer.java | 60 ++++
.../apache/tajo/storage/text/TextLineSerDe.java | 65 ++++
.../tajo/storage/text/TextLineSerializer.java | 45 +++
.../src/main/resources/storage-default.xml | 16 +-
.../apache/tajo/storage/TestFileSystems.java | 267 ++++++++--------
.../org/apache/tajo/storage/TestLineReader.java | 32 +-
.../org/apache/tajo/storage/TestStorages.java | 107 ++++---
.../apache/tajo/storage/avro/TestAvroUtil.java | 2 +-
.../apache/tajo/storage/json/TestJsonSerDe.java | 101 ++++++
.../java/org/apache/tajo/storage/s3/INode.java | 124 --------
.../storage/s3/InMemoryFileSystemStore.java | 175 -----------
.../apache/tajo/storage/s3/S3OutputStream.java | 234 --------------
.../tajo/storage/s3/SmallBlockS3FileSystem.java | 314 -------------------
.../dataset/TestJsonSerDe/testVariousType.json | 1 +
.../src/test/resources/dataset/testLineText.txt | 2 +
.../resources/dataset/testVariousTypes.avsc | 20 ++
.../src/test/resources/storage-default.xml | 16 +-
.../src/test/resources/testVariousTypes.avsc | 21 --
140 files changed, 3157 insertions(+), 1961 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
----------------------------------------------------------------------
diff --cc tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
index 9aaaf3d,d507b97..ae208a8
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java
@@@ -977,19 -980,14 +980,19 @@@ public class TestPhysicalPlanner
}
public final String [] createIndexStmt = {
- "create index idx_employee on employee using bst (name null first, empId desc)"
+ "create index idx_employee on employee using TWO_LEVEL_BIN_TREE (name null first, empId desc)"
};
- //@Test
+ @Test
public final void testCreateIndex() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testCreateIndex");
+ Path indexPath = StorageUtil.concatPath(TajoConf.getWarehouseDir(conf), "default/idx_employee");
+ if (sm.getFileSystem().exists(indexPath)) {
+ sm.getFileSystem().delete(indexPath, true);
+ }
+
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(masterPlan),
new FileFragment[] {frags[0]}, workDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/708453cd/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
[12/14] tajo git commit: TAJO-1210: ByteBufLineReader does not handle
the end of file, if newline is not appeared. (jinho)
Posted by ji...@apache.org.
TAJO-1210: ByteBufLineReader does not handle the end of file, if newline is not appeared. (jinho)
Closes #272
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/bf68b770
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/bf68b770
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/bf68b770
Branch: refs/heads/index_support
Commit: bf68b770e6abbb4c63d696e264a348bb1ddb5982
Parents: cd38dff
Author: jhkim <jh...@apache.org>
Authored: Mon Dec 1 11:31:23 2014 +0900
Committer: jhkim <jh...@apache.org>
Committed: Mon Dec 1 11:31:23 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
tajo-storage/pom.xml | 3 +-
.../tajo/storage/text/ByteBufLineReader.java | 22 ++++++++++++--
.../org/apache/tajo/storage/TestLineReader.java | 32 +++++++++++++++++++-
.../org/apache/tajo/storage/TestStorages.java | 2 +-
.../apache/tajo/storage/avro/TestAvroUtil.java | 2 +-
.../src/test/resources/dataset/testLineText.txt | 2 ++
.../resources/dataset/testVariousTypes.avsc | 20 ++++++++++++
.../src/test/resources/testVariousTypes.avsc | 20 ------------
9 files changed, 78 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 025ae88..33983bc 100644
--- a/CHANGES
+++ b/CHANGES
@@ -79,6 +79,9 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1210: ByteBufLineReader does not handle the end of file,
+ if newline is not appeared. (jinho)
+
TAJO-1119: JDBC driver should support TIMESTAMP type. (jaehwa)
TAJO-1166: S3 related storage causes compilation error in Hadoop 2.6.0-SNAPSHOT. (jaehwa)
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml
index ef26a32..7ede2e1 100644
--- a/tajo-storage/pom.xml
+++ b/tajo-storage/pom.xml
@@ -71,8 +71,7 @@
</executions>
<configuration>
<excludes>
- <exclude>src/test/resources/testVariousTypes.avsc</exclude>
- <exclude>src/test/resources/dataset/TestJsonSerDe/*.json</exclude>
+ <exclude>src/test/resources/dataset/**</exclude>
</excludes>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java
index 1448885..86319e1 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/ByteBufLineReader.java
@@ -32,6 +32,7 @@ public class ByteBufLineReader implements Closeable {
private int bufferSize;
private long readBytes;
+ private boolean eof = false;
private ByteBuf buffer;
private final ByteBufInputChannel channel;
private final AtomicInteger tempReadBytes = new AtomicInteger();
@@ -92,6 +93,10 @@ public class ByteBufLineReader implements Closeable {
for (; ; ) {
int localReadBytes = buffer.writeBytes(channel, bufferSize - readBytes);
if (localReadBytes < 0) {
+ if (tailBytes == readBytes) {
+ // no more bytes are in the channel
+ eof = true;
+ }
break;
}
readBytes += localReadBytes;
@@ -101,7 +106,9 @@ public class ByteBufLineReader implements Closeable {
}
this.readBytes += (readBytes - tailBytes);
release = false;
- this.buffer.readerIndex(this.buffer.readerIndex() + tailBytes); //skip past buffer (tail)
+ if (!eof) {
+ this.buffer.readerIndex(this.buffer.readerIndex() + tailBytes); //skip past buffer (tail)
+ }
} finally {
if (release) {
buffer.release();
@@ -113,6 +120,8 @@ public class ByteBufLineReader implements Closeable {
* Read a line terminated by one of CR, LF, or CRLF.
*/
public ByteBuf readLineBuf(AtomicInteger reads) throws IOException {
+ if(eof) return null;
+
int startIndex = buffer.readerIndex();
int readBytes;
int readable;
@@ -127,14 +136,21 @@ public class ByteBufLineReader implements Closeable {
if (!buffer.isReadable()) {
return null;
} else {
- startIndex = 0; // reset the line start position
+ if (!eof) startIndex = 0; // reset the line start position
+ else startIndex = buffer.readerIndex();
}
readable = buffer.readableBytes();
}
int endIndex = buffer.forEachByte(buffer.readerIndex(), readable, processor);
if (endIndex < 0) {
- buffer.readerIndex(buffer.writerIndex());
+ //does not appeared terminating newline
+ buffer.readerIndex(buffer.writerIndex()); // set to end buffer
+ if(eof){
+ readBytes = buffer.readerIndex() - startIndex;
+ newlineLength = 0;
+ break loop;
+ }
} else {
buffer.readerIndex(endIndex + 1);
readBytes = buffer.readerIndex() - startIndex;
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/src/test/java/org/apache/tajo/storage/TestLineReader.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestLineReader.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestLineReader.java
index ef6efdf..4512d00 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestLineReader.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestLineReader.java
@@ -34,11 +34,14 @@ import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.datum.NullDatum;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.text.ByteBufLineReader;
-import org.apache.tajo.storage.text.DelimitedTextFile;
import org.apache.tajo.storage.text.DelimitedLineReader;
+import org.apache.tajo.storage.text.DelimitedTextFile;
import org.apache.tajo.util.CommonTestingUtil;
+import org.apache.tajo.util.FileUtil;
import org.junit.Test;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -160,4 +163,31 @@ public class TestLineReader {
assertEquals(tupleNum, i);
}
+
+ @Test
+ public void testByteBufLineReaderWithoutTerminating() throws IOException {
+ String path = FileUtil.getResourcePath("dataset/testLineText.txt").getFile();
+ File file = new File(path);
+ String data = FileUtil.readTextFile(file);
+
+ ByteBufInputChannel channel = new ByteBufInputChannel(new FileInputStream(file));
+
+ assertEquals(file.length(), channel.available());
+ ByteBufLineReader reader = new ByteBufLineReader(channel);
+ assertEquals(file.length(), reader.available());
+
+ long totalRead = 0;
+ int i = 0;
+ AtomicInteger bytes = new AtomicInteger();
+ for(;;){
+ ByteBuf buf = reader.readLineBuf(bytes);
+ if(buf == null) break;
+ totalRead += bytes.get();
+ i++;
+ }
+ IOUtils.cleanup(null, reader);
+ assertEquals(file.length(), totalRead);
+ assertEquals(file.length(), reader.readBytes());
+ assertEquals(data.split("\n").length, i);
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index c581926..bd1a1f9 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -320,7 +320,7 @@ public class TestStorages {
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
meta.setOptions(CatalogUtil.newPhysicalProperties(storeType));
if (storeType == StoreType.AVRO) {
- String path = FileUtil.getResourcePath("testVariousTypes.avsc").toString();
+ String path = FileUtil.getResourcePath("dataset/testVariousTypes.avsc").toString();
meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java b/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
index 6186e9e..a79e8ab 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/avro/TestAvroUtil.java
@@ -49,7 +49,7 @@ public class TestAvroUtil {
@Before
public void setUp() throws Exception {
- schemaUrl = FileUtil.getResourcePath("testVariousTypes.avsc");
+ schemaUrl = FileUtil.getResourcePath("dataset/testVariousTypes.avsc");
assertNotNull(schemaUrl);
File file = new File(schemaUrl.getPath());
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/src/test/resources/dataset/testLineText.txt
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/resources/dataset/testLineText.txt b/tajo-storage/src/test/resources/dataset/testLineText.txt
new file mode 100644
index 0000000..7403c26
--- /dev/null
+++ b/tajo-storage/src/test/resources/dataset/testLineText.txt
@@ -0,0 +1,2 @@
+1|25|emiya muljomdao
+2|25|emiya muljomdao
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/src/test/resources/dataset/testVariousTypes.avsc
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/resources/dataset/testVariousTypes.avsc b/tajo-storage/src/test/resources/dataset/testVariousTypes.avsc
new file mode 100644
index 0000000..d4250a9
--- /dev/null
+++ b/tajo-storage/src/test/resources/dataset/testVariousTypes.avsc
@@ -0,0 +1,20 @@
+{
+ "type": "record",
+ "namespace": "org.apache.tajo",
+ "name": "testVariousTypes",
+ "fields": [
+ { "name": "col1", "type": "boolean" },
+ { "name": "col2", "type": "string" },
+ { "name": "col3", "type": "int" },
+ { "name": "col4", "type": "int" },
+ { "name": "col5", "type": "long" },
+ { "name": "col6", "type": "float" },
+ { "name": "col7", "type": "double" },
+ { "name": "col8", "type": "string" },
+ { "name": "col9", "type": "bytes" },
+ { "name": "col10", "type": "bytes" },
+ { "name": "col11", "type": "null" },
+ { "name": "col12", "type": "bytes" }
+ ]
+}
+
http://git-wip-us.apache.org/repos/asf/tajo/blob/bf68b770/tajo-storage/src/test/resources/testVariousTypes.avsc
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/resources/testVariousTypes.avsc b/tajo-storage/src/test/resources/testVariousTypes.avsc
deleted file mode 100644
index d4250a9..0000000
--- a/tajo-storage/src/test/resources/testVariousTypes.avsc
+++ /dev/null
@@ -1,20 +0,0 @@
-{
- "type": "record",
- "namespace": "org.apache.tajo",
- "name": "testVariousTypes",
- "fields": [
- { "name": "col1", "type": "boolean" },
- { "name": "col2", "type": "string" },
- { "name": "col3", "type": "int" },
- { "name": "col4", "type": "int" },
- { "name": "col5", "type": "long" },
- { "name": "col6", "type": "float" },
- { "name": "col7", "type": "double" },
- { "name": "col8", "type": "string" },
- { "name": "col9", "type": "bytes" },
- { "name": "col10", "type": "bytes" },
- { "name": "col11", "type": "null" },
- { "name": "col12", "type": "bytes" }
- ]
-}
-
[11/14] tajo git commit: TAJO-1095: Implement Json file scanner.
Posted by ji...@apache.org.
TAJO-1095: Implement Json file scanner.
Closes #181
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/cd38dffb
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/cd38dffb
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/cd38dffb
Branch: refs/heads/index_support
Commit: cd38dffb908a3959472f5ddb705db71d0e48ad89
Parents: 7d41c67
Author: Hyunsik Choi <hy...@apache.org>
Authored: Fri Nov 28 17:26:38 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Fri Nov 28 17:26:38 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/catalog/CatalogUtil.java | 9 +-
.../org/apache/tajo/catalog/SchemaUtil.java | 8 +
.../src/main/proto/CatalogProtos.proto | 1 +
.../apache/tajo/storage/StorageConstants.java | 2 +-
.../java/org/apache/tajo/storage/VTuple.java | 4 +-
tajo-storage/pom.xml | 6 +
.../tajo/storage/json/JsonLineDeserializer.java | 220 +++++++++++++++++++
.../apache/tajo/storage/json/JsonLineSerDe.java | 37 ++++
.../tajo/storage/json/JsonLineSerializer.java | 134 +++++++++++
.../tajo/storage/text/CSVLineDeserializer.java | 4 +-
.../apache/tajo/storage/text/CSVLineSerDe.java | 4 -
.../tajo/storage/text/CSVLineSerializer.java | 15 +-
.../tajo/storage/text/DelimitedTextFile.java | 8 +-
.../text/TextFieldSerializerDeserializer.java | 2 +-
.../tajo/storage/text/TextLineDeserializer.java | 4 +-
.../src/main/resources/storage-default.xml | 16 +-
.../org/apache/tajo/storage/TestStorages.java | 105 +++++----
.../apache/tajo/storage/json/TestJsonSerDe.java | 101 +++++++++
.../dataset/TestJsonSerDe/testVariousType.json | 1 +
.../src/test/resources/storage-default.xml | 16 +-
.../src/test/resources/testVariousTypes.avsc | 19 +-
22 files changed, 637 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 0d0677c..025ae88 100644
--- a/CHANGES
+++ b/CHANGES
@@ -11,6 +11,8 @@ Release 0.9.1 - unreleased
TAJO-235: Support Oracle CatalogStore. (Jihun Kang via hyunsik)
+ TAJO-1095: Implement Json file scanner. (hyunsik)
+
IMPROVEMENT
TAJO-1165: Needs to show error messages on query_executor.jsp. (Jihun Kang via jaehwa)
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 737c9ae..f2d9b9c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -31,9 +31,11 @@ import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.exception.InvalidOperationException;
import org.apache.tajo.storage.StorageConstants;
+import org.apache.tajo.unit.StorageUnit;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.StringUtils;
import org.apache.tajo.util.TUtil;
+import org.mortbay.util.ajax.JSON;
import java.sql.Connection;
import java.sql.ResultSet;
@@ -278,13 +280,16 @@ public class CatalogUtil {
return StoreType.AVRO;
} else if (typeStr.equalsIgnoreCase(StoreType.TEXTFILE.name())) {
return StoreType.TEXTFILE;
+ } else if (typeStr.equalsIgnoreCase(StoreType.JSON.name())) {
+ return StoreType.JSON;
} else {
return null;
}
}
public static TableMeta newTableMeta(StoreType type) {
- return new TableMeta(type, new KeyValueSet());
+ KeyValueSet defaultProperties = CatalogUtil.newPhysicalProperties(type);
+ return new TableMeta(type, defaultProperties);
}
public static TableMeta newTableMeta(StoreType type, KeyValueSet options) {
@@ -821,6 +826,8 @@ public class CatalogUtil {
KeyValueSet options = new KeyValueSet();
if (StoreType.CSV == type || StoreType.TEXTFILE == type) {
options.set(StorageConstants.TEXT_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER);
+ } else if (StoreType.JSON == type) {
+ options.set(StorageConstants.TEXT_SERDE_CLASS, "org.apache.tajo.storage.json.JsonLineSerDe");
} else if (StoreType.RCFILE == type) {
options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE);
} else if (StoreType.SEQUENCEFILE == type) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
index ee670ef..23ebe1b 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java
@@ -100,4 +100,12 @@ public class SchemaUtil {
}
return types;
}
+
+ public static String [] toSimpleNames(Schema schema) {
+ String [] names = new String[schema.size()];
+ for (int i = 0; i < schema.size(); i++) {
+ names[i] = schema.getColumn(i).getSimpleName();
+ }
+ return names;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
index 99f594a..f29bc6c 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
+++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
@@ -36,6 +36,7 @@ enum StoreType {
SEQUENCEFILE = 8;
AVRO = 9;
TEXTFILE = 10;
+ JSON = 11;
}
enum OrderType {
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
index 3065d31..a3d8de0 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java
@@ -33,7 +33,7 @@ public class StorageConstants {
public static final String TEXT_DELIMITER = "text.delimiter";
public static final String TEXT_NULL = "text.null";
public static final String TEXT_SERDE_CLASS = "text.serde.class";
- public static final String DEFAULT_TEXT_SERDE_CLASS = "org.apache.tajo.storage.text.CSVLineSerde";
+ public static final String DEFAULT_TEXT_SERDE_CLASS = "org.apache.tajo.storage.text.CSVLineSerDe";
@Deprecated
public static final String SEQUENCEFILE_DELIMITER = "sequencefile.delimiter";
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java b/tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java
index 6304734..5e839b7 100644
--- a/tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java
+++ b/tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java
@@ -56,7 +56,7 @@ public class VTuple implements Tuple, Cloneable {
@Override
public boolean isNull(int fieldid) {
- return values[fieldid].isNull();
+ return values[fieldid] == null || values[fieldid].isNull();
}
@Override
@@ -93,7 +93,7 @@ public class VTuple implements Tuple, Cloneable {
}
public void put(Datum [] values) {
- System.arraycopy(values, 0, this.values, 0, size());
+ System.arraycopy(values, 0, this.values, 0, values.length);
}
//////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml
index c6877c4..ef26a32 100644
--- a/tajo-storage/pom.xml
+++ b/tajo-storage/pom.xml
@@ -72,6 +72,7 @@
<configuration>
<excludes>
<exclude>src/test/resources/testVariousTypes.avsc</exclude>
+ <exclude>src/test/resources/dataset/TestJsonSerDe/*.json</exclude>
</excludes>
</configuration>
</plugin>
@@ -313,6 +314,11 @@
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
+ <dependency>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ <version>2.0</version>
+ </dependency>
</dependencies>
<profiles>
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
new file mode 100644
index 0000000..37cd9f3
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineDeserializer.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.json;
+
+
+import io.netty.buffer.ByteBuf;
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+import net.minidev.json.parser.JSONParser;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.SchemaUtil;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.common.exception.NotImplementedException;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.datum.protobuf.ProtobufJsonFormat;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.text.TextLineDeserializer;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+public class JsonLineDeserializer extends TextLineDeserializer {
+ private JSONParser parser;
+ private Type [] types;
+ private String [] columnNames;
+
+ public JsonLineDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
+ super(schema, meta, targetColumnIndexes);
+ }
+
+ @Override
+ public void init() {
+ types = SchemaUtil.toTypes(schema);
+ columnNames = SchemaUtil.toSimpleNames(schema);
+
+ parser = new JSONParser(JSONParser.MODE_JSON_SIMPLE);
+ }
+
+ @Override
+ public void deserialize(ByteBuf buf, Tuple output) throws IOException {
+ byte [] line = new byte[buf.readableBytes()];
+ buf.readBytes(line);
+
+ try {
+ JSONObject object = (JSONObject) parser.parse(line);
+
+ for (int i = 0; i < targetColumnIndexes.length; i++) {
+ int actualIdx = targetColumnIndexes[i];
+ String fieldName = columnNames[actualIdx];
+
+ if (!object.containsKey(fieldName)) {
+ output.put(actualIdx, NullDatum.get());
+ continue;
+ }
+
+ switch (types[actualIdx]) {
+ case BOOLEAN:
+ String boolStr = object.getAsString(fieldName);
+ if (boolStr != null) {
+ output.put(actualIdx, DatumFactory.createBool(boolStr.equals("true")));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case CHAR:
+ String charStr = object.getAsString(fieldName);
+ if (charStr != null) {
+ output.put(actualIdx, DatumFactory.createChar(charStr));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case INT1:
+ case INT2:
+ Number int2Num = object.getAsNumber(fieldName);
+ if (int2Num != null) {
+ output.put(actualIdx, DatumFactory.createInt2(int2Num.shortValue()));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case INT4:
+ Number int4Num = object.getAsNumber(fieldName);
+ if (int4Num != null) {
+ output.put(actualIdx, DatumFactory.createInt4(int4Num.intValue()));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case INT8:
+ Number int8Num = object.getAsNumber(fieldName);
+ if (int8Num != null) {
+ output.put(actualIdx, DatumFactory.createInt8(int8Num.longValue()));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case FLOAT4:
+ Number float4Num = object.getAsNumber(fieldName);
+ if (float4Num != null) {
+ output.put(actualIdx, DatumFactory.createFloat4(float4Num.floatValue()));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case FLOAT8:
+ Number float8Num = object.getAsNumber(fieldName);
+ if (float8Num != null) {
+ output.put(actualIdx, DatumFactory.createFloat8(float8Num.doubleValue()));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case TEXT:
+ String textStr = object.getAsString(fieldName);
+ if (textStr != null) {
+ output.put(actualIdx, DatumFactory.createText(textStr));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case TIMESTAMP:
+ String timestampStr = object.getAsString(fieldName);
+ if (timestampStr != null) {
+ output.put(actualIdx, DatumFactory.createTimestamp(timestampStr));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case TIME:
+ String timeStr = object.getAsString(fieldName);
+ if (timeStr != null) {
+ output.put(actualIdx, DatumFactory.createTime(timeStr));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case DATE:
+ String dateStr = object.getAsString(fieldName);
+ if (dateStr != null) {
+ output.put(actualIdx, DatumFactory.createDate(dateStr));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ case BIT:
+ case BINARY:
+ case VARBINARY:
+ case BLOB: {
+ Object jsonObject = object.get(fieldName);
+
+ if (jsonObject == null) {
+ output.put(actualIdx, NullDatum.get());
+ break;
+ } if (jsonObject instanceof String) {
+ output.put(actualIdx, DatumFactory.createBlob((String)jsonObject));
+ } else if (jsonObject instanceof JSONArray) {
+ JSONArray jsonArray = (JSONArray) jsonObject;
+ byte[] bytes = new byte[jsonArray.size()];
+ Iterator<Object> it = jsonArray.iterator();
+ int arrayIdx = 0;
+ while (it.hasNext()) {
+ bytes[arrayIdx++] = ((Long) it.next()).byteValue();
+ }
+ if (bytes.length > 0) {
+ output.put(actualIdx, DatumFactory.createBlob(bytes));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+ } else {
+ throw new IOException("Unknown json object: " + object.getClass().getSimpleName());
+ }
+ break;
+ }
+ case INET4:
+ String inetStr = object.getAsString(fieldName);
+ if (inetStr != null) {
+ output.put(actualIdx, DatumFactory.createInet4(inetStr));
+ } else {
+ output.put(actualIdx, NullDatum.get());
+ }
+ break;
+
+ case NULL_TYPE:
+ output.put(actualIdx, NullDatum.get());
+ break;
+
+ default:
+ throw new NotImplementedException(types[actualIdx].name() + " is not supported.");
+ }
+ }
+
+ } catch (Throwable e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void release() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java b/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
new file mode 100644
index 0000000..6db2c29
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerDe.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.json;
+
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.storage.text.TextLineDeserializer;
+import org.apache.tajo.storage.text.TextLineSerDe;
+import org.apache.tajo.storage.text.TextLineSerializer;
+
+public class JsonLineSerDe extends TextLineSerDe {
+ @Override
+ public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
+ return new JsonLineDeserializer(schema, meta, targetColumnIndexes);
+ }
+
+ @Override
+ public TextLineSerializer createSerializer(Schema schema, TableMeta meta) {
+ return new JsonLineSerializer(schema, meta);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
new file mode 100644
index 0000000..c7007d8
--- /dev/null
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/json/JsonLineSerializer.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.json;
+
+
+import net.minidev.json.JSONObject;
+import org.apache.commons.lang.CharSet;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.SchemaUtil;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.common.TajoDataTypes.Type;
+import org.apache.tajo.common.exception.NotImplementedException;
+import org.apache.tajo.datum.ProtobufDatum;
+import org.apache.tajo.datum.TextDatum;
+import org.apache.tajo.datum.protobuf.ProtobufJsonFormat;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.text.TextLineSerDe;
+import org.apache.tajo.storage.text.TextLineSerializer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+
+public class JsonLineSerializer extends TextLineSerializer {
+ private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
+
+ private Type [] types;
+ private String [] simpleNames;
+ private int columnNum;
+
+
+ public JsonLineSerializer(Schema schema, TableMeta meta) {
+ super(schema, meta);
+ }
+
+ @Override
+ public void init() {
+ types = SchemaUtil.toTypes(schema);
+ simpleNames = SchemaUtil.toSimpleNames(schema);
+ columnNum = schema.size();
+ }
+
+ @Override
+ public int serialize(OutputStream out, Tuple input) throws IOException {
+ JSONObject jsonObject = new JSONObject();
+
+ for (int i = 0; i < columnNum; i++) {
+ if (input.isNull(i)) {
+ continue;
+ }
+
+ String fieldName = simpleNames[i];
+ Type type = types[i];
+
+ switch (type) {
+
+ case BOOLEAN:
+ jsonObject.put(fieldName, input.getBool(i));
+ break;
+
+ case INT1:
+ case INT2:
+ jsonObject.put(fieldName, input.getInt2(i));
+ break;
+
+ case INT4:
+ jsonObject.put(fieldName, input.getInt4(i));
+ break;
+
+ case INT8:
+ jsonObject.put(fieldName, input.getInt8(i));
+ break;
+
+ case FLOAT4:
+ jsonObject.put(fieldName, input.getFloat4(i));
+ break;
+
+ case FLOAT8:
+ jsonObject.put(fieldName, input.getFloat8(i));
+ break;
+
+ case CHAR:
+ case TEXT:
+ case VARCHAR:
+ case INET4:
+ case TIMESTAMP:
+ case DATE:
+ case TIME:
+ case INTERVAL:
+ jsonObject.put(fieldName, input.getText(i));
+ break;
+
+ case BIT:
+ case BINARY:
+ case BLOB:
+ case VARBINARY:
+ jsonObject.put(fieldName, input.getBytes(i));
+ break;
+
+ case NULL_TYPE:
+ break;
+
+ default:
+ throw new NotImplementedException(types[i].name() + " is not supported.");
+ }
+ }
+
+ String jsonStr = jsonObject.toJSONString();
+ byte [] jsonBytes = jsonStr.getBytes(TextDatum.DEFAULT_CHARSET);
+ out.write(jsonBytes);
+ return jsonBytes.length;
+ }
+
+ @Override
+ public void release() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
index f580da1..0e2dfb0 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineDeserializer.java
@@ -48,7 +48,7 @@ public class CSVLineDeserializer extends TextLineDeserializer {
fieldSerDer = new TextFieldSerializerDeserializer();
}
- public void deserialize(final ByteBuf lineBuf, Tuple tuple) throws IOException {
+ public void deserialize(final ByteBuf lineBuf, Tuple output) throws IOException {
int[] projection = targetColumnIndexes;
if (lineBuf == null || targetColumnIndexes == null || targetColumnIndexes.length == 0) {
return;
@@ -73,7 +73,7 @@ public class CSVLineDeserializer extends TextLineDeserializer {
if (projection.length > currentTarget && currentIndex == projection[currentTarget]) {
lineBuf.setIndex(start, start + fieldLength);
Datum datum = fieldSerDer.deserialize(lineBuf, schema.getColumn(currentIndex), currentIndex, nullChars);
- tuple.put(currentIndex, datum);
+ output.put(currentIndex, datum);
currentTarget++;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
index e2686a6..2fe7f23 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerDe.java
@@ -24,10 +24,6 @@ import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.storage.StorageConstants;
public class CSVLineSerDe extends TextLineSerDe {
-
- public CSVLineSerDe() {
- }
-
@Override
public TextLineDeserializer createDeserializer(Schema schema, TableMeta meta, int[] targetColumnIndexes) {
return new CSVLineDeserializer(schema, meta, targetColumnIndexes);
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
index 684519c..7397000 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/CSVLineSerializer.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage.text;
+import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.datum.Datum;
@@ -32,6 +33,7 @@ public class CSVLineSerializer extends TextLineSerializer {
private byte [] nullChars;
private char delimiter;
+ private int columnNum;
public CSVLineSerializer(Schema schema, TableMeta meta) {
super(schema, meta);
@@ -41,25 +43,26 @@ public class CSVLineSerializer extends TextLineSerializer {
public void init() {
nullChars = TextLineSerDe.getNullCharsAsBytes(meta);
delimiter = CSVLineSerDe.getFieldDelimiter(meta);
+ columnNum = schema.size();
serde = new TextFieldSerializerDeserializer();
}
@Override
public int serialize(OutputStream out, Tuple input) throws IOException {
- int rowBytes = 0;
+ int writtenBytes = 0;
- for (int i = 0; i < schema.size(); i++) {
+ for (int i = 0; i < columnNum; i++) {
Datum datum = input.get(i);
- rowBytes += serde.serialize(out, datum, schema.getColumn(i), i, nullChars);
+ writtenBytes += serde.serialize(out, datum, schema.getColumn(i), i, nullChars);
- if (schema.size() - 1 > i) {
+ if (columnNum - 1 > i) {
out.write((byte) delimiter);
- rowBytes += 1;
+ writtenBytes += 1;
}
}
- return rowBytes;
+ return writtenBytes;
}
@Override
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
index d15f394..2218fae 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
@@ -79,12 +79,12 @@ public class DelimitedTextFile {
if (serdeClassCache.containsKey(serDeClassName)) {
serdeClass = serdeClassCache.get(serDeClassName);
} else {
- serdeClass = (Class<? extends TextLineSerDe>) Class.forName(CSVLineSerDe.class.getName());
+ serdeClass = (Class<? extends TextLineSerDe>) Class.forName(serDeClassName);
serdeClassCache.put(serDeClassName, serdeClass);
}
lineSerder = (TextLineSerDe) ReflectionUtil.newInstance(serdeClass);
} catch (Throwable e) {
- throw new RuntimeException("TextLineSerde class cannot be initialized");
+ throw new RuntimeException("TextLineSerde class cannot be initialized.", e);
}
return lineSerder;
@@ -382,7 +382,9 @@ public class DelimitedTextFile {
@Override
public void close() throws IOException {
try {
- deserializer.release();
+ if (deserializer != null) {
+ deserializer.release();
+ }
if (tableStats != null && reader != null) {
tableStats.setReadBytes(reader.getReadBytes()); //Actual Processed Bytes. (decompressed bytes + overhead)
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java
index 9722959..95d0407 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextFieldSerializerDeserializer.java
@@ -38,7 +38,7 @@ import java.nio.charset.CharsetDecoder;
public class TextFieldSerializerDeserializer implements FieldSerializerDeserializer {
public static final byte[] trueBytes = "true".getBytes();
public static final byte[] falseBytes = "false".getBytes();
- private ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
+ private static ProtobufJsonFormat protobufJsonFormat = ProtobufJsonFormat.getInstance();
private final CharsetDecoder decoder = CharsetUtil.getDecoder(CharsetUtil.UTF_8);
private static boolean isNull(ByteBuf val, ByteBuf nullBytes) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
index 645d118..b0d3c3a 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/text/TextLineDeserializer.java
@@ -48,10 +48,10 @@ public abstract class TextLineDeserializer {
* It fills a tuple with a read fields in a given line.
*
* @param buf Read line
- * @param tuple Tuple to be filled with read fields
+ * @param output Tuple to be filled with read fields
* @throws java.io.IOException
*/
- public abstract void deserialize(final ByteBuf buf, Tuple tuple) throws IOException;
+ public abstract void deserialize(final ByteBuf buf, Tuple output) throws IOException;
/**
* Release external resources
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/main/resources/storage-default.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/resources/storage-default.xml b/tajo-storage/src/main/resources/storage-default.xml
index 064f250..e861b7d 100644
--- a/tajo-storage/src/main/resources/storage-default.xml
+++ b/tajo-storage/src/main/resources/storage-default.xml
@@ -35,7 +35,7 @@
<!--- Registered Scanner Handler -->
<property>
<name>tajo.storage.scanner-handler</name>
- <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value>
+ <value>textfile,csv,json,raw,rcfile,row,parquet,sequencefile,avro</value>
</property>
<!--- Fragment Class Configurations -->
@@ -48,6 +48,10 @@
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
<property>
+ <name>tajo.storage.fragment.json.class</name>
+ <value>org.apache.tajo.storage.fragment.FileFragment</value>
+ </property>
+ <property>
<name>tajo.storage.fragment.raw.class</name>
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
@@ -84,6 +88,11 @@
</property>
<property>
+ <name>tajo.storage.scanner-handler.json.class</name>
+ <value>org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner</value>
+ </property>
+
+ <property>
<name>tajo.storage.scanner-handler.raw.class</name>
<value>org.apache.tajo.storage.RawFile$RawFileScanner</value>
</property>
@@ -130,6 +139,11 @@
</property>
<property>
+ <name>tajo.storage.appender-handler.json.class</name>
+ <value>org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender</value>
+ </property>
+
+ <property>
<name>tajo.storage.appender-handler.raw.class</name>
<value>org.apache.tajo.storage.RawFile$RawFileAppender</value>
</property>
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
index 6e2bc35..c581926 100644
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
@@ -80,18 +80,17 @@ public class TestStorages {
" \"name\": \"testNullHandlingTypes\",\n" +
" \"fields\": [\n" +
" { \"name\": \"col1\", \"type\": [\"null\", \"boolean\"] },\n" +
- " { \"name\": \"col2\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col3\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col2\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col3\", \"type\": [\"null\", \"int\"] },\n" +
" { \"name\": \"col4\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col5\", \"type\": [\"null\", \"int\"] },\n" +
- " { \"name\": \"col6\", \"type\": [\"null\", \"long\"] },\n" +
- " { \"name\": \"col7\", \"type\": [\"null\", \"float\"] },\n" +
- " { \"name\": \"col8\", \"type\": [\"null\", \"double\"] },\n" +
- " { \"name\": \"col9\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col5\", \"type\": [\"null\", \"long\"] },\n" +
+ " { \"name\": \"col6\", \"type\": [\"null\", \"float\"] },\n" +
+ " { \"name\": \"col7\", \"type\": [\"null\", \"double\"] },\n" +
+ " { \"name\": \"col8\", \"type\": [\"null\", \"string\"] },\n" +
+ " { \"name\": \"col9\", \"type\": [\"null\", \"bytes\"] },\n" +
" { \"name\": \"col10\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col11\", \"type\": [\"null\", \"bytes\"] },\n" +
- " { \"name\": \"col12\", \"type\": \"null\" },\n" +
- " { \"name\": \"col13\", \"type\": [\"null\", \"bytes\"] }\n" +
+ " { \"name\": \"col11\", \"type\": \"null\" },\n" +
+ " { \"name\": \"col12\", \"type\": [\"null\", \"bytes\"] }\n" +
" ]\n" +
"}\n";
@@ -129,6 +128,7 @@ public class TestStorages {
{StoreType.SEQUENCEFILE, true, true, false},
{StoreType.AVRO, false, false, false},
{StoreType.TEXTFILE, true, true, false},
+ {StoreType.JSON, true, true, false},
});
}
@@ -298,20 +298,23 @@ public class TestStorages {
@Test
public void testVariousTypes() throws IOException {
+ boolean handleProtobuf = storeType != StoreType.JSON;
+
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col2", Type.CHAR, 7);
+ schema.addColumn("col3", Type.INT2);
+ schema.addColumn("col4", Type.INT4);
+ schema.addColumn("col5", Type.INT8);
+ schema.addColumn("col6", Type.FLOAT4);
+ schema.addColumn("col7", Type.FLOAT8);
+ schema.addColumn("col8", Type.TEXT);
+ schema.addColumn("col9", Type.BLOB);
+ schema.addColumn("col10", Type.INET4);
+ schema.addColumn("col11", Type.NULL_TYPE);
+ if (handleProtobuf) {
+ schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ }
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
@@ -328,10 +331,9 @@ public class TestStorages {
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
- Tuple tuple = new VTuple(13);
+ Tuple tuple = new VTuple(11 + (handleProtobuf ? 1 : 0));
tuple.put(new Datum[] {
DatumFactory.createBool(true),
- DatumFactory.createBit((byte) 0x99),
DatumFactory.createChar("hyunsik"),
DatumFactory.createInt2((short) 17),
DatumFactory.createInt4(59),
@@ -341,9 +343,12 @@ public class TestStorages {
DatumFactory.createText("hyunsik"),
DatumFactory.createBlob("hyunsik".getBytes()),
DatumFactory.createInet4("192.168.0.1"),
- NullDatum.get(),
- factory.createDatum(queryid.getProto())
+ NullDatum.get()
});
+ if (handleProtobuf) {
+ tuple.put(11, factory.createDatum(queryid.getProto()));
+ }
+
appender.addTuple(tuple);
appender.flush();
appender.close();
@@ -364,20 +369,24 @@ public class TestStorages {
@Test
public void testNullHandlingTypes() throws IOException {
+ boolean handleProtobuf = storeType != StoreType.JSON;
+
Schema schema = new Schema();
schema.addColumn("col1", Type.BOOLEAN);
- schema.addColumn("col2", Type.BIT);
- schema.addColumn("col3", Type.CHAR, 7);
- schema.addColumn("col4", Type.INT2);
- schema.addColumn("col5", Type.INT4);
- schema.addColumn("col6", Type.INT8);
- schema.addColumn("col7", Type.FLOAT4);
- schema.addColumn("col8", Type.FLOAT8);
- schema.addColumn("col9", Type.TEXT);
- schema.addColumn("col10", Type.BLOB);
- schema.addColumn("col11", Type.INET4);
- schema.addColumn("col12", Type.NULL_TYPE);
- schema.addColumn("col13", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ schema.addColumn("col2", Type.CHAR, 7);
+ schema.addColumn("col3", Type.INT2);
+ schema.addColumn("col4", Type.INT4);
+ schema.addColumn("col5", Type.INT8);
+ schema.addColumn("col6", Type.FLOAT4);
+ schema.addColumn("col7", Type.FLOAT8);
+ schema.addColumn("col8", Type.TEXT);
+ schema.addColumn("col9", Type.BLOB);
+ schema.addColumn("col10", Type.INET4);
+ schema.addColumn("col11", Type.NULL_TYPE);
+
+ if (handleProtobuf) {
+ schema.addColumn("col12", CatalogUtil.newDataType(Type.PROTOBUF, TajoIdProtos.QueryIdProto.class.getName()));
+ }
KeyValueSet options = new KeyValueSet();
TableMeta meta = CatalogUtil.newTableMeta(storeType, options);
@@ -397,11 +406,10 @@ public class TestStorages {
QueryId queryid = new QueryId("12345", 5);
ProtobufDatumFactory factory = ProtobufDatumFactory.get(TajoIdProtos.QueryIdProto.class.getName());
-
- Tuple seedTuple = new VTuple(13);
+ int columnNum = 11 + (handleProtobuf ? 1 : 0);
+ Tuple seedTuple = new VTuple(columnNum);
seedTuple.put(new Datum[]{
DatumFactory.createBool(true), // 0
- DatumFactory.createBit((byte) 0x99), // 1
DatumFactory.createChar("hyunsik"), // 2
DatumFactory.createInt2((short) 17), // 3
DatumFactory.createInt4(59), // 4
@@ -412,14 +420,17 @@ public class TestStorages {
DatumFactory.createBlob("hyunsik".getBytes()),// 9
DatumFactory.createInet4("192.168.0.1"), // 10
NullDatum.get(), // 11
- factory.createDatum(queryid.getProto()) // 12
});
+ if (handleProtobuf) {
+ seedTuple.put(11, factory.createDatum(queryid.getProto())); // 12
+ }
+
// Making tuples with different null column positions
Tuple tuple;
- for (int i = 0; i < 13; i++) {
- tuple = new VTuple(13);
- for (int j = 0; j < 13; j++) {
+ for (int i = 0; i < columnNum; i++) {
+ tuple = new VTuple(columnNum);
+ for (int j = 0; j < columnNum; j++) {
if (i == j) { // i'th column will have NULL value
tuple.put(j, NullDatum.get());
} else {
@@ -439,8 +450,8 @@ public class TestStorages {
Tuple retrieved;
int i = 0;
while ((retrieved = scanner.next()) != null) {
- assertEquals(13, retrieved.size());
- for (int j = 0; j < 13; j++) {
+ assertEquals(columnNum, retrieved.size());
+ for (int j = 0; j < columnNum; j++) {
if (i == j) {
assertEquals(NullDatum.get(), retrieved.get(j));
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java b/tajo-storage/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
new file mode 100644
index 0000000..038bc17
--- /dev/null
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/json/TestJsonSerDe.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.storage.json;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.catalog.Schema;
+import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.common.TajoDataTypes;
+import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.datum.Datum;
+import org.apache.tajo.datum.DatumFactory;
+import org.apache.tajo.datum.NullDatum;
+import org.apache.tajo.storage.Scanner;
+import org.apache.tajo.storage.StorageManager;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.VTuple;
+import org.apache.tajo.storage.fragment.FileFragment;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URL;
+
+import static org.junit.Assert.*;
+
+public class TestJsonSerDe {
+ private static Schema schema = new Schema();
+
+ static {
+ schema.addColumn("col1", TajoDataTypes.Type.BOOLEAN);
+ schema.addColumn("col2", TajoDataTypes.Type.CHAR, 7);
+ schema.addColumn("col3", TajoDataTypes.Type.INT2);
+ schema.addColumn("col4", TajoDataTypes.Type.INT4);
+ schema.addColumn("col5", TajoDataTypes.Type.INT8);
+ schema.addColumn("col6", TajoDataTypes.Type.FLOAT4);
+ schema.addColumn("col7", TajoDataTypes.Type.FLOAT8);
+ schema.addColumn("col8", TajoDataTypes.Type.TEXT);
+ schema.addColumn("col9", TajoDataTypes.Type.BLOB);
+ schema.addColumn("col10", TajoDataTypes.Type.INET4);
+ schema.addColumn("col11", TajoDataTypes.Type.NULL_TYPE);
+ }
+
+ public static Path getResourcePath(String path, String suffix) {
+ URL resultBaseURL = ClassLoader.getSystemResource(path);
+ return new Path(resultBaseURL.toString(), suffix);
+ }
+
+ @Test
+ public void testVarioutType() throws IOException {
+ TajoConf conf = new TajoConf();
+
+ TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.JSON);
+ Path tablePath = new Path(getResourcePath("dataset", "TestJsonSerDe"), "testVariousType.json");
+ FileSystem fs = FileSystem.getLocal(conf);
+ FileStatus status = fs.getFileStatus(tablePath);
+ FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen());
+ Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment);
+ scanner.init();
+
+ Tuple tuple = scanner.next();
+ assertNotNull(tuple);
+ assertNull(scanner.next());
+ scanner.close();
+
+ Tuple baseTuple = new VTuple(11);
+ baseTuple.put(new Datum[] {
+ DatumFactory.createBool(true), // 0
+ DatumFactory.createChar("hyunsik"), // 1
+ DatumFactory.createInt2((short) 17), // 2
+ DatumFactory.createInt4(59), // 3
+ DatumFactory.createInt8(23l), // 4
+ DatumFactory.createFloat4(77.9f), // 5
+ DatumFactory.createFloat8(271.9d), // 6
+ DatumFactory.createText("hyunsik"), // 7
+ DatumFactory.createBlob("hyunsik".getBytes()), // 8
+ DatumFactory.createInet4("192.168.0.1"), // 9
+ NullDatum.get(), // 10
+ });
+
+ assertEquals(baseTuple, tuple);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/test/resources/dataset/TestJsonSerDe/testVariousType.json
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/resources/dataset/TestJsonSerDe/testVariousType.json b/tajo-storage/src/test/resources/dataset/TestJsonSerDe/testVariousType.json
new file mode 100644
index 0000000..8ee3408
--- /dev/null
+++ b/tajo-storage/src/test/resources/dataset/TestJsonSerDe/testVariousType.json
@@ -0,0 +1 @@
+{"col1": "true", "col2": "hyunsik", "col3": 17, "col4": 59, "col5": 23, "col6": 77.9, "col7": 271.9, "col8": "hyunsik", "col9": "hyunsik", "col10": "192.168.0.1"}
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/test/resources/storage-default.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/resources/storage-default.xml b/tajo-storage/src/test/resources/storage-default.xml
index 790d5a8..f4c81c7 100644
--- a/tajo-storage/src/test/resources/storage-default.xml
+++ b/tajo-storage/src/test/resources/storage-default.xml
@@ -28,7 +28,7 @@
<!--- Registered Scanner Handler -->
<property>
<name>tajo.storage.scanner-handler</name>
- <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value>
+ <value>textfile,csv,json,raw,rcfile,row,parquet,sequencefile,avro</value>
</property>
<!--- Fragment Class Configurations -->
@@ -41,6 +41,10 @@
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
<property>
+ <name>tajo.storage.fragment.json.class</name>
+ <value>org.apache.tajo.storage.fragment.FileFragment</value>
+ </property>
+ <property>
<name>tajo.storage.fragment.raw.class</name>
<value>org.apache.tajo.storage.fragment.FileFragment</value>
</property>
@@ -77,6 +81,11 @@
</property>
<property>
+ <name>tajo.storage.scanner-handler.json.class</name>
+ <value>org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileScanner</value>
+ </property>
+
+ <property>
<name>tajo.storage.scanner-handler.raw.class</name>
<value>org.apache.tajo.storage.RawFile$RawFileScanner</value>
</property>
@@ -123,6 +132,11 @@
</property>
<property>
+ <name>tajo.storage.appender-handler.json.class</name>
+ <value>org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender</value>
+ </property>
+
+ <property>
<name>tajo.storage.appender-handler.raw.class</name>
<value>org.apache.tajo.storage.RawFile$RawFileAppender</value>
</property>
http://git-wip-us.apache.org/repos/asf/tajo/blob/cd38dffb/tajo-storage/src/test/resources/testVariousTypes.avsc
----------------------------------------------------------------------
diff --git a/tajo-storage/src/test/resources/testVariousTypes.avsc b/tajo-storage/src/test/resources/testVariousTypes.avsc
index 611b97f..d4250a9 100644
--- a/tajo-storage/src/test/resources/testVariousTypes.avsc
+++ b/tajo-storage/src/test/resources/testVariousTypes.avsc
@@ -4,18 +4,17 @@
"name": "testVariousTypes",
"fields": [
{ "name": "col1", "type": "boolean" },
- { "name": "col2", "type": "int" },
- { "name": "col3", "type": "string" },
+ { "name": "col2", "type": "string" },
+ { "name": "col3", "type": "int" },
{ "name": "col4", "type": "int" },
- { "name": "col5", "type": "int" },
- { "name": "col6", "type": "long" },
- { "name": "col7", "type": "float" },
- { "name": "col8", "type": "double" },
- { "name": "col9", "type": "string" },
+ { "name": "col5", "type": "long" },
+ { "name": "col6", "type": "float" },
+ { "name": "col7", "type": "double" },
+ { "name": "col8", "type": "string" },
+ { "name": "col9", "type": "bytes" },
{ "name": "col10", "type": "bytes" },
- { "name": "col11", "type": "bytes" },
- { "name": "col12", "type": "null" },
- { "name": "col13", "type": "bytes" }
+ { "name": "col11", "type": "null" },
+ { "name": "col12", "type": "bytes" }
]
}
[05/14] tajo git commit: TAJO-1166: S3 related storage causes
compilation error in Hadoop 2.6.0-SNAPSHOT. (jaehwa)
Posted by ji...@apache.org.
TAJO-1166: S3 related storage causes compilation error in Hadoop 2.6.0-SNAPSHOT. (jaehwa)
Closes #262
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3d485ecb
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3d485ecb
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3d485ecb
Branch: refs/heads/index_support
Commit: 3d485ecb0112af12258a5a2bdc4a400b8df4fae8
Parents: 0c97fc0
Author: JaeHwa Jung <bl...@apache.org>
Authored: Thu Nov 27 18:42:24 2014 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Thu Nov 27 18:45:18 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
tajo-core/pom.xml | 1 +
.../org/apache/tajo/master/ContainerProxy.java | 10 +-
.../tajo/master/DefaultTaskScheduler.java | 18 +-
.../tajo/master/LaunchTaskRunnersEvent.java | 5 +-
.../apache/tajo/master/LazyTaskScheduler.java | 9 +-
.../apache/tajo/master/TajoContainerProxy.java | 22 +-
.../apache/tajo/master/TajoMasterService.java | 5 +-
.../tajo/master/TaskRunnerGroupEvent.java | 8 +-
.../tajo/master/container/TajoContainer.java | 173 ++++++++++
.../tajo/master/container/TajoContainerId.java | 172 ++++++++++
.../master/container/TajoConverterUtils.java | 263 ++++++++++++++++
.../master/container/TajoRecordFactory.java | 31 ++
.../container/TajoRecordFactoryPBImpl.java | 104 ++++++
.../container/TajoRecordFactoryProvider.java | 70 +++++
.../tajo/master/container/TajoRecords.java | 39 +++
.../impl/pb/TajoContainerIdPBImpl.java | 100 ++++++
.../tajo/master/event/LocalTaskEvent.java | 9 +-
.../event/QueryUnitAttemptScheduleEvent.java | 10 +-
.../event/SubQueryContainerAllocationEvent.java | 8 +-
.../master/event/TaskAttemptAssignedEvent.java | 8 +-
.../tajo/master/event/TaskRequestEvent.java | 8 +-
.../master/querymaster/QueryInProgress.java | 3 +-
.../querymaster/QueryMasterManagerService.java | 4 +-
.../master/querymaster/QueryUnitAttempt.java | 6 +-
.../tajo/master/querymaster/SubQuery.java | 17 +-
.../apache/tajo/master/rm/TajoRMContext.java | 8 +-
.../tajo/master/rm/TajoWorkerContainer.java | 15 +-
.../tajo/master/rm/TajoWorkerContainerId.java | 53 ++--
.../master/rm/TajoWorkerResourceManager.java | 63 ++--
.../tajo/master/rm/WorkerResourceManager.java | 4 +-
.../tajo/worker/AbstractResourceAllocator.java | 14 +-
.../apache/tajo/worker/ResourceAllocator.java | 6 +-
.../tajo/worker/TajoResourceAllocator.java | 101 +++---
.../java/org/apache/tajo/worker/TaskRunner.java | 16 +-
.../apache/tajo/worker/TaskRunnerHistory.java | 14 +-
.../src/main/proto/ContainerProtocol.proto | 48 +++
.../src/main/proto/QueryMasterProtocol.proto | 3 +
.../main/proto/ResourceTrackerProtocol.proto | 3 +
.../src/main/proto/TajoMasterProtocol.proto | 7 +-
.../src/main/proto/TajoWorkerProtocol.proto | 5 +-
.../tajo/master/rm/TestTajoResourceManager.java | 16 +-
.../org/apache/tajo/storage/StorageManager.java | 1 +
.../apache/tajo/storage/TestFileSystems.java | 267 ++++++++--------
.../java/org/apache/tajo/storage/s3/INode.java | 124 --------
.../storage/s3/InMemoryFileSystemStore.java | 175 -----------
.../apache/tajo/storage/s3/S3OutputStream.java | 234 --------------
.../tajo/storage/s3/SmallBlockS3FileSystem.java | 314 -------------------
48 files changed, 1392 insertions(+), 1204 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6521c1c..1657c20 100644
--- a/CHANGES
+++ b/CHANGES
@@ -68,6 +68,8 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1166: S3 related storage causes compilation error in Hadoop 2.6.0-SNAPSHOT. (jaehwa)
+
TAJO-1208: Failure of create table using textfile on hivemeta.
(jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml
index fce96e4..060ac1b 100644
--- a/tajo-core/pom.xml
+++ b/tajo-core/pom.xml
@@ -162,6 +162,7 @@
<argument>--proto_path=../tajo-client/src/main/proto</argument>
<argument>--proto_path=../tajo-plan/src/main/proto</argument>
<argument>--java_out=target/generated-sources/proto</argument>
+ <argument>src/main/proto/ContainerProtocol.proto</argument>
<argument>src/main/proto/ResourceTrackerProtocol.proto</argument>
<argument>src/main/proto/QueryMasterProtocol.proto</argument>
<argument>src/main/proto/TajoMasterProtocol.proto</argument>
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java
index 59b071a..462de91 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/ContainerProxy.java
@@ -22,11 +22,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.master.querymaster.QueryMasterTask;
+import org.apache.tajo.master.container.TajoContainer;
+import org.apache.tajo.master.container.TajoContainerId;
public abstract class ContainerProxy {
protected static final Log LOG = LogFactory.getLog(ContainerProxy.class);
@@ -45,8 +45,8 @@ public abstract class ContainerProxy {
protected ContainerState state;
// store enough information to be able to cleanup the container
- protected Container container;
- protected ContainerId containerID;
+ protected TajoContainer container;
+ protected TajoContainerId containerID;
protected String hostName;
protected int port = -1;
@@ -54,7 +54,7 @@ public abstract class ContainerProxy {
public abstract void stopContainer();
public ContainerProxy(QueryMasterTask.QueryMasterTaskContext context, Configuration conf,
- ExecutionBlockId executionBlockId, Container container) {
+ ExecutionBlockId executionBlockId, TajoContainer container) {
this.context = context;
this.conf = conf;
this.state = ContainerState.PREP;
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
index 62d4892..77e3257 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/DefaultTaskScheduler.java
@@ -41,6 +41,7 @@ import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
import org.apache.tajo.master.querymaster.QueryUnit;
import org.apache.tajo.master.querymaster.QueryUnitAttempt;
import org.apache.tajo.master.querymaster.SubQuery;
+import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.storage.DataLocation;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.NetUtils;
@@ -338,7 +339,8 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
private Map<Integer, LinkedHashSet<QueryUnitAttempt>> unassignedTaskForEachVolume =
Collections.synchronizedMap(new HashMap<Integer, LinkedHashSet<QueryUnitAttempt>>());
/** A value is last assigned volume id for each task runner */
- private HashMap<ContainerId, Integer> lastAssignedVolumeId = new HashMap<ContainerId, Integer>();
+ private HashMap<TajoContainerId, Integer> lastAssignedVolumeId = new HashMap<TajoContainerId,
+ Integer>();
/**
* A key is disk volume id, and a value is the load of this volume.
* This load is measured by counting how many number of tasks are running.
@@ -378,7 +380,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
* 2. unknown block or Non-splittable task in host
* 3. remote tasks. unassignedTaskForEachVolume is only contained local task. so it will be null
*/
- public synchronized QueryUnitAttemptId getLocalTask(ContainerId containerId) {
+ public synchronized QueryUnitAttemptId getLocalTask(TajoContainerId containerId) {
int volumeId;
QueryUnitAttemptId queryUnitAttemptId = null;
@@ -489,7 +491,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
* @param volumeId Volume identifier
* @return the volume load (i.e., how many running tasks use this volume)
*/
- private synchronized int increaseConcurrency(ContainerId containerId, int volumeId) {
+ private synchronized int increaseConcurrency(TajoContainerId containerId, int volumeId) {
int concurrency = 1;
if (diskVolumeLoads.containsKey(volumeId)) {
@@ -514,7 +516,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
/**
* Decrease the count of running tasks of a certain task runner
*/
- private synchronized void decreaseConcurrency(ContainerId containerId){
+ private synchronized void decreaseConcurrency(TajoContainerId containerId){
Integer volumeId = lastAssignedVolumeId.get(containerId);
if(volumeId != null && diskVolumeLoads.containsKey(volumeId)){
Integer concurrency = diskVolumeLoads.get(volumeId);
@@ -552,11 +554,11 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
- public boolean isAssigned(ContainerId containerId){
+ public boolean isAssigned(TajoContainerId containerId){
return lastAssignedVolumeId.containsKey(containerId);
}
- public boolean isRemote(ContainerId containerId){
+ public boolean isRemote(TajoContainerId containerId){
Integer volumeId = lastAssignedVolumeId.get(containerId);
if(volumeId == null || volumeId > REMOTE){
return false;
@@ -647,7 +649,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
public Set<QueryUnitAttemptId> assignedRequest = new HashSet<QueryUnitAttemptId>();
- private QueryUnitAttemptId allocateLocalTask(String host, ContainerId containerId){
+ private QueryUnitAttemptId allocateLocalTask(String host, TajoContainerId containerId){
HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
if (hostVolumeMapping != null) { //tajo host is located in hadoop datanode
@@ -778,7 +780,7 @@ public class DefaultTaskScheduler extends AbstractTaskScheduler {
}
}
- ContainerId containerId = taskRequest.getContainerId();
+ TajoContainerId containerId = taskRequest.getContainerId();
LOG.debug("assignToLeafTasks: " + taskRequest.getExecutionBlockId() + "," +
"containerId=" + containerId);
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java
index 9a4a01d..e620afa 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/LaunchTaskRunnersEvent.java
@@ -18,9 +18,9 @@
package org.apache.tajo.master;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.master.container.TajoContainer;
import java.util.Collection;
@@ -29,7 +29,8 @@ public class LaunchTaskRunnersEvent extends TaskRunnerGroupEvent {
private final String planJson;
public LaunchTaskRunnersEvent(ExecutionBlockId executionBlockId,
- Collection<Container> containers, QueryContext queryContext, String planJson) {
+ Collection<TajoContainer> containers, QueryContext queryContext,
+ String planJson) {
super(EventType.CONTAINER_REMOTE_LAUNCH, executionBlockId, containers);
this.queryContext = queryContext;
this.planJson = planJson;
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
index f7953e0..b2883cc 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/LazyTaskScheduler.java
@@ -21,7 +21,6 @@ package org.apache.tajo.master;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryIdFactory;
@@ -38,6 +37,7 @@ import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
import org.apache.tajo.master.querymaster.QueryUnit;
import org.apache.tajo.master.querymaster.QueryUnitAttempt;
import org.apache.tajo.master.querymaster.SubQuery;
+import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.NetUtils;
import org.apache.tajo.worker.FetchImpl;
@@ -246,7 +246,8 @@ public class LazyTaskScheduler extends AbstractTaskScheduler {
}
private static class DiskBalancer {
- private HashMap<ContainerId, Integer> containerDiskMap = new HashMap<ContainerId, Integer>();
+ private HashMap<TajoContainerId, Integer> containerDiskMap = new HashMap<TajoContainerId,
+ Integer>();
private HashMap<Integer, Integer> diskReferMap = new HashMap<Integer, Integer>();
private String host;
@@ -260,7 +261,7 @@ public class LazyTaskScheduler extends AbstractTaskScheduler {
}
}
- public Integer getDiskId(ContainerId containerId) {
+ public Integer getDiskId(TajoContainerId containerId) {
if (!containerDiskMap.containsKey(containerId)) {
assignVolumeId(containerId);
}
@@ -268,7 +269,7 @@ public class LazyTaskScheduler extends AbstractTaskScheduler {
return containerDiskMap.get(containerId);
}
- public void assignVolumeId(ContainerId containerId){
+ public void assignVolumeId(TajoContainerId containerId){
Map.Entry<Integer, Integer> volumeEntry = null;
for (Map.Entry<Integer, Integer> entry : diskReferMap.entrySet()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
index c236c20..158316e 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoContainerProxy.java
@@ -20,17 +20,17 @@ package org.apache.tajo.master;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
+import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.querymaster.QueryMasterTask;
+import org.apache.tajo.master.container.TajoContainer;
+import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.rm.TajoWorkerContainer;
import org.apache.tajo.master.rm.TajoWorkerContainerId;
import org.apache.tajo.rpc.NettyClientBase;
@@ -47,7 +47,7 @@ public class TajoContainerProxy extends ContainerProxy {
private final String planJson;
public TajoContainerProxy(QueryMasterTask.QueryMasterTaskContext context,
- Configuration conf, Container container,
+ Configuration conf, TajoContainer container,
QueryContext queryContext, ExecutionBlockId executionBlockId, String planJson) {
super(context, conf, executionBlockId, container);
this.queryContext = queryContext;
@@ -89,7 +89,7 @@ public class TajoContainerProxy extends ContainerProxy {
}
}
- private void assignExecutionBlock(ExecutionBlockId executionBlockId, Container container) {
+ private void assignExecutionBlock(ExecutionBlockId executionBlockId, TajoContainer container) {
NettyClientBase tajoWorkerRpc = null;
try {
InetSocketAddress myAddr= context.getQueryMasterContext().getWorkerContext()
@@ -149,8 +149,8 @@ public class TajoContainerProxy extends ContainerProxy {
public static void releaseWorkerResource(QueryMasterTask.QueryMasterTaskContext context,
ExecutionBlockId executionBlockId,
- ContainerId containerId) throws Exception {
- List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ TajoContainerId containerId) throws Exception {
+ List<TajoContainerId> containerIds = new ArrayList<TajoContainerId>();
containerIds.add(containerId);
releaseWorkerResource(context, executionBlockId, containerIds);
@@ -158,11 +158,11 @@ public class TajoContainerProxy extends ContainerProxy {
public static void releaseWorkerResource(QueryMasterTask.QueryMasterTaskContext context,
ExecutionBlockId executionBlockId,
- List<ContainerId> containerIds) throws Exception {
- List<YarnProtos.ContainerIdProto> containerIdProtos =
- new ArrayList<YarnProtos.ContainerIdProto>();
+ List<TajoContainerId> containerIds) throws Exception {
+ List<ContainerProtocol.TajoContainerIdProto> containerIdProtos =
+ new ArrayList<ContainerProtocol.TajoContainerIdProto>();
- for(ContainerId eachContainerId: containerIds) {
+ for(TajoContainerId eachContainerId: containerIds) {
containerIdProtos.add(TajoWorkerContainerId.getContainerIdProto(eachContainerId));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterService.java
index ddf24d3..1e3501c 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterService.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.tajo.QueryId;
import org.apache.tajo.TajoIdProtos;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.ipc.TajoMasterProtocol;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
import org.apache.tajo.master.querymaster.QueryJobManager;
@@ -128,9 +129,9 @@ public class TajoMasterService extends AbstractService {
public void releaseWorkerResource(RpcController controller,
TajoMasterProtocol.WorkerResourceReleaseRequest request,
RpcCallback<PrimitiveProtos.BoolProto> done) {
- List<YarnProtos.ContainerIdProto> containerIds = request.getContainerIdsList();
+ List<ContainerProtocol.TajoContainerIdProto> containerIds = request.getContainerIdsList();
- for(YarnProtos.ContainerIdProto eachContainer: containerIds) {
+ for(ContainerProtocol.TajoContainerIdProto eachContainer: containerIds) {
context.getResourceManager().releaseWorkerResource(eachContainer);
}
done.run(BOOL_TRUE);
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java
index 1e6655c..c1c6522 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TaskRunnerGroupEvent.java
@@ -18,10 +18,10 @@
package org.apache.tajo.master;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.master.TaskRunnerGroupEvent.EventType;
+import org.apache.tajo.master.container.TajoContainer;
import java.util.Collection;
@@ -32,16 +32,16 @@ public class TaskRunnerGroupEvent extends AbstractEvent<EventType> {
}
protected final ExecutionBlockId executionBlockId;
- protected final Collection<Container> containers;
+ protected final Collection<TajoContainer> containers;
public TaskRunnerGroupEvent(EventType eventType,
ExecutionBlockId executionBlockId,
- Collection<Container> containers) {
+ Collection<TajoContainer> containers) {
super(eventType);
this.executionBlockId = executionBlockId;
this.containers = containers;
}
- public Collection<Container> getContainers() {
+ public Collection<TajoContainer> getContainers() {
return containers;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java
new file mode 100644
index 0000000..77562b5
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainer.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.container;
+
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
+ *
+ * <p><code>TajoContainer</code> represents an allocated resource in the cluster.
+ * </p>
+ *
+ * <p>The <code>ResourceManager</code> is the sole authority to allocate any
+ * <code>TajoContainer</code> to applications. The allocated <code>TajoContainer</code>
+ * is always on a single node and has a unique {@link org.apache.tajo.master.container.TajoContainerId}. It has
+ * a specific amount of {@link org.apache.hadoop.yarn.api.records.Resource} allocated.</p>
+ *
+ * <p>It includes details such as:
+ * <ul>
+ * <li>{@link org.apache.tajo.master.container.TajoContainerId} for the container, which is globally unique.</li>
+ * <li>
+ * {@link org.apache.hadoop.yarn.api.records.NodeId} of the node on which it is allocated.
+ * </li>
+ * <li>HTTP uri of the node.</li>
+ * <li>{@link org.apache.hadoop.yarn.api.records.Resource} allocated to the container.</li>
+ * <li>{@link org.apache.hadoop.yarn.api.records.Priority} at which the container was allocated.</li>
+ * <li>
+ * TajoContainer {@link org.apache.hadoop.yarn.api.records.Token} of the container, used to securely verify
+ * authenticity of the allocation.
+ * </li>
+ * </ul>
+ * </p>
+ *
+ * <p>Typically, an <code>ApplicationMaster</code> receives the
+ * <code>TajoContainer</code> from the <code>ResourceManager</code> during
+ * resource-negotiation and then talks to the <code>NodeManager</code> to
+ * start/stop containers.</p>
+ *
+ * @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ * @see org.apache.hadoop.yarn.api.ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+ * @see org.apache.hadoop.yarn.api.ContainerManagementProtocol#stopContainers(org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest)
+ */
+@Public
+@Stable
+public abstract class TajoContainer implements Comparable<TajoContainer> {
+
+ @Private
+ @Unstable
+ public static TajoContainer newInstance(TajoContainerId containerId, NodeId nodeId,
+ String nodeHttpAddress, Resource resource, Priority priority,
+ Token containerToken) {
+ TajoContainer container = Records.newRecord(TajoContainer.class);
+ container.setId(containerId);
+ container.setNodeId(nodeId);
+ container.setNodeHttpAddress(nodeHttpAddress);
+ container.setResource(resource);
+ container.setPriority(priority);
+ container.setContainerToken(containerToken);
+ return container;
+ }
+
+ /**
+ * Get the globally unique identifier for the container.
+ * @return globally unique identifier for the container
+ */
+ @Public
+ @Stable
+ public abstract TajoContainerId getId();
+
+ @Private
+ @Unstable
+ public abstract void setId(TajoContainerId id);
+
+ /**
+ * Get the identifier of the node on which the container is allocated.
+ * @return identifier of the node on which the container is allocated
+ */
+ @Public
+ @Stable
+ public abstract NodeId getNodeId();
+
+ @Private
+ @Unstable
+ public abstract void setNodeId(NodeId nodeId);
+
+ /**
+ * Get the http uri of the node on which the container is allocated.
+ * @return http uri of the node on which the container is allocated
+ */
+ @Public
+ @Stable
+ public abstract String getNodeHttpAddress();
+
+ @Private
+ @Unstable
+ public abstract void setNodeHttpAddress(String nodeHttpAddress);
+
+ /**
+ * Get the <code>Resource</code> allocated to the container.
+ * @return <code>Resource</code> allocated to the container
+ */
+ @Public
+ @Stable
+ public abstract Resource getResource();
+
+ @Private
+ @Unstable
+ public abstract void setResource(Resource resource);
+
+ /**
+ * Get the <code>Priority</code> at which the <code>TajoContainer</code> was
+ * allocated.
+ * @return <code>Priority</code> at which the <code>TajoContainer</code> was
+ * allocated
+ */
+ @Public
+ @Stable
+ public abstract Priority getPriority();
+
+ @Private
+ @Unstable
+ public abstract void setPriority(Priority priority);
+
+ /**
+ * Get the <code>TajoContainerToken</code> for the container.
+ * <p><code>TajoContainerToken</code> is the security token used by the framework
+ * to verify authenticity of any <code>TajoContainer</code>.</p>
+ *
+ * <p>The <code>ResourceManager</code>, on container allocation provides a
+ * secure token which is verified by the <code>NodeManager</code> on
+ * container launch.</p>
+ *
+ * <p>Applications do not need to care about <code>TajoContainerToken</code>, they
+ * are transparently handled by the framework - the allocated
+ * <code>TajoContainer</code> includes the <code>TajoContainerToken</code>.</p>
+ *
+ * @see ApplicationMasterProtocol#allocate(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest)
+ * @see org.apache.hadoop.yarn.api.ContainerManagementProtocol#startContainers(org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest)
+ *
+ * @return <code>TajoContainerToken</code> for the container
+ */
+ @Public
+ @Stable
+ public abstract Token getContainerToken();
+
+ @Private
+ @Unstable
+ public abstract void setContainerToken(Token containerToken);
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
new file mode 100644
index 0000000..0de5fe0
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoContainerId.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.container;
+
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/api/records/ContainerId.java
+ *
+ * <p><code>TajoContainerId</code> represents a globally unique identifier
+ * for a {@link org.apache.tajo.master.container.TajoContainer} in the cluster.</p>
+ */
+@Public
+@Stable
+public abstract class TajoContainerId implements Comparable<TajoContainerId>{
+
+ @Private
+ @Unstable
+ public static TajoContainerId newInstance(ApplicationAttemptId appAttemptId,
+ int containerId) {
+ TajoContainerId id = TajoRecords.newRecord(TajoContainerId.class);
+ id.setId(containerId);
+ id.setApplicationAttemptId(appAttemptId);
+ id.build();
+ return id;
+ }
+
+ /**
+ * Get the <code>ApplicationAttemptId</code> of the application to which the
+ * <code>Container</code> was assigned.
+ * <p>
+ * Note: If containers are kept alive across application attempts via
+ * {@link org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setKeepContainersAcrossApplicationAttempts(boolean)}
+ * the <code>TajoContainerId</code> does not necessarily contain the current
+ * running application attempt's <code>ApplicationAttemptId</code> This
+ * container can be allocated by previously exited application attempt and
+ * managed by the current running attempt thus have the previous application
+ * attempt's <code>ApplicationAttemptId</code>.
+ * </p>
+ *
+ * @return <code>ApplicationAttemptId</code> of the application to which the
+ * <code>Container</code> was assigned
+ */
+ @Public
+ @Stable
+ public abstract ApplicationAttemptId getApplicationAttemptId();
+
+ @Private
+ @Unstable
+ protected abstract void setApplicationAttemptId(ApplicationAttemptId atId);
+
+ /**
+ * Get the identifier of the <code>TajoContainerId</code>.
+ * @return identifier of the <code>TajoContainerId</code>
+ */
+ @Public
+ @Stable
+ public abstract int getId();
+
+ @Private
+ @Unstable
+ protected abstract void setId(int id);
+
+
+ // TODO: fail the app submission if attempts are more than 10 or something
+ private static final ThreadLocal<NumberFormat> appAttemptIdFormat =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(2);
+ return fmt;
+ }
+ };
+ // TODO: Why thread local?
+ // ^ NumberFormat instances are not threadsafe
+ private static final ThreadLocal<NumberFormat> containerIdFormat =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(6);
+ return fmt;
+ }
+ };
+
+ @Override
+ public int hashCode() {
+ // Generated by eclipse.
+ final int prime = 435569;
+ int result = 7507;
+ result = prime * result + getId();
+ result = prime * result + getApplicationAttemptId().hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ TajoContainerId other = (TajoContainerId) obj;
+ if (!this.getApplicationAttemptId().equals(other.getApplicationAttemptId()))
+ return false;
+ if (this.getId() != other.getId())
+ return false;
+ return true;
+ }
+
+ @Override
+ public int compareTo(TajoContainerId other) {
+ if (this.getApplicationAttemptId().compareTo(
+ other.getApplicationAttemptId()) == 0) {
+ return this.getId() - other.getId();
+ } else {
+ return this.getApplicationAttemptId().compareTo(
+ other.getApplicationAttemptId());
+ }
+
+ }
+
+ @Override
+ public String toString() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(4);
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("container_");
+ ApplicationId appId = getApplicationAttemptId().getApplicationId();
+ sb.append(appId.getClusterTimestamp()).append("_");
+ sb.append(fmt.format(appId.getId()))
+ .append("_");
+ sb.append(
+ appAttemptIdFormat.get().format(
+ getApplicationAttemptId().getAttemptId())).append("_");
+ sb.append(containerIdFormat.get().format(getId()));
+ return sb.toString();
+ }
+
+ protected abstract void build();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
new file mode 100644
index 0000000..a6db654
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoConverterUtils.java
@@ -0,0 +1,263 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tajo.master.container;
+
+
+import static org.apache.hadoop.yarn.util.StringHelper._split;
+
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
+ *
+ * This class contains a set of utilities which help converting data structures
+ * from/to 'serializableFormat' to/from hadoop/nativejava data structures.
+ *
+ */
+@Private
+public class TajoConverterUtils {
+
+ public static final String APPLICATION_PREFIX = "application";
+ public static final String CONTAINER_PREFIX = "container";
+ public static final String APPLICATION_ATTEMPT_PREFIX = "appattempt";
+
+ /**
+ * return a hadoop path from a given url
+ *
+ * @param url
+ * url to convert
+ * @return path from {@link URL}
+ * @throws URISyntaxException
+ */
+ public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
+ String scheme = url.getScheme() == null ? "" : url.getScheme();
+
+ String authority = "";
+ if (url.getHost() != null) {
+ authority = url.getHost();
+ if (url.getUserInfo() != null) {
+ authority = url.getUserInfo() + "@" + authority;
+ }
+ if (url.getPort() > 0) {
+ authority += ":" + url.getPort();
+ }
+ }
+
+ return new Path(
+ (new URI(scheme, authority, url.getFile(), null, null)).normalize());
+ }
+
+ /**
+ * change from CharSequence to string for map key and value
+ * @param env map for converting
+ * @return string,string map
+ */
+ public static Map<String, String> convertToString(
+ Map<CharSequence, CharSequence> env) {
+
+ Map<String, String> stringMap = new HashMap<String, String>();
+ for (Entry<CharSequence, CharSequence> entry: env.entrySet()) {
+ stringMap.put(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return stringMap;
+ }
+
+ public static URL getYarnUrlFromPath(Path path) {
+ return getYarnUrlFromURI(path.toUri());
+ }
+
+ public static URL getYarnUrlFromURI(URI uri) {
+ URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
+ if (uri.getHost() != null) {
+ url.setHost(uri.getHost());
+ }
+ if (uri.getUserInfo() != null) {
+ url.setUserInfo(uri.getUserInfo());
+ }
+ url.setPort(uri.getPort());
+ url.setScheme(uri.getScheme());
+ url.setFile(uri.getPath());
+ return url;
+ }
+
+ public static String toString(ApplicationId appId) {
+ return appId.toString();
+ }
+
+ public static ApplicationId toApplicationId(RecordFactory recordFactory,
+ String appIdStr) {
+ Iterator<String> it = _split(appIdStr).iterator();
+ it.next(); // prefix. TODO: Validate application prefix
+ return toApplicationId(recordFactory, it);
+ }
+
+ private static ApplicationId toApplicationId(RecordFactory recordFactory,
+ Iterator<String> it) {
+ ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+ Integer.parseInt(it.next()));
+ return appId;
+ }
+
+ private static ApplicationAttemptId toApplicationAttemptId(
+ Iterator<String> it) throws NumberFormatException {
+ ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+ Integer.parseInt(it.next()));
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, Integer.parseInt(it.next()));
+ return appAttemptId;
+ }
+
+ private static ApplicationId toApplicationId(
+ Iterator<String> it) throws NumberFormatException {
+ ApplicationId appId = ApplicationId.newInstance(Long.parseLong(it.next()),
+ Integer.parseInt(it.next()));
+ return appId;
+ }
+
+ public static String toString(TajoContainerId cId) {
+ return cId == null ? null : cId.toString();
+ }
+
+ public static NodeId toNodeId(String nodeIdStr) {
+ String[] parts = nodeIdStr.split(":");
+ if (parts.length != 2) {
+ throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
+ + "]. Expected host:port");
+ }
+ try {
+ NodeId nodeId =
+ NodeId.newInstance(parts[0], Integer.parseInt(parts[1]));
+ return nodeId;
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Invalid port: " + parts[1], e);
+ }
+ }
+
+ public static TajoContainerId toTajoContainerId(String containerIdStr) {
+ Iterator<String> it = _split(containerIdStr).iterator();
+ if (!it.next().equals(CONTAINER_PREFIX)) {
+ throw new IllegalArgumentException("Invalid TajoContainerId prefix: "
+ + containerIdStr);
+ }
+ try {
+ ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
+ TajoContainerId containerId =
+ TajoContainerId.newInstance(appAttemptID, Integer.parseInt(it.next()));
+ return containerId;
+ } catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid TajoContainerId: "
+ + containerIdStr, n);
+ }
+ }
+
+ public static ApplicationAttemptId toApplicationAttemptId(
+ String applicationAttmeptIdStr) {
+ Iterator<String> it = _split(applicationAttmeptIdStr).iterator();
+ if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) {
+ throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
+ + applicationAttmeptIdStr);
+ }
+ try {
+ return toApplicationAttemptId(it);
+ } catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid AppAttemptId: "
+ + applicationAttmeptIdStr, n);
+ }
+ }
+
+ public static ApplicationId toApplicationId(
+ String appIdStr) {
+ Iterator<String> it = _split(appIdStr).iterator();
+ if (!it.next().equals(APPLICATION_PREFIX)) {
+ throw new IllegalArgumentException("Invalid ApplicationId prefix: "
+ + appIdStr + ". The valid ApplicationId should start with prefix "
+ + APPLICATION_PREFIX);
+ }
+ try {
+ return toApplicationId(it);
+ } catch (NumberFormatException n) {
+ throw new IllegalArgumentException("Invalid AppAttemptId: "
+ + appIdStr, n);
+ }
+ }
+
+ /**
+ * Convert a protobuf token into a rpc token and set its service. Supposed
+ * to be used for tokens other than RMDelegationToken. For
+ * RMDelegationToken, use
+ * {@link #convertFromYarn(org.apache.hadoop.yarn.api.records.Token,
+ * org.apache.hadoop.io.Text)} instead.
+ *
+ * @param protoToken the yarn token
+ * @param serviceAddr the connect address for the service
+ * @return rpc token
+ */
+ public static <T extends TokenIdentifier> Token<T> convertFromYarn(
+ org.apache.hadoop.yarn.api.records.Token protoToken,
+ InetSocketAddress serviceAddr) {
+ Token<T> token = new Token<T>(protoToken.getIdentifier().array(),
+ protoToken.getPassword().array(),
+ new Text(protoToken.getKind()),
+ new Text(protoToken.getService()));
+ if (serviceAddr != null) {
+ SecurityUtil.setTokenService(token, serviceAddr);
+ }
+ return token;
+ }
+
+ /**
+ * Convert a protobuf token into a rpc token and set its service.
+ *
+ * @param protoToken the yarn token
+ * @param service the service for the token
+ */
+ public static <T extends TokenIdentifier> Token<T> convertFromYarn(
+ org.apache.hadoop.yarn.api.records.Token protoToken,
+ Text service) {
+ Token<T> token = new Token<T>(protoToken.getIdentifier().array(),
+ protoToken.getPassword().array(),
+ new Text(protoToken.getKind()),
+ new Text(protoToken.getService()));
+
+ if (service != null) {
+ token.setService(service);
+ }
+ return token;
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java
new file mode 100644
index 0000000..2fd8697
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactory.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.container;
+
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/factories/RecordFactory.java
+ *
+ */
+@Unstable
+public interface TajoRecordFactory {
+ public <T> T newRecordInstance(Class<T> clazz);
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java
new file mode 100644
index 0000000..c352a28
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryPBImpl.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.tajo.master.container;
+
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.tajo.master.container.TajoRecordFactory;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/RecordFactoryPBImpl.java
+ */
+@Private
+public class TajoRecordFactoryPBImpl implements TajoRecordFactory {
+
+ private static final String PB_IMPL_PACKAGE_SUFFIX = "impl.pb";
+ private static final String PB_IMPL_CLASS_SUFFIX = "PBImpl";
+
+ private static final TajoRecordFactoryPBImpl self = new TajoRecordFactoryPBImpl();
+ private Configuration localConf = new Configuration();
+ private ConcurrentMap<Class<?>, Constructor<?>> cache = new ConcurrentHashMap<Class<?>, Constructor<?>>();
+
+ private TajoRecordFactoryPBImpl() {
+ }
+
+ public static TajoRecordFactory get() {
+ return self;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T newRecordInstance(Class<T> clazz) {
+
+ Constructor<?> constructor = cache.get(clazz);
+ if (constructor == null) {
+ Class<?> pbClazz = null;
+ try {
+ pbClazz = localConf.getClassByName(getPBImplClassName(clazz));
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException("Failed to load class: ["
+ + getPBImplClassName(clazz) + "]", e);
+ }
+ try {
+ constructor = pbClazz.getConstructor();
+ constructor.setAccessible(true);
+ cache.putIfAbsent(clazz, constructor);
+ } catch (NoSuchMethodException e) {
+ throw new YarnRuntimeException("Could not find 0 argument constructor", e);
+ }
+ }
+ try {
+ Object retObject = constructor.newInstance();
+ return (T)retObject;
+ } catch (InvocationTargetException e) {
+ throw new YarnRuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new YarnRuntimeException(e);
+ } catch (InstantiationException e) {
+ throw new YarnRuntimeException(e);
+ }
+ }
+
+ private String getPBImplClassName(Class<?> clazz) {
+ String srcPackagePart = getPackageName(clazz);
+ String srcClassName = getClassName(clazz);
+ String destPackagePart = srcPackagePart + "." + PB_IMPL_PACKAGE_SUFFIX;
+ String destClassPart = srcClassName + PB_IMPL_CLASS_SUFFIX;
+ return destPackagePart + "." + destClassPart;
+ }
+
+ private String getClassName(Class<?> clazz) {
+ String fqName = clazz.getName();
+ return (fqName.substring(fqName.lastIndexOf(".") + 1, fqName.length()));
+ }
+
+ private String getPackageName(Class<?> clazz) {
+ return clazz.getPackage().getName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java
new file mode 100644
index 0000000..c260e85
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecordFactoryProvider.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.container;
+
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/factory/providers/RecordFactoryProvider.java
+ */
+@Unstable
+public class TajoRecordFactoryProvider {
+ private static Configuration defaultConf;
+
+ static {
+ defaultConf = new Configuration();
+ }
+
+ private TajoRecordFactoryProvider() {
+ }
+
+ public static TajoRecordFactory getRecordFactory(Configuration conf) {
+ if (conf == null) {
+ //Assuming the default configuration has the correct factories set.
+ //Users can specify a particular factory by providing a configuration.
+ conf = defaultConf;
+ }
+ return (TajoRecordFactory) getFactoryClassInstance(TajoRecordFactoryPBImpl.class.getCanonicalName());
+ }
+
+ private static Object getFactoryClassInstance(String factoryClassName) {
+ try {
+ Class<?> clazz = Class.forName(factoryClassName);
+ Method method = clazz.getMethod("get", null);
+ method.setAccessible(true);
+ return method.invoke(null, null);
+ } catch (ClassNotFoundException e) {
+ throw new YarnRuntimeException(e);
+ } catch (NoSuchMethodException e) {
+ throw new YarnRuntimeException(e);
+ } catch (InvocationTargetException e) {
+ throw new YarnRuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new YarnRuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java
new file mode 100644
index 0000000..e85edf9
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/TajoRecords.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.container;
+
+
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-api}/src/main/java/org/apache/hadoop/yarn/util/Records.java
+ *
+ * Convenient API record utils
+ */
+@Unstable
+public class TajoRecords {
+ // The default record factory
+ private static final TajoRecordFactory factory =
+ TajoRecordFactoryProvider.getRecordFactory(null);
+
+ public static <T> T newRecord(Class<T> cls) {
+ return factory.newRecordInstance(cls);
+ }
+}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java b/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java
new file mode 100644
index 0000000..9d31050
--- /dev/null
+++ b/tajo-core/src/main/java/org/apache/tajo/master/container/impl/pb/TajoContainerIdPBImpl.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.master.container.impl.pb;
+
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+
+import com.google.common.base.Preconditions;
+import org.apache.tajo.ipc.ContainerProtocol;
+import org.apache.tajo.master.container.TajoContainerId;
+
+/**
+ * This class is borrowed from the following source code :
+ * ${hadoop-yarn-common}/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerIdPBImpl.java
+ *
+ */
+@Private
+@Unstable
+public class TajoContainerIdPBImpl extends TajoContainerId {
+ ContainerProtocol.TajoContainerIdProto proto = null;
+ ContainerProtocol.TajoContainerIdProto.Builder builder = null;
+ private ApplicationAttemptId applicationAttemptId = null;
+
+ public TajoContainerIdPBImpl() {
+ builder = ContainerProtocol.TajoContainerIdProto.newBuilder();
+ }
+
+ public TajoContainerIdPBImpl(ContainerProtocol.TajoContainerIdProto proto) {
+ this.proto = proto;
+ this.applicationAttemptId = convertFromProtoFormat(proto.getAppAttemptId());
+ }
+
+ public ContainerProtocol.TajoContainerIdProto getProto() {
+ return proto;
+ }
+
+ @Override
+ public int getId() {
+ Preconditions.checkNotNull(proto);
+ return proto.getId();
+ }
+
+ @Override
+ protected void setId(int id) {
+ Preconditions.checkNotNull(builder);
+ builder.setId((id));
+ }
+
+
+ @Override
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return this.applicationAttemptId;
+ }
+
+ @Override
+ protected void setApplicationAttemptId(ApplicationAttemptId atId) {
+ if (atId != null) {
+ Preconditions.checkNotNull(builder);
+ builder.setAppAttemptId(convertToProtoFormat(atId));
+ }
+ this.applicationAttemptId = atId;
+ }
+
+ private ApplicationAttemptIdPBImpl convertFromProtoFormat(
+ ApplicationAttemptIdProto p) {
+ return new ApplicationAttemptIdPBImpl(p);
+ }
+
+ private ApplicationAttemptIdProto convertToProtoFormat(
+ ApplicationAttemptId t) {
+ return ((ApplicationAttemptIdPBImpl)t).getProto();
+ }
+
+ @Override
+ protected void build() {
+ proto = builder.build();
+ builder = null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
index 92e6695..cab2202 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/LocalTaskEvent.java
@@ -18,18 +18,19 @@
package org.apache.tajo.master.event;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tajo.QueryUnitAttemptId;
+import org.apache.tajo.master.container.TajoContainerId;
/**
* This event is sent to a running TaskAttempt on a worker.
*/
public class LocalTaskEvent extends AbstractEvent<LocalTaskEventType> {
private final QueryUnitAttemptId taskAttemptId;
- private final ContainerId containerId;
+ private final TajoContainerId containerId;
- public LocalTaskEvent(QueryUnitAttemptId taskAttemptId, ContainerId containerId, LocalTaskEventType eventType) {
+ public LocalTaskEvent(QueryUnitAttemptId taskAttemptId, TajoContainerId containerId,
+ LocalTaskEventType eventType) {
super(eventType);
this.taskAttemptId = taskAttemptId;
this.containerId = containerId;
@@ -39,7 +40,7 @@ public class LocalTaskEvent extends AbstractEvent<LocalTaskEventType> {
return taskAttemptId;
}
- public ContainerId getContainerId() {
+ public TajoContainerId getContainerId() {
return containerId;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java
index a2acc7e..6e0d9fd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/QueryUnitAttemptScheduleEvent.java
@@ -19,10 +19,10 @@
package org.apache.tajo.master.event;
import com.google.protobuf.RpcCallback;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.ipc.TajoWorkerProtocol.QueryUnitRequestProto;
import org.apache.tajo.master.querymaster.QueryUnitAttempt;
+import org.apache.tajo.master.container.TajoContainerId;
public class QueryUnitAttemptScheduleEvent extends TaskSchedulerEvent {
private final QueryUnitAttemptScheduleContext context;
@@ -44,7 +44,7 @@ public class QueryUnitAttemptScheduleEvent extends TaskSchedulerEvent {
}
public static class QueryUnitAttemptScheduleContext {
- private ContainerId containerId;
+ private TajoContainerId containerId;
private String host;
private RpcCallback<QueryUnitRequestProto> callback;
@@ -52,7 +52,7 @@ public class QueryUnitAttemptScheduleEvent extends TaskSchedulerEvent {
}
- public QueryUnitAttemptScheduleContext(ContainerId containerId,
+ public QueryUnitAttemptScheduleContext(TajoContainerId containerId,
String host,
RpcCallback<QueryUnitRequestProto> callback) {
this.containerId = containerId;
@@ -60,11 +60,11 @@ public class QueryUnitAttemptScheduleEvent extends TaskSchedulerEvent {
this.callback = callback;
}
- public ContainerId getContainerId() {
+ public TajoContainerId getContainerId() {
return containerId;
}
- public void setContainerId(ContainerId containerId) {
+ public void setContainerId(TajoContainerId containerId) {
this.containerId = containerId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryContainerAllocationEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryContainerAllocationEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryContainerAllocationEvent.java
index a8f4800..e617d53 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryContainerAllocationEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/SubQueryContainerAllocationEvent.java
@@ -18,21 +18,21 @@
package org.apache.tajo.master.event;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.tajo.ExecutionBlockId;
+import org.apache.tajo.master.container.TajoContainer;
import java.util.List;
public class SubQueryContainerAllocationEvent extends SubQueryEvent {
- private List<Container> allocatedContainer;
+ private List<TajoContainer> allocatedContainer;
public SubQueryContainerAllocationEvent(final ExecutionBlockId id,
- List<Container> allocatedContainer) {
+ List<TajoContainer> allocatedContainer) {
super(id, SubQueryEventType.SQ_CONTAINER_ALLOCATED);
this.allocatedContainer = allocatedContainer;
}
- public List<Container> getAllocatedContainer() {
+ public List<TajoContainer> getAllocatedContainer() {
return this.allocatedContainer;
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
index e0928c5..3b9edcb 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskAttemptAssignedEvent.java
@@ -18,22 +18,22 @@
package org.apache.tajo.master.event;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.master.cluster.WorkerConnectionInfo;
+import org.apache.tajo.master.container.TajoContainerId;
public class TaskAttemptAssignedEvent extends TaskAttemptEvent {
- private final ContainerId cId;
+ private final TajoContainerId cId;
private final WorkerConnectionInfo workerConnectionInfo;
- public TaskAttemptAssignedEvent(QueryUnitAttemptId id, ContainerId cId,
+ public TaskAttemptAssignedEvent(QueryUnitAttemptId id, TajoContainerId cId,
WorkerConnectionInfo connectionInfo) {
super(id, TaskAttemptEventType.TA_ASSIGNED);
this.cId = cId;
this.workerConnectionInfo = connectionInfo;
}
- public ContainerId getContainerId() {
+ public TajoContainerId getContainerId() {
return cId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
index 2197c33..9e8e3dd 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/event/TaskRequestEvent.java
@@ -19,11 +19,11 @@
package org.apache.tajo.master.event;
import com.google.protobuf.RpcCallback;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.tajo.ExecutionBlockId;
import org.apache.tajo.ipc.TajoWorkerProtocol.QueryUnitRequestProto;
import org.apache.tajo.master.event.TaskRequestEvent.TaskRequestEventType;
+import org.apache.tajo.master.container.TajoContainerId;
public class TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {
@@ -32,13 +32,13 @@ public class TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {
}
private final int workerId;
- private final ContainerId containerId;
+ private final TajoContainerId containerId;
private final ExecutionBlockId executionBlockId;
private final RpcCallback<QueryUnitRequestProto> callback;
public TaskRequestEvent(int workerId,
- ContainerId containerId,
+ TajoContainerId containerId,
ExecutionBlockId executionBlockId,
RpcCallback<QueryUnitRequestProto> callback) {
super(TaskRequestEventType.TASK_REQ);
@@ -52,7 +52,7 @@ public class TaskRequestEvent extends AbstractEvent<TaskRequestEventType> {
return this.workerId;
}
- public ContainerId getContainerId() {
+ public TajoContainerId getContainerId() {
return this.containerId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
index d949ca4..e361c7f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.tajo.QueryId;
import org.apache.tajo.TajoProtos;
import org.apache.tajo.conf.TajoConf;
+import org.apache.tajo.ipc.ContainerProtocol;
import org.apache.tajo.plan.logical.LogicalRootNode;
import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.ipc.QueryMasterProtocol;
@@ -74,7 +75,7 @@ public class QueryInProgress extends CompositeService {
private QueryMasterProtocolService queryMasterRpcClient;
- private YarnProtos.ContainerIdProto qmContainerId;
+ private ContainerProtocol.TajoContainerIdProto qmContainerId;
public QueryInProgress(
TajoMaster.MasterContext masterContext,
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
index f953995..f4bd8a3 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterManagerService.java
@@ -25,7 +25,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.tajo.*;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.engine.query.QueryContext;
@@ -33,6 +32,7 @@ import org.apache.tajo.ipc.QueryMasterProtocol;
import org.apache.tajo.ipc.TajoWorkerProtocol;
import org.apache.tajo.master.LazyTaskScheduler;
import org.apache.tajo.master.event.*;
+import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.master.session.Session;
import org.apache.tajo.rpc.AsyncRpcServer;
import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos;
@@ -130,7 +130,7 @@ public class QueryMasterManagerService extends CompositeService
if(queryMasterTask == null || queryMasterTask.isStopped()) {
done.run(LazyTaskScheduler.stopTaskRunnerReq);
} else {
- ContainerId cid =
+ TajoContainerId cid =
queryMasterTask.getQueryTaskContext().getResourceAllocator().makeContainerId(request.getContainerId());
LOG.debug("getTask:" + cid + ", ebId:" + ebId);
queryMasterTask.handleTaskRequestEvent(new TaskRequestEvent(request.getWorkerId(), cid, ebId, done));
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
index db6f130..d88173f 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnitAttempt.java
@@ -21,7 +21,6 @@ package org.apache.tajo.master.querymaster;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.state.*;
import org.apache.tajo.QueryUnitAttemptId;
@@ -35,6 +34,7 @@ import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttem
import org.apache.tajo.master.event.TaskSchedulerEvent.EventType;
import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry;
import org.apache.tajo.master.querymaster.QueryUnit.PullHost;
+import org.apache.tajo.master.container.TajoContainerId;
import java.util.ArrayList;
import java.util.EnumSet;
@@ -55,7 +55,7 @@ public class QueryUnitAttempt implements EventHandler<TaskAttemptEvent> {
private final QueryUnit queryUnit;
final EventHandler eventHandler;
- private ContainerId containerId;
+ private TajoContainerId containerId;
private WorkerConnectionInfo workerConnectionInfo;
private int expire;
@@ -214,7 +214,7 @@ public class QueryUnitAttempt implements EventHandler<TaskAttemptEvent> {
return this.workerConnectionInfo;
}
- public void setContainerId(ContainerId containerId) {
+ public void setContainerId(TajoContainerId containerId) {
this.containerId = containerId;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index 476da04..39bb7ed 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -24,7 +24,6 @@ import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -58,6 +57,8 @@ import org.apache.tajo.master.TaskRunnerGroupEvent.EventType;
import org.apache.tajo.master.event.*;
import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext;
import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry;
+import org.apache.tajo.master.container.TajoContainer;
+import org.apache.tajo.master.container.TajoContainerId;
import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.logical.*;
import org.apache.tajo.storage.StorageManager;
@@ -105,7 +106,8 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
private long finishTime;
volatile Map<QueryUnitId, QueryUnit> tasks = new ConcurrentHashMap<QueryUnitId, QueryUnit>();
- volatile Map<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>();
+ volatile Map<TajoContainerId, TajoContainer> containers = new ConcurrentHashMap<TajoContainerId,
+ TajoContainer>();
private static final DiagnosticsUpdateTransition DIAGNOSTIC_UPDATE_TRANSITION = new DiagnosticsUpdateTransition();
private static final InternalErrorTransition INTERNAL_ERROR_TRANSITION = new InternalErrorTransition();
@@ -663,13 +665,6 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
eventHandler.handle(new TaskRunnerGroupEvent(EventType.CONTAINER_REMOTE_CLEANUP, getId(), containers.values()));
}
- public void releaseContainer(ContainerId containerId) {
- // try to kill the container.
- ArrayList<Container> list = new ArrayList<Container>();
- list.add(containers.get(containerId));
- eventHandler.handle(new TaskRunnerGroupEvent(EventType.CONTAINER_REMOTE_CLEANUP, getId(), list));
- }
-
/**
* It computes all stats and sets the intermediate result.
*/
@@ -1129,8 +1124,8 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
try {
SubQueryContainerAllocationEvent allocationEvent =
(SubQueryContainerAllocationEvent) event;
- for (Container container : allocationEvent.getAllocatedContainer()) {
- ContainerId cId = container.getId();
+ for (TajoContainer container : allocationEvent.getAllocatedContainer()) {
+ TajoContainerId cId = container.getId();
if (subQuery.containers.containsKey(cId)) {
subQuery.eventHandler.handle(new SubQueryDiagnosticsUpdateEvent(subQuery.getId(),
"Duplicated containers are allocated: " + cId.toString()));
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
index 5d07ff2..bb8cc12 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoRMContext.java
@@ -21,14 +21,13 @@ package org.apache.tajo.master.rm;
import com.google.common.collect.Maps;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.tajo.QueryId;
+import org.apache.tajo.ipc.ContainerProtocol;
import java.util.Collections;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import static org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
-
/**
* It's a worker resource manager context. It contains all context data about TajoWorkerResourceManager.
*/
@@ -43,7 +42,8 @@ public class TajoRMContext {
private final ConcurrentMap<Integer, Worker> inactiveWorkers = Maps.newConcurrentMap();
/** map between queryIds and query master ContainerId */
- private final ConcurrentMap<QueryId, ContainerIdProto> qmContainerMap = Maps.newConcurrentMap();
+ private final ConcurrentMap<QueryId, ContainerProtocol.TajoContainerIdProto> qmContainerMap = Maps
+ .newConcurrentMap();
private final Set<Integer> liveQueryMasterWorkerResources =
Collections.newSetFromMap(new ConcurrentHashMap<Integer, Boolean>());
@@ -77,7 +77,7 @@ public class TajoRMContext {
*
* @return The Map for query master containers
*/
- public ConcurrentMap<QueryId, ContainerIdProto> getQueryMasterContainer() {
+ public ConcurrentMap<QueryId, ContainerProtocol.TajoContainerIdProto> getQueryMasterContainer() {
return qmContainerMap;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java
index 4d6cbd2..3d28d85 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainer.java
@@ -19,9 +19,12 @@
package org.apache.tajo.master.rm;
import org.apache.hadoop.yarn.api.records.*;
+import org.apache.tajo.master.container.TajoContainer;
+import org.apache.tajo.master.container.TajoContainerId;
-public class TajoWorkerContainer extends Container {
- ContainerId id;
+
+public class TajoWorkerContainer extends TajoContainer {
+ TajoContainerId id;
NodeId nodeId;
Worker worker;
@@ -34,12 +37,12 @@ public class TajoWorkerContainer extends Container {
}
@Override
- public ContainerId getId() {
+ public TajoContainerId getId() {
return id;
}
@Override
- public void setId(ContainerId id) {
+ public void setId(TajoContainerId id) {
this.id = id;
}
@@ -94,7 +97,7 @@ public class TajoWorkerContainer extends Container {
}
@Override
- public int compareTo(Container container) {
- return 0; //To change body of implemented methods use File | Settings | File Templates.
+ public int compareTo(TajoContainer container) {
+ return getId().compareTo(container.getId());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d485ecb/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java
index 634ad2b..184de71 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/rm/TajoWorkerContainerId.java
@@ -19,10 +19,11 @@
package org.apache.tajo.master.rm;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.proto.YarnProtos;
+import org.apache.tajo.ipc.ContainerProtocol;
+import org.apache.tajo.master.container.TajoContainerId;
-public class TajoWorkerContainerId extends ContainerId {
+public class TajoWorkerContainerId extends TajoContainerId {
ApplicationAttemptId applicationAttemptId;
int id;
@@ -46,43 +47,43 @@ public class TajoWorkerContainerId extends ContainerId {
this.id = id;
}
- public YarnProtos.ContainerIdProto getProto() {
+ public ContainerProtocol.TajoContainerIdProto getProto() {
YarnProtos.ApplicationIdProto appIdProto = YarnProtos.ApplicationIdProto.newBuilder()
- .setClusterTimestamp(applicationAttemptId.getApplicationId().getClusterTimestamp())
- .setId(applicationAttemptId.getApplicationId().getId())
- .build();
+ .setClusterTimestamp(applicationAttemptId.getApplicationId().getClusterTimestamp())
+ .setId(applicationAttemptId.getApplicationId().getId())
+ .build();
YarnProtos.ApplicationAttemptIdProto attemptIdProto = YarnProtos.ApplicationAttemptIdProto.newBuilder()
- .setAttemptId(applicationAttemptId.getAttemptId())
- .setApplicationId(appIdProto)
- .build();
+ .setAttemptId(applicationAttemptId.getAttemptId())
+ .setApplicationId(appIdProto)
+ .build();
- return YarnProtos.ContainerIdProto.newBuilder()
- .setAppAttemptId(attemptIdProto)
- .setAppId(appIdProto)
- .setId(id)
- .build();
+ return ContainerProtocol.TajoContainerIdProto.newBuilder()
+ .setAppAttemptId(attemptIdProto)
+ .setAppId(appIdProto)
+ .setId(id)
+ .build();
}
- public static YarnProtos.ContainerIdProto getContainerIdProto(ContainerId containerId) {
+ public static ContainerProtocol.TajoContainerIdProto getContainerIdProto(TajoContainerId containerId) {
if(containerId instanceof TajoWorkerContainerId) {
return ((TajoWorkerContainerId)containerId).getProto();
} else {
YarnProtos.ApplicationIdProto appIdProto = YarnProtos.ApplicationIdProto.newBuilder()
- .setClusterTimestamp(containerId.getApplicationAttemptId().getApplicationId().getClusterTimestamp())
- .setId(containerId.getApplicationAttemptId().getApplicationId().getId())
- .build();
+ .setClusterTimestamp(containerId.getApplicationAttemptId().getApplicationId().getClusterTimestamp())
+ .setId(containerId.getApplicationAttemptId().getApplicationId().getId())
+ .build();
YarnProtos.ApplicationAttemptIdProto attemptIdProto = YarnProtos.ApplicationAttemptIdProto.newBuilder()
- .setAttemptId(containerId.getApplicationAttemptId().getAttemptId())
- .setApplicationId(appIdProto)
- .build();
+ .setAttemptId(containerId.getApplicationAttemptId().getAttemptId())
+ .setApplicationId(appIdProto)
+ .build();
- return YarnProtos.ContainerIdProto.newBuilder()
- .setAppAttemptId(attemptIdProto)
- .setAppId(appIdProto)
- .setId(containerId.getId())
- .build();
+ return ContainerProtocol.TajoContainerIdProto.newBuilder()
+ .setAppAttemptId(attemptIdProto)
+ .setAppId(appIdProto)
+ .setId(containerId.getId())
+ .build();
}
}
[02/14] tajo git commit: TAJO-1163: TableDesc should use URI instead
of Path.
Posted by ji...@apache.org.
TAJO-1163: TableDesc should use URI instead of Path.
Closes #235
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f6e09a5a
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f6e09a5a
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f6e09a5a
Branch: refs/heads/index_support
Commit: f6e09a5ae6f35dd6fe7f9ae895f0517f902b9b4d
Parents: a56f8b5
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Nov 25 15:32:13 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Nov 25 15:32:13 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 +
.../org/apache/tajo/catalog/CatalogUtil.java | 2 +-
.../java/org/apache/tajo/catalog/TableDesc.java | 21 ++++---
.../org/apache/tajo/catalog/TestTableDesc.java | 10 +--
.../tajo/catalog/store/HCatalogStore.java | 11 ++--
.../tajo/catalog/store/TestHCatalogStore.java | 22 +++----
.../org/apache/tajo/catalog/TestCatalog.java | 16 ++---
.../org/apache/tajo/jdbc/TajoResultSet.java | 4 +-
.../engine/planner/global/GlobalPlanner.java | 2 +-
.../planner/physical/PhysicalPlanUtil.java | 7 ++-
.../org/apache/tajo/master/GlobalEngine.java | 10 +--
.../apache/tajo/master/querymaster/Query.java | 6 +-
.../tajo/master/querymaster/Repartitioner.java | 6 +-
.../tajo/master/querymaster/SubQuery.java | 2 +-
.../apache/tajo/LocalTajoTestingUtility.java | 2 +-
.../java/org/apache/tajo/QueryTestCaseBase.java | 4 +-
.../apache/tajo/cli/tools/TestDDLBuilder.java | 8 +--
.../org/apache/tajo/client/TestTajoClient.java | 18 +++---
.../apache/tajo/engine/eval/ExprTestBase.java | 2 +-
.../tajo/engine/eval/TestEvalTreeUtil.java | 2 +-
.../engine/planner/TestLogicalOptimizer.java | 6 +-
.../tajo/engine/planner/TestLogicalPlanner.java | 6 +-
.../tajo/engine/planner/TestPlannerUtil.java | 8 +--
.../planner/physical/TestBNLJoinExec.java | 14 +++--
.../planner/physical/TestBSTIndexExec.java | 2 +-
.../planner/physical/TestExternalSortExec.java | 6 +-
.../physical/TestFullOuterHashJoinExec.java | 23 ++++---
.../physical/TestFullOuterMergeJoinExec.java | 36 ++++++-----
.../planner/physical/TestHashAntiJoinExec.java | 8 +--
.../planner/physical/TestHashJoinExec.java | 14 +++--
.../planner/physical/TestHashSemiJoinExec.java | 18 +++---
.../physical/TestLeftOuterHashJoinExec.java | 38 ++++++------
.../physical/TestLeftOuterNLJoinExec.java | 20 +++---
.../planner/physical/TestMergeJoinExec.java | 6 +-
.../engine/planner/physical/TestNLJoinExec.java | 16 ++---
.../planner/physical/TestPhysicalPlanner.java | 65 ++++++++++----------
.../physical/TestProgressExternalSortExec.java | 6 +-
.../physical/TestRightOuterHashJoinExec.java | 23 ++++---
.../physical/TestRightOuterMergeJoinExec.java | 26 ++++----
.../engine/planner/physical/TestSortExec.java | 2 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 8 +--
.../tajo/engine/query/TestCreateTable.java | 9 +--
.../tajo/engine/query/TestInsertQuery.java | 6 +-
.../tajo/engine/query/TestJoinBroadcast.java | 9 +--
.../tajo/engine/query/TestTablePartitions.java | 26 ++++----
.../org/apache/tajo/jdbc/TestResultSet.java | 2 +-
.../tajo/worker/TestRangeRetrieverHandler.java | 4 +-
.../org/apache/tajo/plan/LogicalPlanner.java | 9 +--
.../apache/tajo/plan/logical/InsertNode.java | 2 +-
.../rewrite/rules/PartitionedTableRewriter.java | 4 +-
50 files changed, 309 insertions(+), 270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 6b52026..06a0f47 100644
--- a/CHANGES
+++ b/CHANGES
@@ -60,6 +60,8 @@ Release 0.9.1 - unreleased
TAJO-1172: Remove Trevni storage type and its related classes.
(DaeMyung Kang via hyunsik)
+ TAJO-1163: TableDesc should use URI instead of Path. (hyunsik)
+
BUG FIXES
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
index 4171e80..737c9ae 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
@@ -292,7 +292,7 @@ public class CatalogUtil {
}
public static TableDesc newTableDesc(String tableName, Schema schema, TableMeta meta, Path path) {
- return new TableDesc(tableName, schema, meta, path);
+ return new TableDesc(tableName, schema, meta, path.toUri());
}
public static TableDesc newTableDesc(TableDescProto proto) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
index 64255d5..ce167e1 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TableDesc.java
@@ -24,7 +24,6 @@ import com.google.gson.GsonBuilder;
import com.google.gson.annotations.Expose;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.json.CatalogGsonHelper;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
@@ -35,14 +34,16 @@ import org.apache.tajo.json.GsonObject;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.util.TUtil;
+import java.net.URI;
+
public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Cloneable {
private final Log LOG = LogFactory.getLog(TableDesc.class);
@Expose protected String tableName; // required
@Expose protected Schema schema;
@Expose protected TableMeta meta; // required
- /** uri is set if external flag is TRUE.*/
- @Expose protected Path uri; // optional
+ /** uri is set if external flag is TRUE. */
+ @Expose protected URI uri; // optional
@Expose protected TableStats stats; // optional
/** the description of table partition */
@Expose protected PartitionMethodDesc partitionMethodDesc; // optional
@@ -53,26 +54,26 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
}
public TableDesc(String tableName, Schema schema, TableMeta meta,
- Path path, boolean external) {
+ URI uri, boolean external) {
this();
this.tableName = tableName;
this.schema = schema;
this.meta = meta;
- this.uri = path;
+ this.uri = uri;
this.external = external;
}
- public TableDesc(String tableName, Schema schema, TableMeta meta, Path path) {
+ public TableDesc(String tableName, Schema schema, TableMeta meta, URI path) {
this(tableName, schema, meta, path, true);
}
- public TableDesc(String tableName, Schema schema, StoreType type, KeyValueSet options, Path path) {
+ public TableDesc(String tableName, Schema schema, StoreType type, KeyValueSet options, URI path) {
this(tableName, schema, new TableMeta(type, options), path);
}
public TableDesc(TableDescProto proto) {
this(proto.getTableName(), new Schema(proto.getSchema()),
- new TableMeta(proto.getMeta()), proto.hasPath() ? new Path(proto.getPath()) : null, proto.getIsExternal());
+ new TableMeta(proto.getMeta()), proto.hasPath() ? URI.create(proto.getPath()) : null, proto.getIsExternal());
if(proto.hasStats()) {
this.stats = new TableStats(proto.getStats());
}
@@ -89,11 +90,11 @@ public class TableDesc implements ProtoObject<TableDescProto>, GsonObject, Clone
return this.tableName;
}
- public void setPath(Path uri) {
+ public void setPath(URI uri) {
this.uri = uri;
}
- public Path getPath() {
+ public URI getPath() {
return this.uri;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
index b7a9906..feeaadd 100644
--- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
+++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestTableDesc.java
@@ -47,7 +47,7 @@ public class TestTableDesc {
schema.addColumn("addr", Type.TEXT);
info = CatalogUtil.newTableMeta(StoreType.CSV);
path = new Path(CommonTestingUtil.getTestDir(), "table1");
- desc = new TableDesc("table1", schema, info, path);
+ desc = new TableDesc("table1", schema, info, path.toUri());
stats = new TableStats();
stats.setNumRows(957685);
stats.setNumBytes(1023234);
@@ -76,10 +76,10 @@ public class TestTableDesc {
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
- TableDesc desc = new TableDesc("table1", schema, info, path);
+ TableDesc desc = new TableDesc("table1", schema, info, path.toUri());
assertEquals("table1", desc.getName());
- assertEquals(path, desc.getPath());
+ assertEquals(path.toUri(), desc.getPath());
assertEquals(info, desc.getMeta());
testClone(desc);
}
@@ -87,7 +87,7 @@ public class TestTableDesc {
@Test
public void testGetProto() throws CloneNotSupportedException, IOException {
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
- TableDesc desc = new TableDesc("table1", schema, info, path);
+ TableDesc desc = new TableDesc("table1", schema, info, path.toUri());
desc.setStats(stats);
CatalogProtos.TableDescProto proto = desc.getProto();
@@ -98,7 +98,7 @@ public class TestTableDesc {
@Test
public void testToJson() throws CloneNotSupportedException, IOException {
Path path = new Path(CommonTestingUtil.getTestDir(), "tajo");
- TableDesc desc = new TableDesc("table1", schema, info, path);
+ TableDesc desc = new TableDesc("table1", schema, info, path.toUri());
desc.setStats(stats);
String json = desc.toJson();
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
index c54a395..fa1cfd6 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/main/java/org/apache/tajo/catalog/store/HCatalogStore.java
@@ -262,7 +262,7 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
if(client != null) client.release();
}
TableMeta meta = new TableMeta(storeType, options);
- TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path);
+ TableDesc tableDesc = new TableDesc(databaseName + "." + tableName, schema, meta, path.toUri());
if (table.getTableType().equals(TableType.EXTERNAL_TABLE)) {
tableDesc.setExternal(true);
}
@@ -442,12 +442,13 @@ public class HCatalogStore extends CatalogConstants implements CatalogStore {
table.setTableType(TableType.EXTERNAL_TABLE.name());
table.putToParameters("EXTERNAL", "TRUE");
- FileSystem fs = tableDesc.getPath().getFileSystem(conf);
- if (fs.isFile(tableDesc.getPath())) {
+ Path tablePath = new Path(tableDesc.getPath());
+ FileSystem fs = tablePath.getFileSystem(conf);
+ if (fs.isFile(tablePath)) {
LOG.warn("A table path is a file, but HCatalog does not allow a file path.");
- sd.setLocation(tableDesc.getPath().getParent().toString());
+ sd.setLocation(tablePath.getParent().toString());
} else {
- sd.setLocation(tableDesc.getPath().toString());
+ sd.setLocation(tablePath.toString());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
index 5f4ad45..725f665 100644
--- a/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
+++ b/tajo-catalog/tajo-catalog-drivers/tajo-hcatalog/src/test/java/org/apache/tajo/catalog/store/TestHCatalogStore.java
@@ -97,7 +97,7 @@ public class TestHCatalogStore {
schema.addColumn("c_comment", TajoDataTypes.Type.TEXT);
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, CUSTOMER)));
+ new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, CUSTOMER));
@@ -126,7 +126,7 @@ public class TestHCatalogStore {
schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, REGION)));
+ new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, REGION));
@@ -155,7 +155,7 @@ public class TestHCatalogStore {
schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, REGION)));
+ new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, REGION));
@@ -188,7 +188,7 @@ public class TestHCatalogStore {
schema.addColumn("s_comment", TajoDataTypes.Type.TEXT);
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, SUPPLIER)));
+ new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, SUPPLIER));
@@ -228,7 +228,7 @@ public class TestHCatalogStore {
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, NATION), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, NATION)));
+ new Path(warehousePath, new Path(DB_NAME, NATION)).toUri());
org.apache.tajo.catalog.Schema expressionSchema = new org.apache.tajo.catalog.Schema();
expressionSchema.addColumn("n_nationkey", TajoDataTypes.Type.INT4);
@@ -274,7 +274,7 @@ public class TestHCatalogStore {
for(String tableName : tableNames){
TableDesc table = new TableDesc(CatalogUtil.buildFQName("default", tableName), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, tableName)));
+ new Path(warehousePath, new Path(DB_NAME, tableName)).toUri());
store.createTable(table.getProto());
}
@@ -299,13 +299,13 @@ public class TestHCatalogStore {
schema.addColumn("n_comment", TajoDataTypes.Type.TEXT);
String tableName = "table1";
- TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath);
+ TableDesc table = new TableDesc(DB_NAME + "." + tableName, schema, meta, warehousePath.toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, tableName));
TableDesc table1 = new TableDesc(store.getTable(DB_NAME, tableName));
FileSystem fs = FileSystem.getLocal(new Configuration());
- assertTrue(fs.exists(table1.getPath()));
+ assertTrue(fs.exists(new Path(table1.getPath())));
store.dropTable(DB_NAME, tableName);
assertFalse(store.existTable(DB_NAME, tableName));
@@ -324,7 +324,7 @@ public class TestHCatalogStore {
schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, REGION)));
+ new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, REGION));
@@ -353,7 +353,7 @@ public class TestHCatalogStore {
schema.addColumn("r_comment", TajoDataTypes.Type.TEXT);
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, REGION)));
+ new Path(warehousePath, new Path(DB_NAME, REGION)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, REGION));
@@ -385,7 +385,7 @@ public class TestHCatalogStore {
schema.addColumn("c_comment", TajoDataTypes.Type.TEXT);
TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta,
- new Path(warehousePath, new Path(DB_NAME, CUSTOMER)));
+ new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, CUSTOMER));
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
index 4a45abc..a044d64 100644
--- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
+++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java
@@ -246,7 +246,7 @@ public class TestCatalog {
CatalogUtil.buildFQName(databaseName, tableName),
schema1,
new TableMeta(StoreType.CSV, new KeyValueSet()),
- path, true);
+ path.toUri(), true);
return table;
}
@@ -367,7 +367,7 @@ public class TestCatalog {
schema1,
StoreType.CSV,
new KeyValueSet(),
- path);
+ path.toUri());
assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable"));
catalog.createTable(meta);
@@ -407,7 +407,7 @@ public class TestCatalog {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
return new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), schema, meta,
- new Path(CommonTestingUtil.getTestDir(), "indexed"));
+ new Path(CommonTestingUtil.getTestDir(), "indexed").toUri());
}
@Test
@@ -587,7 +587,7 @@ public class TestCatalog {
TableDesc desc =
new TableDesc(tableName, schema, meta,
- new Path(CommonTestingUtil.getTestDir(), "addedtable"));
+ new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
@@ -625,7 +625,7 @@ public class TestCatalog {
TableDesc desc =
new TableDesc(tableName, schema, meta,
- new Path(CommonTestingUtil.getTestDir(), "addedtable"));
+ new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
@@ -663,7 +663,7 @@ public class TestCatalog {
TableDesc desc =
new TableDesc(tableName, schema, meta,
- new Path(CommonTestingUtil.getTestDir(), "addedtable"));
+ new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
@@ -700,7 +700,7 @@ public class TestCatalog {
TableDesc desc =
new TableDesc(tableName, schema, meta,
- new Path(CommonTestingUtil.getTestDir(), "addedtable"));
+ new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
@@ -738,7 +738,7 @@ public class TestCatalog {
TableDesc desc =
new TableDesc(tableName, schema, meta,
- new Path(CommonTestingUtil.getTestDir(), "addedtable"));
+ new Path(CommonTestingUtil.getTestDir(), "addedtable").toUri());
desc.setPartitionMethod(partitionDesc);
assertFalse(catalog.existsTable(tableName));
catalog.createTable(desc);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
index d78b04f..e99c0ff 100644
--- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
+++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSet.java
@@ -81,7 +81,7 @@ public class TajoResultSet extends TajoResultSetBase {
private void initScanner() throws IOException {
if(desc != null) {
schema = desc.getSchema();
- fs = FileScanner.getFileSystem(conf, desc.getPath());
+ fs = FileScanner.getFileSystem(conf, new Path(desc.getPath()));
if (maxRowNum != null) {
this.totalRow = maxRowNum;
} else {
@@ -92,7 +92,7 @@ public class TajoResultSet extends TajoResultSetBase {
totalRow = INFINITE_ROW_NUM;
}
- List<FileFragment> frags = getFragments(desc.getPath());
+ List<FileFragment> frags = getFragments(new Path(desc.getPath()));
scanner = new MergeScanner(conf, desc.getSchema(), desc.getMeta(), frags);
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
index b33e4a0..c75b348 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java
@@ -177,7 +177,7 @@ public class GlobalPlanner {
"Channel schema (" + channel.getSrcId().getId() + " -> " + channel.getTargetId().getId() +
") is not initialized");
TableMeta meta = new TableMeta(channel.getStoreType(), new KeyValueSet());
- TableDesc desc = new TableDesc(channel.getSrcId().toString(), channel.getSchema(), meta, new Path("/"));
+ TableDesc desc = new TableDesc(channel.getSrcId().toString(), channel.getSchema(), meta, new Path("/").toUri());
ScanNode scanNode = plan.createNode(ScanNode.class);
scanNode.init(desc);
return scanNode;
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
index da5e073..fe1f795 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java
@@ -74,11 +74,12 @@ public class PhysicalPlanUtil {
*/
public static CatalogProtos.FragmentProto[] getNonZeroLengthDataFiles(TajoConf tajoConf,TableDesc tableDesc,
int fileIndex, int numResultFiles) throws IOException {
- FileSystem fs = tableDesc.getPath().getFileSystem(tajoConf);
+ Path path = new Path(tableDesc.getPath());
+ FileSystem fs = path.getFileSystem(tajoConf);
List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
- if (fs.exists(tableDesc.getPath())) {
- getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, fileIndex, numResultFiles,
+ if (fs.exists(path)) {
+ getNonZeroLengthDataFiles(fs, path, nonZeroLengthFiles, fileIndex, numResultFiles,
new AtomicInteger(0));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 9831c00..47a8750 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -344,7 +344,7 @@ public class GlobalEngine extends AbstractService {
Path finalOutputDir = null;
if (insertNode.getTableName() != null) {
tableDesc = this.catalog.getTableDesc(insertNode.getTableName());
- finalOutputDir = tableDesc.getPath();
+ finalOutputDir = new Path(tableDesc.getPath());
} else {
finalOutputDir = insertNode.getPath();
}
@@ -655,7 +655,7 @@ public class GlobalEngine extends AbstractService {
Path warehousePath = new Path(TajoConf.getWarehouseDir(context.getConf()), databaseName);
TableDesc tableDesc = catalog.getTableDesc(databaseName, simpleTableName);
- Path tablePath = tableDesc.getPath();
+ Path tablePath = new Path(tableDesc.getPath());
if (tablePath.getParent() == null ||
!tablePath.getParent().toUri().getPath().equals(warehousePath.toUri().getPath())) {
throw new IOException("Can't truncate external table:" + eachTableName + ", data dir=" + tablePath +
@@ -665,7 +665,7 @@ public class GlobalEngine extends AbstractService {
}
for (TableDesc eachTable: tableDescList) {
- Path path = eachTable.getPath();
+ Path path = new Path(eachTable.getPath());
LOG.info("Truncate table: " + eachTable.getName() + ", delete all data files in " + path);
FileSystem fs = path.getFileSystem(context.getConf());
@@ -768,7 +768,7 @@ public class GlobalEngine extends AbstractService {
}
TableDesc desc = new TableDesc(CatalogUtil.buildFQName(databaseName, simpleTableName),
- schema, meta, path, isExternal);
+ schema, meta, path.toUri(), isExternal);
desc.setStats(stats);
if (partitionDesc != null) {
desc.setPartitionMethod(partitionDesc);
@@ -867,7 +867,7 @@ public class GlobalEngine extends AbstractService {
}
}
- Path path = catalog.getTableDesc(qualifiedName).getPath();
+ Path path = new Path(catalog.getTableDesc(qualifiedName).getPath());
catalog.dropTable(qualifiedName);
if (purge) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 7114d39..7db6d8b 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -794,7 +794,7 @@ public class Query implements EventHandler<QueryEvent> {
query.getId().toString(),
lastStage.getSchema(),
meta,
- finalOutputDir);
+ finalOutputDir.toUri());
resultTableDesc.setExternal(true);
stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir));
@@ -827,7 +827,7 @@ public class Query implements EventHandler<QueryEvent> {
createTableNode.getTableName(),
createTableNode.getTableSchema(),
meta,
- finalOutputDir);
+ finalOutputDir.toUri());
tableDescTobeCreated.setExternal(createTableNode.isExternal());
if (createTableNode.hasPartition()) {
@@ -869,7 +869,7 @@ public class Query implements EventHandler<QueryEvent> {
finalTable = catalog.getTableDesc(tableName);
} else {
String tableName = query.getId().toString();
- finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir);
+ finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir.toUri());
}
long volume = getTableVolume(query.systemConf, finalOutputDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
index 447c164..55b1895 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java
@@ -107,7 +107,7 @@ public class Repartitioner {
}
fragments[i] = new FileFragment(scans[i].getCanonicalName(), tablePath, 0, 0, new String[]{UNKNOWN_HOST});
} else {
- tablePath = tableDesc.getPath();
+ tablePath = new Path(tableDesc.getPath());
try {
stats[i] = GlobalPlanner.computeDescendentVolume(scans[i]);
} catch (PlanningException e) {
@@ -411,7 +411,7 @@ public class Repartitioner {
partitionScan.setInputPaths(partitionScanPaths);
} else {
Collection<FileFragment> scanFragments = subQuery.getStorageManager().getSplits(eachScan.getCanonicalName(),
- tableDesc.getMeta(), tableDesc.getSchema(), tableDesc.getPath());
+ tableDesc.getMeta(), tableDesc.getSchema(), new Path(tableDesc.getPath()));
if (scanFragments != null) {
rightFragments.addAll(scanFragments);
}
@@ -527,7 +527,7 @@ public class Repartitioner {
scanFragments = getFragmentsFromPartitionedTable(subQuery.getStorageManager(), scan, desc);
} else {
scanFragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, desc.getSchema(),
- desc.getPath());
+ new Path(desc.getPath()));
}
if (scanFragments != null) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
index 96534df..476da04 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java
@@ -1059,7 +1059,7 @@ public class SubQuery implements EventHandler<SubQueryEvent> {
// After calling this method, partition paths are removed from the physical plan.
fragments = Repartitioner.getFragmentsFromPartitionedTable(subQuery.getStorageManager(), scan, table);
} else {
- Path inputPath = table.getPath();
+ Path inputPath = new Path(table.getPath());
fragments = subQuery.getStorageManager().getSplits(scan.getCanonicalName(), meta, table.getSchema(), inputPath);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
index c6cac32..f6a5fed 100644
--- a/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
+++ b/tajo-core/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java
@@ -122,7 +122,7 @@ public class LocalTajoTestingUtility {
stats.setNumBytes(TPCH.tableVolumes.get(names[i]));
TableDesc tableDesc = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, names[i]), schemas[i], meta,
- tablePath);
+ tablePath.toUri());
tableDesc.setStats(stats);
util.getMaster().getCatalog().createTable(tableDesc);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index 604a56b..4e4964e 100644
--- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -684,7 +684,7 @@ public class QueryTestCaseBase {
return null;
}
- Path path = tableDesc.getPath();
+ Path path = new Path(tableDesc.getPath());
return getTableFileContents(path);
}
@@ -694,7 +694,7 @@ public class QueryTestCaseBase {
return null;
}
- Path path = tableDesc.getPath();
+ Path path = new Path(tableDesc.getPath());
FileSystem fs = path.getFileSystem(conf);
return listFiles(fs, path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
index 83ce459..704980b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java
@@ -58,7 +58,7 @@ public class TestDDLBuilder {
@Test
public void testBuildDDLForExternalTable() throws Exception {
- TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1"));
+ TableDesc desc = new TableDesc("db1.table1", schema1, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod1);
desc.setExternal(true);
assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForExternalTable.result"),
@@ -84,13 +84,13 @@ public class TestDDLBuilder {
"key,key2",
expressionSchema2);
- TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1"));
+ TableDesc desc = new TableDesc("db1.TABLE2", schema2, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod2);
desc.setExternal(true);
assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName1.result"),
DDLBuilder.buildDDLForExternalTable(desc));
- desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1"));
+ desc = new TableDesc("db1.TABLE1", schema2, meta1, new Path("/table1").toUri());
desc.setPartitionMethod(partitionMethod2);
desc.setExternal(false);
assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLQuotedTableName2.result"),
@@ -99,7 +99,7 @@ public class TestDDLBuilder {
@Test
public void testBuildDDLForBaseTable() throws Exception {
- TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1"));
+ TableDesc desc = new TableDesc("db1.table2", schema1, meta1, new Path("/table1").toUri());
assertEquals(FileUtil.readTextFileFromResource("results/testDDLBuilder/testBuildDDLForBaseTable.result"),
DDLBuilder.buildDDLForBaseTable(desc));
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
index 6e255ff..616799f 100644
--- a/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
+++ b/tajo-core/src/test/java/org/apache/tajo/client/TestTajoClient.java
@@ -324,7 +324,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -345,7 +345,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -422,7 +422,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -446,7 +446,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -470,7 +470,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -495,7 +495,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -521,7 +521,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -574,7 +574,7 @@ public class TestTajoClient {
client.updateQuery(sql);
assertTrue(client.existTable(tableName));
- Path tablePath = client.getTableDesc(tableName).getPath();
+ Path tablePath = new Path(client.getTableDesc(tableName).getPath());
FileSystem hdfs = tablePath.getFileSystem(conf);
assertTrue(hdfs.exists(tablePath));
@@ -702,7 +702,7 @@ public class TestTajoClient {
assertEquals(res.getTableDesc().getMeta().getOption(StorageConstants.TEXT_NULL), "\\\\T");
- Path path = res.getTableDesc().getPath();
+ Path path = new Path(res.getTableDesc().getPath());
FileSystem fs = path.getFileSystem(tajoConf);
FileStatus[] files = fs.listStatus(path);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 73d89ec..df9ef65 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -241,7 +241,7 @@ public class ExprTestBase {
}
}
cat.createTable(new TableDesc(qualifiedTableName, inputSchema,
- CatalogProtos.StoreType.CSV, new KeyValueSet(), CommonTestingUtil.getTestDir()));
+ CatalogProtos.StoreType.CSV, new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()));
}
Target [] targets;
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
index dcacff2..c8e3745 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java
@@ -106,7 +106,7 @@ public class TestEvalTreeUtil {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
TableDesc desc = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta,
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(desc);
FunctionDesc funcMeta = new FunctionDesc("test_sum", TestSum.class,
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
index 87bc757..c9b52fd 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java
@@ -82,19 +82,19 @@ public class TestLogicalOptimizer {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(people);
TableDesc student =
new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(),
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(student);
TableDesc score =
new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(),
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(score);
FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.GENERAL,
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
index e4e7916..180033d 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java
@@ -95,17 +95,17 @@ public class TestLogicalPlanner {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(people);
TableDesc student = new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV, new KeyValueSet(),
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(student);
TableDesc score = new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV, new KeyValueSet(),
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(score);
FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.AGGREGATION,
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
index 5cc89b8..3437e3a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java
@@ -88,19 +88,19 @@ public class TestPlannerUtil {
TableMeta meta = CatalogUtil.newTableMeta(StoreType.CSV);
TableDesc people = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
- CommonTestingUtil.getTestDir());
+ CommonTestingUtil.getTestDir().toUri());
catalog.createTable(people);
TableDesc student =
new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, StoreType.CSV,
- new KeyValueSet(), CommonTestingUtil.getTestDir());
+ new KeyValueSet(), CommonTestingUtil.getTestDir().toUri());
catalog.createTable(student);
TableDesc score =
new TableDesc(
CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, StoreType.CSV,
- new KeyValueSet(), CommonTestingUtil.getTestDir());
+ new KeyValueSet(), CommonTestingUtil.getTestDir().toUri());
catalog.createTable(score);
FunctionDesc funcDesc = new FunctionDesc("sumtest", SumInt.class, FunctionType.AGGREGATION,
@@ -338,7 +338,7 @@ public class TestPlannerUtil {
TableDesc tableDesc = new TableDesc();
tableDesc.setName("Test");
- tableDesc.setPath(path);
+ tableDesc.setPath(path.toUri());
FileSystem fs = path.getFileSystem(util.getConfiguration());
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
index df7f897..aef8064 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java
@@ -150,9 +150,11 @@ public class TestBNLJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.BLOCK_NESTED_LOOP_JOIN);
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()),
Integer.MAX_VALUE);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(),
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testBNLCrossJoin");
@@ -181,10 +183,10 @@ public class TestBNLJoinExec {
LogicalNode plan = planner.createPlan(LocalTajoTestingUtility.createDummyContext(conf),
context).getRootBlock().getRoot();
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
index e12d6eb..60ae849 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java
@@ -146,7 +146,7 @@ public class TestBSTIndexExec {
TableDesc desc = new TableDesc(
CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta,
- sm.getTablePath("employee"));
+ sm.getTablePath("employee").toUri());
catalog.createTable(desc);
analyzer = new SQLAnalyzer();
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
index f09d104..bb40b28 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java
@@ -103,7 +103,7 @@ public class TestExternalSortExec {
System.out.println(appender.getStats().getNumRows() + " rows (" + (appender.getStats().getNumBytes() / 1048576) +
" MB)");
- employee = new TableDesc("default.employee", schema, employeeMeta, employeePath);
+ employee = new TableDesc("default.employee", schema, employeeMeta, employeePath.toUri());
catalog.createTable(employee);
analyzer = new SQLAnalyzer();
planner = new LogicalPlanner(catalog);
@@ -121,8 +121,8 @@ public class TestExternalSortExec {
@Test
public final void testNext() throws IOException, PlanningException {
- FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] frags = StorageManager.splitNG(conf, "default.employee", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
Path workDir = new Path(testDir, TestExternalSortExec.class.getName());
TaskAttemptContext ctx = new TaskAttemptContext(new QueryContext(conf),
LocalTajoTestingUtility.newQueryUnitAttemptId(), new FileFragment[] { frags[0] }, workDir);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
index 104ca52..c4ce43b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java
@@ -266,8 +266,10 @@ public class TestFullOuterHashJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
- FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()),
+ Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
+ Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec0");
@@ -303,8 +305,10 @@ public class TestFullOuterHashJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()),
+ Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
+ Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuter_HashJoinExec1");
@@ -339,8 +343,10 @@ public class TestFullOuterHashJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
+ Integer.MAX_VALUE);
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()),
+ Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestFullOuterHashJoinExec2");
@@ -376,8 +382,9 @@ public class TestFullOuterHashJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
- FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
+ Integer.MAX_VALUE);
+ FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
index 23c0fb4..a82de92 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java
@@ -30,16 +30,20 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.parser.SQLAnalyzer;
-import org.apache.tajo.engine.planner.*;
+import org.apache.tajo.engine.planner.PhysicalPlanner;
+import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
import org.apache.tajo.engine.planner.enforce.Enforcer;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.plan.LogicalPlanner;
-import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.logical.JoinNode;
import org.apache.tajo.plan.logical.LogicalNode;
import org.apache.tajo.plan.logical.NodeType;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.storage.*;
+import org.apache.tajo.plan.util.PlannerUtil;
+import org.apache.tajo.storage.Appender;
+import org.apache.tajo.storage.StorageManager;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.VTuple;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.TUtil;
@@ -314,9 +318,9 @@ public class TestFullOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] dep3Frags =
- StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, dep3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin0");
@@ -351,9 +355,9 @@ public class TestFullOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] job3Frags =
- StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin1");
@@ -388,9 +392,9 @@ public class TestFullOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] job3Frags =
- StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin2");
@@ -426,9 +430,9 @@ public class TestFullOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] dep4Frags =
- StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), dep4.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, DEP4_NAME, dep4.getMeta(), new Path(dep4.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, dep4Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testFullOuterMergeJoin3");
@@ -466,9 +470,9 @@ public class TestFullOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] phone3Frags =
- StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
+ StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
@@ -505,9 +509,9 @@ public class TestFullOuterMergeJoinExec {
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
FileFragment[] emp3Frags =
- StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] phone3Frags =
- StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
+ StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(phone3Frags,emp3Frags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
index e941767..20d4651 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java
@@ -150,10 +150,10 @@ public class TestHashAntiJoinExec {
@Test
public final void testHashAntiJoin() throws IOException, PlanningException {
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
index e25ca8f..d1fa28a 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java
@@ -152,8 +152,10 @@ public class TestHashJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE);
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin");
@@ -193,10 +195,10 @@ public class TestHashJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()), Integer.MAX_VALUE);
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testHashInnerJoin");
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
index 4998012..7a43a55 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java
@@ -29,15 +29,19 @@ import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.Datum;
import org.apache.tajo.datum.DatumFactory;
import org.apache.tajo.engine.parser.SQLAnalyzer;
-import org.apache.tajo.engine.planner.*;
+import org.apache.tajo.engine.planner.PhysicalPlanner;
+import org.apache.tajo.engine.planner.PhysicalPlannerImpl;
import org.apache.tajo.engine.planner.enforce.Enforcer;
+import org.apache.tajo.engine.query.QueryContext;
import org.apache.tajo.plan.LogicalOptimizer;
import org.apache.tajo.plan.LogicalPlan;
import org.apache.tajo.plan.LogicalPlanner;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.logical.LogicalNode;
-import org.apache.tajo.engine.query.QueryContext;
-import org.apache.tajo.storage.*;
+import org.apache.tajo.storage.Appender;
+import org.apache.tajo.storage.StorageManager;
+import org.apache.tajo.storage.Tuple;
+import org.apache.tajo.storage.VTuple;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.TUtil;
@@ -154,10 +158,10 @@ public class TestHashSemiJoinExec {
@Test
public final void testHashSemiJoin() throws IOException, PlanningException {
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
index f20984a..ec9daa7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java
@@ -270,8 +270,10 @@ public class TestLeftOuterHashJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.IN_MEMORY_HASH_JOIN);
- FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(), Integer.MAX_VALUE);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(), Integer.MAX_VALUE);
+ FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(),
+ new Path(dep3.getPath()), Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(),
+ new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/TestLeftOuterHashJoinExec0");
@@ -298,10 +300,10 @@ public class TestLeftOuterHashJoinExec {
@Test
public final void testLeftOuter_HashJoinExec1() throws IOException, PlanningException {
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(),
+ new Path(job3.getPath()), Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(),
+ new Path(emp3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
@@ -339,10 +341,10 @@ public class TestLeftOuterHashJoinExec {
@Test
public final void testLeftOuter_HashJoinExec2() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(),
+ new Path(emp3.getPath()), Integer.MAX_VALUE);
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(),
+ new Path(job3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
@@ -381,10 +383,10 @@ public class TestLeftOuterHashJoinExec {
@Test
public final void testLeftOuter_HashJoinExec3() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(),
+ new Path(emp3.getPath()), Integer.MAX_VALUE);
+ FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(),
+ new Path(phone3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
@@ -423,10 +425,10 @@ public class TestLeftOuterHashJoinExec {
@Test
public final void testLeftOuter_HashJoinExec4() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, "default.emp3", emp3.getMeta(), emp3.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] phone3Frags = StorageManager.splitNG(conf, "default.phone3", phone3.getMeta(), phone3.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, "default.emp3", emp3.getMeta(),
+ new Path(emp3.getPath()), Integer.MAX_VALUE);
+ FileFragment[] phone3Frags = StorageManager.splitNG(conf, "default.phone3", phone3.getMeta(),
+ new Path(phone3.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
index 4b453fe..b3d1f33 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java
@@ -254,9 +254,9 @@ public class TestLeftOuterNLJoinExec {
@Test
public final void testLeftOuterNLJoinExec0() throws IOException, PlanningException {
- FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), dep3.getPath(),
+ FileFragment[] dep3Frags = StorageManager.splitNG(conf, DEP3_NAME, dep3.getMeta(), new Path(dep3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(dep3Frags, emp3Frags);
@@ -295,9 +295,9 @@ public class TestLeftOuterNLJoinExec {
@Test
public final void testLeftOuterNLJoinExec1() throws IOException, PlanningException {
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(),
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(job3Frags, emp3Frags);
@@ -340,9 +340,9 @@ public class TestLeftOuterNLJoinExec {
@Test
public final void testLeftOuter_NLJoinExec2() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), job3.getPath(),
+ FileFragment[] job3Frags = StorageManager.splitNG(conf, JOB3_NAME, job3.getMeta(), new Path(job3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, job3Frags);
@@ -385,9 +385,9 @@ public class TestLeftOuterNLJoinExec {
@Test
public final void testLeftOuter_NLJoinExec3() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
+ FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(emp3Frags, phone3Frags);
@@ -429,9 +429,9 @@ public class TestLeftOuterNLJoinExec {
@Test
public final void testLeftOuter_NLJoinExec4() throws IOException, PlanningException {
- FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), emp3.getPath(),
+ FileFragment[] emp3Frags = StorageManager.splitNG(conf, EMP3_NAME, emp3.getMeta(), new Path(emp3.getPath()),
Integer.MAX_VALUE);
- FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), phone3.getPath(),
+ FileFragment[] phone3Frags = StorageManager.splitNG(conf, PHONE3_NAME, phone3.getMeta(), new Path(phone3.getPath()),
Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(phone3Frags, emp3Frags);
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
index 7a8426f..cae5de5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java
@@ -165,8 +165,10 @@ public class TestMergeJoinExec {
Enforcer enforcer = new Enforcer();
enforcer.enforceJoinAlgorithm(joinNode.getPID(), JoinAlgorithm.MERGE_JOIN);
- FileFragment[] empFrags = sm.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(), Integer.MAX_VALUE);
- FileFragment[] peopleFrags = sm.splitNG(conf, "default.p", people.getMeta(), people.getPath(), Integer.MAX_VALUE);
+ FileFragment[] empFrags = sm.splitNG(conf, "default.e", employee.getMeta(), new Path(employee.getPath()),
+ Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = sm.splitNG(conf, "default.p", people.getMeta(), new Path(people.getPath()),
+ Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
Path workDir = CommonTestingUtil.getTestDir("target/test-data/testMergeInnerJoin");
http://git-wip-us.apache.org/repos/asf/tajo/blob/f6e09a5a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
index fc67a38..a5d1dc5 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java
@@ -144,10 +144,10 @@ public class TestNLJoinExec {
@Test
public final void testNLCrossJoin() throws IOException, PlanningException {
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
@@ -173,10 +173,10 @@ public class TestNLJoinExec {
@Test
public final void testNLInnerJoin() throws IOException, PlanningException {
- FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(), employee.getPath(),
- Integer.MAX_VALUE);
- FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(), people.getPath(),
- Integer.MAX_VALUE);
+ FileFragment[] empFrags = StorageManager.splitNG(conf, "default.e", employee.getMeta(),
+ new Path(employee.getPath()), Integer.MAX_VALUE);
+ FileFragment[] peopleFrags = StorageManager.splitNG(conf, "default.p", people.getMeta(),
+ new Path(people.getPath()), Integer.MAX_VALUE);
FileFragment[] merged = TUtil.concat(empFrags, peopleFrags);
[06/14] tajo git commit: TAJO-1053: ADD PARTITIONS for HCatalogStore.
(jaehwa)
Posted by ji...@apache.org.
TAJO-1053: ADD PARTITIONS for HCatalogStore. (jaehwa)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3ae44b1d
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3ae44b1d
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3ae44b1d
Branch: refs/heads/index_support
Commit: 3ae44b1d2a1cf49123eb1d1c30f081a8a8d0e7fb
Parents: 3d485ec
Author: JaeHwa Jung <bl...@apache.org>
Authored: Thu Nov 27 19:27:10 2014 +0900
Committer: JaeHwa Jung <bl...@apache.org>
Committed: Thu Nov 27 19:27:10 2014 +0900
----------------------------------------------------------------------
CHANGES | 2 ++
tajo-docs/src/main/sphinx/hcatalog_integration.rst | 13 +++++++++++++
2 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/3ae44b1d/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 1657c20..f2141a6 100644
--- a/CHANGES
+++ b/CHANGES
@@ -13,6 +13,8 @@ Release 0.9.1 - unreleased
IMPROVEMENT
+ TAJO-1053: ADD PARTITIONS for HCatalogStore. (jaehwa)
+
TAJO-1195: Remove unused CachedDNSResolver Class. (DaeMyung Kang via jaehwa)
TAJO-1184: Upgrade netty-buffer to 4.0.24.Final. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/3ae44b1d/tajo-docs/src/main/sphinx/hcatalog_integration.rst
----------------------------------------------------------------------
diff --git a/tajo-docs/src/main/sphinx/hcatalog_integration.rst b/tajo-docs/src/main/sphinx/hcatalog_integration.rst
index 7337346..d81975d 100644
--- a/tajo-docs/src/main/sphinx/hcatalog_integration.rst
+++ b/tajo-docs/src/main/sphinx/hcatalog_integration.rst
@@ -37,3 +37,16 @@ Finally, you should specify HCatalogStore as Tajo catalog driver class in ``conf
<name>tajo.catalog.store.class</name>
<value>org.apache.tajo.catalog.store.HCatalogStore</value>
</property>
+
+.. note::
+
+ Hive stores a list of partitions for each table in its metastore. If new partitions are
+ directly added to HDFS, HiveMetastore will not able aware of these partitions unless the user
+ ``ALTER TABLE table_name ADD PARTITION`` commands on each of the newly added partitions or
+ ``MSCK REPAIR TABLE table_name`` command.
+
+ But current tajo doesn't provide ``ADD PARTITION`` command and hive doesn't provide an api for
+ responding to ``MSK REPAIR TABLE`` command. Thus, if you insert data to hive partitioned
+ table and you want to scan the updated partitions through Tajo, you must run following command on hive ::
+
+ $ MSCK REPAIR TABLE [table_name];
[03/14] tajo git commit: TAJO-1188: Fix testcase
testTimestampConstructor in TestTimestampDatum. (DaeMyung Kang via hyunsik)
Posted by ji...@apache.org.
TAJO-1188: Fix testcase testTimestampConstructor in TestTimestampDatum. (DaeMyung Kang via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/0c97fc03
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/0c97fc03
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/0c97fc03
Branch: refs/heads/index_support
Commit: 0c97fc0319b0f33c3a76f591d48cb52f1df92567
Parents: f6e09a5
Author: Hyunsik Choi <hy...@apache.org>
Authored: Tue Nov 25 17:42:22 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Tue Nov 25 17:42:22 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +++
.../src/test/java/org/apache/tajo/datum/TestTimestampDatum.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/0c97fc03/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 06a0f47..6521c1c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -62,6 +62,9 @@ Release 0.9.1 - unreleased
TAJO-1163: TableDesc should use URI instead of Path. (hyunsik)
+ TAJO-1188: Fix testcase testTimestampConstructor in TestTimestampDatum.
+ (DaeMyung Kang via hyunsik)
+
BUG FIXES
http://git-wip-us.apache.org/repos/asf/tajo/blob/0c97fc03/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
index 5f27cfa..5886083 100644
--- a/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
+++ b/tajo-common/src/test/java/org/apache/tajo/datum/TestTimestampDatum.java
@@ -151,7 +151,8 @@ public class TestTimestampDatum {
assertEquals(datum2, datum);
for (int i = 0; i < 100; i++) {
- Calendar cal = Calendar.getInstance();
+ TimeZone timeZone = TimeZone.getTimeZone("UTC");
+ Calendar cal = Calendar.getInstance(timeZone);
long jTime = System.currentTimeMillis();
int uTime = (int)(jTime / 1000);
cal.setTimeInMillis(jTime);
[13/14] tajo git commit: TAJO-1211: Staging directory for CTAS and
INSERT should be in the output dir.
Posted by ji...@apache.org.
TAJO-1211: Staging directory for CTAS and INSERT should be in the output dir.
Closes #274
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/b4adc18c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/b4adc18c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/b4adc18c
Branch: refs/heads/index_support
Commit: b4adc18cd25de550fe04a43ef69d715c146976db
Parents: bf68b77
Author: Hyunsik Choi <hy...@apache.org>
Authored: Mon Dec 1 17:23:35 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Mon Dec 1 17:23:35 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 +
.../java/org/apache/tajo/conf/TajoConf.java | 12 +++-
.../org/apache/tajo/master/GlobalEngine.java | 7 +-
.../java/org/apache/tajo/master/TajoMaster.java | 2 +-
.../apache/tajo/master/querymaster/Query.java | 74 +++++++++++++++-----
.../master/querymaster/QueryMasterTask.java | 23 ++++--
.../src/main/resources/webapps/admin/index.jsp | 2 +-
.../apache/tajo/engine/query/TestCTASQuery.java | 1 +
.../tajo/engine/query/TestInsertQuery.java | 14 +++-
9 files changed, 105 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 33983bc..8d51d44 100644
--- a/CHANGES
+++ b/CHANGES
@@ -79,6 +79,9 @@ Release 0.9.1 - unreleased
BUG FIXES
+ TAJO-1211: Staging directory for CTAS and INSERT should be in
+ the output dir. (hyunsik)
+
TAJO-1210: ByteBufLineReader does not handle the end of file,
if newline is not appeared. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
index f3ae453..312abfb 100644
--- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
+++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
@@ -671,7 +671,15 @@ public class TajoConf extends Configuration {
return path.indexOf("file:/") == 0 || path.indexOf("hdfs:/") == 0;
}
- public static Path getStagingDir(TajoConf conf) throws IOException {
+ /**
+ * It returns the default root staging directory used by queries without a target table or
+ * a specified output directory. An example query is <pre>SELECT a,b,c FROM XXX;</pre>.
+ *
+ * @param conf TajoConf
+ * @return Path which points the default staging directory
+ * @throws IOException
+ */
+ public static Path getDefaultRootStagingDir(TajoConf conf) throws IOException {
String stagingDirString = conf.getVar(ConfVars.STAGING_ROOT_DIR);
if (!hasScheme(stagingDirString)) {
Path warehousePath = getWarehouseDir(conf);
@@ -686,7 +694,7 @@ public class TajoConf extends Configuration {
public static Path getQueryHistoryDir(TajoConf conf) throws IOException {
String historyDirString = conf.getVar(ConfVars.HISTORY_QUERY_DIR);
if (!hasScheme(historyDirString)) {
- Path stagingPath = getStagingDir(conf);
+ Path stagingPath = getDefaultRootStagingDir(conf);
FileSystem fs = stagingPath.getFileSystem(conf);
Path path = new Path(fs.getUri().toString(), historyDirString);
conf.setVar(ConfVars.HISTORY_QUERY_DIR, path.toString());
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 47a8750..9bf9a75 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -335,10 +335,8 @@ public class GlobalEngine extends AbstractService {
String queryId = nodeUniqName + "_" + System.currentTimeMillis();
FileSystem fs = TajoConf.getWarehouseDir(context.getConf()).getFileSystem(context.getConf());
- Path stagingDir = QueryMasterTask.initStagingDir(context.getConf(), fs, queryId.toString());
-
+ Path stagingDir = QueryMasterTask.initStagingDir(context.getConf(), queryId.toString(), queryContext);
Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME);
- fs.mkdirs(stagingResultDir);
TableDesc tableDesc = null;
Path finalOutputDir = null;
@@ -349,8 +347,7 @@ public class GlobalEngine extends AbstractService {
finalOutputDir = insertNode.getPath();
}
- TaskAttemptContext taskAttemptContext =
- new TaskAttemptContext(queryContext, null, null, (CatalogProtos.FragmentProto[]) null, stagingDir);
+ TaskAttemptContext taskAttemptContext = new TaskAttemptContext(queryContext, null, null, null, stagingDir);
taskAttemptContext.setOutputPath(new Path(stagingResultDir, "part-01-000000"));
EvalExprExec evalExprExec = new EvalExprExec(taskAttemptContext, (EvalExprNode) insertNode.getChild());
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index 6e585af..795983d 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@ -271,7 +271,7 @@ public class TajoMaster extends CompositeService {
LOG.info("Warehouse dir '" + wareHousePath + "' is created");
}
- Path stagingPath = TajoConf.getStagingDir(systemConf);
+ Path stagingPath = TajoConf.getDefaultRootStagingDir(systemConf);
LOG.info("Staging dir: " + wareHousePath);
if (!defaultFS.exists(stagingPath)) {
defaultFS.mkdirs(stagingPath, new FsPermission(STAGING_ROOTDIR_PERMISSION));
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
index 7db6d8b..07b47c1 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java
@@ -411,9 +411,9 @@ public class Query implements EventHandler<QueryEvent> {
public QueryState transition(Query query, QueryEvent queryEvent) {
QueryCompletedEvent subQueryEvent = (QueryCompletedEvent) queryEvent;
QueryState finalState;
+
if (subQueryEvent.getState() == SubQueryState.SUCCEEDED) {
- finalizeQuery(query, subQueryEvent);
- finalState = QueryState.QUERY_SUCCEEDED;
+ finalState = finalizeQuery(query, subQueryEvent);
} else if (subQueryEvent.getState() == SubQueryState.FAILED) {
finalState = QueryState.QUERY_FAILED;
} else if (subQueryEvent.getState() == SubQueryState.KILLED) {
@@ -427,26 +427,28 @@ public class Query implements EventHandler<QueryEvent> {
return finalState;
}
- private void finalizeQuery(Query query, QueryCompletedEvent event) {
+ private QueryState finalizeQuery(Query query, QueryCompletedEvent event) {
MasterPlan masterPlan = query.getPlan();
ExecutionBlock terminal = query.getPlan().getTerminalBlock();
DataChannel finalChannel = masterPlan.getChannel(event.getExecutionBlockId(), terminal.getId());
- Path finalOutputDir = commitOutputData(query);
QueryHookExecutor hookExecutor = new QueryHookExecutor(query.context.getQueryMasterContext());
try {
- hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(),
- finalOutputDir);
- } catch (Exception e) {
- query.eventHandler.handle(new QueryDiagnosticsUpdateEvent(query.id, ExceptionUtils.getStackTrace(e)));
+ Path finalOutputDir = commitOutputData(query);
+ hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(), finalOutputDir);
+ } catch (Throwable t) {
+ query.eventHandler.handle(new QueryDiagnosticsUpdateEvent(query.id, ExceptionUtils.getStackTrace(t)));
+ return QueryState.QUERY_ERROR;
}
+
+ return QueryState.QUERY_SUCCEEDED;
}
/**
* It moves a result data stored in a staging output dir into a final output dir.
*/
- public Path commitOutputData(Query query) {
+ public Path commitOutputData(Query query) throws IOException {
QueryContext queryContext = query.context.getQueryContext();
Path stagingResultDir = new Path(queryContext.getStagingDir(), TajoConstants.RESULT_DIR_NAME);
Path finalOutputDir;
@@ -508,24 +510,49 @@ public class Query implements EventHandler<QueryEvent> {
fs.delete(entry.getValue(), true);
fs.rename(entry.getValue(), entry.getKey());
}
+
throw new IOException(ioe.getMessage());
}
- } else {
+ } else { // no partition
try {
+
+ // if the final output dir exists, move all contents to the temporary table dir.
+ // Otherwise, just make the final output dir. As a result, the final output dir will be empty.
if (fs.exists(finalOutputDir)) {
- fs.rename(finalOutputDir, oldTableDir);
+ fs.mkdirs(oldTableDir);
+
+ for (FileStatus status : fs.listStatus(finalOutputDir, StorageManager.hiddenFileFilter)) {
+ fs.rename(status.getPath(), oldTableDir);
+ }
+
movedToOldTable = fs.exists(oldTableDir);
} else { // if the parent does not exist, make its parent directory.
- fs.mkdirs(finalOutputDir.getParent());
+ fs.mkdirs(finalOutputDir);
}
- fs.rename(stagingResultDir, finalOutputDir);
+ // Move the results to the final output dir.
+ for (FileStatus status : fs.listStatus(stagingResultDir)) {
+ fs.rename(status.getPath(), finalOutputDir);
+ }
+
+ // Check the final output dir
committed = fs.exists(finalOutputDir);
+
} catch (IOException ioe) {
// recover the old table
if (movedToOldTable && !committed) {
- fs.rename(oldTableDir, finalOutputDir);
+
+ // if commit is failed, recover the old data
+ for (FileStatus status : fs.listStatus(finalOutputDir, StorageManager.hiddenFileFilter)) {
+ fs.delete(status.getPath(), true);
+ }
+
+ for (FileStatus status : fs.listStatus(oldTableDir)) {
+ fs.rename(status.getPath(), finalOutputDir);
+ }
}
+
+ throw new IOException(ioe.getMessage());
}
}
} else {
@@ -560,13 +587,24 @@ public class Query implements EventHandler<QueryEvent> {
}
}
} else { // CREATE TABLE AS SELECT (CTAS)
- fs.rename(stagingResultDir, finalOutputDir);
+ if (fs.exists(finalOutputDir)) {
+ for (FileStatus status : fs.listStatus(stagingResultDir)) {
+ fs.rename(status.getPath(), finalOutputDir);
+ }
+ } else {
+ fs.rename(stagingResultDir, finalOutputDir);
+ }
LOG.info("Moved from the staging dir to the output directory '" + finalOutputDir);
}
}
- } catch (IOException e) {
- // TODO report to client
- e.printStackTrace();
+
+ // remove the staging directory if the final output dir is given.
+ Path stagingDirRoot = queryContext.getStagingDir().getParent();
+ fs.delete(stagingDirRoot, true);
+
+ } catch (Throwable t) {
+ LOG.error(t);
+ throw new IOException(t);
}
} else {
finalOutputDir = new Path(queryContext.getStagingDir(), TajoConstants.RESULT_DIR_NAME);
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
index 5cf3df5..75d8ab6 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
@@ -58,6 +58,7 @@ import org.apache.tajo.rpc.CallFuture;
import org.apache.tajo.rpc.NettyClientBase;
import org.apache.tajo.rpc.RpcConnectionPool;
import org.apache.tajo.storage.StorageManager;
+import org.apache.tajo.storage.StorageUtil;
import org.apache.tajo.util.HAServiceUtil;
import org.apache.tajo.util.metrics.TajoMetrics;
import org.apache.tajo.util.metrics.reporter.MetricsConsoleReporter;
@@ -79,6 +80,8 @@ public class QueryMasterTask extends CompositeService {
final public static FsPermission STAGING_DIR_PERMISSION =
FsPermission.createImmutable((short) 0700); // rwx--------
+ public static final String TMP_STAGING_DIR_PREFIX = ".staging";
+
private QueryId queryId;
private Session session;
@@ -399,8 +402,7 @@ public class QueryMasterTask extends CompositeService {
try {
- stagingDir = initStagingDir(systemConf, defaultFS, queryId.toString());
- defaultFS.mkdirs(new Path(stagingDir, TajoConstants.RESULT_DIR_NAME));
+ stagingDir = initStagingDir(systemConf, queryId.toString(), queryContext);
// Create a subdirectories
LOG.info("The staging dir '" + stagingDir + "' is created.");
@@ -423,7 +425,7 @@ public class QueryMasterTask extends CompositeService {
* It initializes the final output and staging directory and sets
* them to variables.
*/
- public static Path initStagingDir(TajoConf conf, FileSystem fs, String queryId) throws IOException {
+ public static Path initStagingDir(TajoConf conf, String queryId, QueryContext context) throws IOException {
String realUser;
String currentUser;
@@ -432,13 +434,21 @@ public class QueryMasterTask extends CompositeService {
realUser = ugi.getShortUserName();
currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
- Path stagingDir = null;
+ FileSystem fs;
+ Path stagingDir;
////////////////////////////////////////////
// Create Output Directory
////////////////////////////////////////////
- stagingDir = new Path(TajoConf.getStagingDir(conf), queryId);
+ if (context.isCreateTable() || context.isInsert()) {
+ stagingDir = StorageUtil.concatPath(context.getOutputPath(), TMP_STAGING_DIR_PREFIX, queryId);
+ } else {
+ stagingDir = new Path(TajoConf.getDefaultRootStagingDir(conf), queryId);
+ }
+
+ // initializ
+ fs = stagingDir.getFileSystem(conf);
if (fs.exists(stagingDir)) {
throw new IOException("The staging directory '" + stagingDir + "' already exists");
@@ -462,6 +472,9 @@ public class QueryMasterTask extends CompositeService {
fs.setPermission(stagingDir, new FsPermission(STAGING_DIR_PERMISSION));
}
+ Path stagingResultDir = new Path(stagingDir, TajoConstants.RESULT_DIR_NAME);
+ fs.mkdirs(stagingResultDir);
+
return stagingDir;
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-core/src/main/resources/webapps/admin/index.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/index.jsp b/tajo-core/src/main/resources/webapps/admin/index.jsp
index 30cbf88..6778725 100644
--- a/tajo-core/src/main/resources/webapps/admin/index.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/index.jsp
@@ -131,7 +131,7 @@
<tr><td width='150'>Root dir:</td><td><%=TajoConf.getTajoRootDir(master.getContext().getConf())%></td></tr>
<tr><td width='150'>System dir:</td><td><%=TajoConf.getSystemDir(master.getContext().getConf())%></td></tr>
<tr><td width='150'>Warehouse dir:</td><td><%=TajoConf.getWarehouseDir(master.getContext().getConf())%></td></tr>
- <tr><td width='150'>Staging dir:</td><td><%=TajoConf.getStagingDir(master.getContext().getConf())%></td></tr>
+ <tr><td width='150'>Staging dir:</td><td><%=TajoConf.getDefaultRootStagingDir(master.getContext().getConf())%></td></tr>
<tr><td width='150'>Client Service:</td><td><%=NetUtils.normalizeInetSocketAddress(master.getTajoMasterClientService().getBindAddress())%></td></tr>
<tr><td width='150'>Catalog Service:</td><td><%=master.getCatalogServer().getCatalogServerName()%></td></tr>
<tr><td width='150'>Heap(Free/Total/Max): </td><td><%=Runtime.getRuntime().freeMemory()/1024/1024%> MB / <%=Runtime.getRuntime().totalMemory()/1024/1024%> MB / <%=Runtime.getRuntime().maxMemory()/1024/1024%> MB</td>
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
index 0e02079..0e89803 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java
@@ -28,6 +28,7 @@ import org.apache.tajo.TajoConstants;
import org.apache.tajo.TajoTestingCluster;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.CatalogUtil;
+import org.apache.tajo.util.CommonTestingUtil;
import org.apache.tajo.util.KeyValueSet;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
http://git-wip-us.apache.org/repos/asf/tajo/blob/b4adc18c/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
index 9c97a55..117f186 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java
@@ -29,6 +29,7 @@ import org.apache.tajo.QueryTestCaseBase;
import org.apache.tajo.catalog.CatalogService;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.TableDesc;
+import org.apache.tajo.util.CommonTestingUtil;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -114,8 +115,19 @@ public class TestInsertQuery extends QueryTestCaseBase {
@Test
public final void testInsertIntoLocation() throws Exception {
+ Path dfsPath = new Path("/tajo-data/testInsertIntoLocation");
+ assertTestInsertIntoLocation(dfsPath);
+ }
+
+ @Test
+ public final void testInsertIntoLocationDifferentFSs() throws Exception {
+ Path localPath = CommonTestingUtil.getTestDir();
+ assertTestInsertIntoLocation(localPath);
+ }
+
+ public final void assertTestInsertIntoLocation(Path path) throws Exception {
FileSystem fs = null;
- Path path = new Path("/tajo-data/testInsertIntoLocation");
+
try {
executeString("insert into location '" + path + "' select l_orderkey, l_partkey, l_linenumber from default.lineitem").close();