You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hj...@apache.org on 2014/11/16 02:13:41 UTC
[4/6] tajo git commit: TAJO-1131: Supports Inserting or Creating
table into the HBase mapped table.
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result b/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result
new file mode 100644
index 0000000..72013f2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result
@@ -0,0 +1,100 @@
+000, value0
+001, value1
+002, value2
+003, value3
+004, value4
+005, value5
+006, value6
+007, value7
+008, value8
+009, value9
+010, value10
+011, value11
+012, value12
+013, value13
+014, value14
+015, value15
+016, value16
+017, value17
+018, value18
+019, value19
+020, value20
+021, value21
+022, value22
+023, value23
+024, value24
+025, value25
+026, value26
+027, value27
+028, value28
+029, value29
+030, value30
+031, value31
+032, value32
+033, value33
+034, value34
+035, value35
+036, value36
+037, value37
+038, value38
+039, value39
+040, value40
+041, value41
+042, value42
+043, value43
+044, value44
+045, value45
+046, value46
+047, value47
+048, value48
+049, value49
+050, value50
+051, value51
+052, value52
+053, value53
+054, value54
+055, value55
+056, value56
+057, value57
+058, value58
+059, value59
+060, value60
+061, value61
+062, value62
+063, value63
+064, value64
+065, value65
+066, value66
+067, value67
+068, value68
+069, value69
+070, value70
+071, value71
+072, value72
+073, value73
+074, value74
+075, value75
+076, value76
+077, value77
+078, value78
+079, value79
+080, value80
+081, value81
+082, value82
+083, value83
+084, value84
+085, value85
+086, value86
+087, value87
+088, value88
+089, value89
+090, value90
+091, value91
+092, value92
+093, value93
+094, value94
+095, value95
+096, value96
+097, value97
+098, value98
+099, value99
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result
new file mode 100644
index 0000000..82d4fd2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result
@@ -0,0 +1,12 @@
+rk1,col2_key,col2_value,col3
+-------------------------------
+rk-0,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-0
+rk-1,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-1
+rk-2,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-2
+rk-3,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-3
+rk-4,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-4
+rk-5,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-5
+rk-6,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-6
+rk-7,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-7
+rk-8,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-8
+rk-9,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-9
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result b/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result
new file mode 100644
index 0000000..f38d238
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result
@@ -0,0 +1,38 @@
+rk,col1,col2,col3
+-------------------------------
+020,a-20,{"k1":"k1-20", "k2":"k2-20"},b-20
+021,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21
+022,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22
+023,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23
+024,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24
+025,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25
+026,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26
+027,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27
+028,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28
+029,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29
+030,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30
+031,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31
+032,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32
+033,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33
+034,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34
+035,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35
+036,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36
+037,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37
+038,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38
+039,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39
+040,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40
+041,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41
+042,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42
+043,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43
+044,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44
+045,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45
+046,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46
+047,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47
+048,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48
+049,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49
+050,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50
+051,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51
+052,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52
+053,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53
+054,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54
+055,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result
new file mode 100644
index 0000000..e0c97ef
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result
@@ -0,0 +1,3 @@
+1, 1996-03-13, {"": "N"}, 7706
+2, 1997-01-28, {"": "N"}, 1191
+3, 1994-02-02, {"": "R"}, 1798
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result
new file mode 100644
index 0000000..c55873a
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result
@@ -0,0 +1,100 @@
+0, value0
+1, value1
+2, value2
+3, value3
+4, value4
+5, value5
+6, value6
+7, value7
+8, value8
+9, value9
+1, value10
+1, value11
+1, value12
+1, value13
+1, value14
+1, value15
+1, value16
+1, value17
+1, value18
+1, value19
+2, value20
+2, value21
+2, value22
+2, value23
+2, value24
+2, value25
+2, value26
+2, value27
+2, value28
+2, value29
+3, value30
+3, value31
+3, value32
+3, value33
+3, value34
+3, value35
+3, value36
+3, value37
+3, value38
+3, value39
+4, value40
+4, value41
+4, value42
+4, value43
+4, value44
+4, value45
+4, value46
+4, value47
+4, value48
+4, value49
+5, value50
+5, value51
+5, value52
+5, value53
+5, value54
+5, value55
+5, value56
+5, value57
+5, value58
+5, value59
+6, value60
+6, value61
+6, value62
+6, value63
+6, value64
+6, value65
+6, value66
+6, value67
+6, value68
+6, value69
+7, value70
+7, value71
+7, value72
+7, value73
+7, value74
+7, value75
+7, value76
+7, value77
+7, value78
+7, value79
+8, value80
+8, value81
+8, value82
+8, value83
+8, value84
+8, value85
+8, value86
+8, value87
+8, value88
+8, value89
+9, value90
+9, value91
+9, value92
+9, value93
+9, value94
+9, value95
+9, value96
+9, value97
+9, value98
+9, value99
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result
new file mode 100644
index 0000000..a37593b
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result
@@ -0,0 +1,21 @@
+0, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-0"}
+1, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-1"}
+10, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-10"}
+11, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-11"}
+12, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-12"}
+13, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-13"}
+14, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-14"}
+15, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-15"}
+16, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-16"}
+17, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-17"}
+18, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-18"}
+19, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-19"}
+2, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-2"}
+20, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-20"}
+3, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-3"}
+4, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-4"}
+5, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-5"}
+6, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-6"}
+7, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-7"}
+8, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-8"}
+9, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-9"}
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result
new file mode 100644
index 0000000..72013f2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result
@@ -0,0 +1,100 @@
+000, value0
+001, value1
+002, value2
+003, value3
+004, value4
+005, value5
+006, value6
+007, value7
+008, value8
+009, value9
+010, value10
+011, value11
+012, value12
+013, value13
+014, value14
+015, value15
+016, value16
+017, value17
+018, value18
+019, value19
+020, value20
+021, value21
+022, value22
+023, value23
+024, value24
+025, value25
+026, value26
+027, value27
+028, value28
+029, value29
+030, value30
+031, value31
+032, value32
+033, value33
+034, value34
+035, value35
+036, value36
+037, value37
+038, value38
+039, value39
+040, value40
+041, value41
+042, value42
+043, value43
+044, value44
+045, value45
+046, value46
+047, value47
+048, value48
+049, value49
+050, value50
+051, value51
+052, value52
+053, value53
+054, value54
+055, value55
+056, value56
+057, value57
+058, value58
+059, value59
+060, value60
+061, value61
+062, value62
+063, value63
+064, value64
+065, value65
+066, value66
+067, value67
+068, value68
+069, value69
+070, value70
+071, value71
+072, value72
+073, value73
+074, value74
+075, value75
+076, value76
+077, value77
+078, value78
+079, value79
+080, value80
+081, value81
+082, value82
+083, value83
+084, value84
+085, value85
+086, value86
+087, value87
+088, value88
+089, value89
+090, value90
+091, value91
+092, value92
+093, value93
+094, value94
+095, value95
+096, value96
+097, value97
+098, value98
+099, value99
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result
new file mode 100644
index 0000000..be11042
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result
@@ -0,0 +1,100 @@
+0, value0
+1, value1
+10, value10
+11, value11
+12, value12
+13, value13
+14, value14
+15, value15
+16, value16
+17, value17
+18, value18
+19, value19
+2, value2
+20, value20
+21, value21
+22, value22
+23, value23
+24, value24
+25, value25
+26, value26
+27, value27
+28, value28
+29, value29
+3, value3
+30, value30
+31, value31
+32, value32
+33, value33
+34, value34
+35, value35
+36, value36
+37, value37
+38, value38
+39, value39
+4, value4
+40, value40
+41, value41
+42, value42
+43, value43
+44, value44
+45, value45
+46, value46
+47, value47
+48, value48
+49, value49
+5, value5
+50, value50
+51, value51
+52, value52
+53, value53
+54, value54
+55, value55
+56, value56
+57, value57
+58, value58
+59, value59
+6, value6
+60, value60
+61, value61
+62, value62
+63, value63
+64, value64
+65, value65
+66, value66
+67, value67
+68, value68
+69, value69
+7, value7
+70, value70
+71, value71
+72, value72
+73, value73
+74, value74
+75, value75
+76, value76
+77, value77
+78, value78
+79, value79
+8, value8
+80, value80
+81, value81
+82, value82
+83, value83
+84, value84
+85, value85
+86, value86
+87, value87
+88, value88
+89, value89
+9, value9
+90, value90
+91, value91
+92, value92
+93, value93
+94, value94
+95, value95
+96, value96
+97, value97
+98, value98
+99, value99
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result
new file mode 100644
index 0000000..0b740c8
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result
@@ -0,0 +1,100 @@
+000_100, value0
+001_101, value1
+002_102, value2
+003_103, value3
+004_104, value4
+005_105, value5
+006_106, value6
+007_107, value7
+008_108, value8
+009_109, value9
+010_110, value10
+011_111, value11
+012_112, value12
+013_113, value13
+014_114, value14
+015_115, value15
+016_116, value16
+017_117, value17
+018_118, value18
+019_119, value19
+020_120, value20
+021_121, value21
+022_122, value22
+023_123, value23
+024_124, value24
+025_125, value25
+026_126, value26
+027_127, value27
+028_128, value28
+029_129, value29
+030_130, value30
+031_131, value31
+032_132, value32
+033_133, value33
+034_134, value34
+035_135, value35
+036_136, value36
+037_137, value37
+038_138, value38
+039_139, value39
+040_140, value40
+041_141, value41
+042_142, value42
+043_143, value43
+044_144, value44
+045_145, value45
+046_146, value46
+047_147, value47
+048_148, value48
+049_149, value49
+050_150, value50
+051_151, value51
+052_152, value52
+053_153, value53
+054_154, value54
+055_155, value55
+056_156, value56
+057_157, value57
+058_158, value58
+059_159, value59
+060_160, value60
+061_161, value61
+062_162, value62
+063_163, value63
+064_164, value64
+065_165, value65
+066_166, value66
+067_167, value67
+068_168, value68
+069_169, value69
+070_170, value70
+071_171, value71
+072_172, value72
+073_173, value73
+074_174, value74
+075_175, value75
+076_176, value76
+077_177, value77
+078_178, value78
+079_179, value79
+080_180, value80
+081_181, value81
+082_182, value82
+083_183, value83
+084_184, value84
+085_185, value85
+086_186, value86
+087_187, value87
+088_188, value88
+089_189, value89
+090_190, value90
+091_191, value91
+092_192, value92
+093_193, value93
+094_194, value94
+095_195, value95
+096_196, value96
+097_197, value97
+098_198, value98
+099_199, value99
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result
new file mode 100644
index 0000000..72013f2
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result
@@ -0,0 +1,100 @@
+000, value0
+001, value1
+002, value2
+003, value3
+004, value4
+005, value5
+006, value6
+007, value7
+008, value8
+009, value9
+010, value10
+011, value11
+012, value12
+013, value13
+014, value14
+015, value15
+016, value16
+017, value17
+018, value18
+019, value19
+020, value20
+021, value21
+022, value22
+023, value23
+024, value24
+025, value25
+026, value26
+027, value27
+028, value28
+029, value29
+030, value30
+031, value31
+032, value32
+033, value33
+034, value34
+035, value35
+036, value36
+037, value37
+038, value38
+039, value39
+040, value40
+041, value41
+042, value42
+043, value43
+044, value44
+045, value45
+046, value46
+047, value47
+048, value48
+049, value49
+050, value50
+051, value51
+052, value52
+053, value53
+054, value54
+055, value55
+056, value56
+057, value57
+058, value58
+059, value59
+060, value60
+061, value61
+062, value62
+063, value63
+064, value64
+065, value65
+066, value66
+067, value67
+068, value68
+069, value69
+070, value70
+071, value71
+072, value72
+073, value73
+074, value74
+075, value75
+076, value76
+077, value77
+078, value78
+079, value79
+080, value80
+081, value81
+082, value82
+083, value83
+084, value84
+085, value85
+086, value86
+087, value87
+088, value88
+089, value89
+090, value90
+091, value91
+092, value92
+093, value93
+094, value94
+095, value95
+096, value96
+097, value97
+098, value98
+099, value99
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result
new file mode 100644
index 0000000..152b91b
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result
@@ -0,0 +1,4 @@
+1_1, 1996-03-13, N, 7706
+2_2, 1997-01-28, N, 1191
+3_2, 1994-02-02, R, 1798
+3_3, 1993-11-09, R, 6540
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result
new file mode 100644
index 0000000..1c22960
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result
@@ -0,0 +1,3 @@
+1, 1996-04-12, {"": "N"}, 7311
+2, 1997-01-28, {"": "N"}, 1191
+3, 1993-11-09, {"": "R"}, 6540
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result
new file mode 100644
index 0000000..fe4fcae
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result
@@ -0,0 +1,7 @@
+rk,col1,col2,col3,l_orderkey,l_linestatus
+-------------------------------
+001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O
+001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O
+002,a-2,{"k1":"k1-2", "k2":"k2-2"},2,2,O
+003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F
+003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result
new file mode 100644
index 0000000..4f5fd8f
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result
@@ -0,0 +1,102 @@
+rk,col1,col2,col3
+-------------------------------
+000,a-0,{"k1":"k1-0", "k2":"k2-0"},0
+001,a-1,{"k1":"k1-1", "k2":"k2-1"},1
+002,a-2,{"k1":"k1-2", "k2":"k2-2"},2
+003,a-3,{"k1":"k1-3", "k2":"k2-3"},3
+004,a-4,{"k1":"k1-4", "k2":"k2-4"},4
+005,a-5,{"k1":"k1-5", "k2":"k2-5"},5
+006,a-6,{"k1":"k1-6", "k2":"k2-6"},6
+007,a-7,{"k1":"k1-7", "k2":"k2-7"},7
+008,a-8,{"k1":"k1-8", "k2":"k2-8"},8
+009,a-9,{"k1":"k1-9", "k2":"k2-9"},9
+010,a-10,{"k1":"k1-10", "k2":"k2-10"},10
+011,a-11,{"k1":"k1-11", "k2":"k2-11"},11
+012,a-12,{"k1":"k1-12", "k2":"k2-12"},12
+013,a-13,{"k1":"k1-13", "k2":"k2-13"},13
+014,a-14,{"k1":"k1-14", "k2":"k2-14"},14
+015,a-15,{"k1":"k1-15", "k2":"k2-15"},15
+016,a-16,{"k1":"k1-16", "k2":"k2-16"},16
+017,a-17,{"k1":"k1-17", "k2":"k2-17"},17
+018,a-18,{"k1":"k1-18", "k2":"k2-18"},18
+019,a-19,{"k1":"k1-19", "k2":"k2-19"},19
+020,a-20,{"k1":"k1-20", "k2":"k2-20"},20
+021,a-21,{"k1":"k1-21", "k2":"k2-21"},21
+022,a-22,{"k1":"k1-22", "k2":"k2-22"},22
+023,a-23,{"k1":"k1-23", "k2":"k2-23"},23
+024,a-24,{"k1":"k1-24", "k2":"k2-24"},24
+025,a-25,{"k1":"k1-25", "k2":"k2-25"},25
+026,a-26,{"k1":"k1-26", "k2":"k2-26"},26
+027,a-27,{"k1":"k1-27", "k2":"k2-27"},27
+028,a-28,{"k1":"k1-28", "k2":"k2-28"},28
+029,a-29,{"k1":"k1-29", "k2":"k2-29"},29
+030,a-30,{"k1":"k1-30", "k2":"k2-30"},30
+031,a-31,{"k1":"k1-31", "k2":"k2-31"},31
+032,a-32,{"k1":"k1-32", "k2":"k2-32"},32
+033,a-33,{"k1":"k1-33", "k2":"k2-33"},33
+034,a-34,{"k1":"k1-34", "k2":"k2-34"},34
+035,a-35,{"k1":"k1-35", "k2":"k2-35"},35
+036,a-36,{"k1":"k1-36", "k2":"k2-36"},36
+037,a-37,{"k1":"k1-37", "k2":"k2-37"},37
+038,a-38,{"k1":"k1-38", "k2":"k2-38"},38
+039,a-39,{"k1":"k1-39", "k2":"k2-39"},39
+040,a-40,{"k1":"k1-40", "k2":"k2-40"},40
+041,a-41,{"k1":"k1-41", "k2":"k2-41"},41
+042,a-42,{"k1":"k1-42", "k2":"k2-42"},42
+043,a-43,{"k1":"k1-43", "k2":"k2-43"},43
+044,a-44,{"k1":"k1-44", "k2":"k2-44"},44
+045,a-45,{"k1":"k1-45", "k2":"k2-45"},45
+046,a-46,{"k1":"k1-46", "k2":"k2-46"},46
+047,a-47,{"k1":"k1-47", "k2":"k2-47"},47
+048,a-48,{"k1":"k1-48", "k2":"k2-48"},48
+049,a-49,{"k1":"k1-49", "k2":"k2-49"},49
+050,a-50,{"k1":"k1-50", "k2":"k2-50"},50
+051,a-51,{"k1":"k1-51", "k2":"k2-51"},51
+052,a-52,{"k1":"k1-52", "k2":"k2-52"},52
+053,a-53,{"k1":"k1-53", "k2":"k2-53"},53
+054,a-54,{"k1":"k1-54", "k2":"k2-54"},54
+055,a-55,{"k1":"k1-55", "k2":"k2-55"},55
+056,a-56,{"k1":"k1-56", "k2":"k2-56"},56
+057,a-57,{"k1":"k1-57", "k2":"k2-57"},57
+058,a-58,{"k1":"k1-58", "k2":"k2-58"},58
+059,a-59,{"k1":"k1-59", "k2":"k2-59"},59
+060,a-60,{"k1":"k1-60", "k2":"k2-60"},60
+061,a-61,{"k1":"k1-61", "k2":"k2-61"},61
+062,a-62,{"k1":"k1-62", "k2":"k2-62"},62
+063,a-63,{"k1":"k1-63", "k2":"k2-63"},63
+064,a-64,{"k1":"k1-64", "k2":"k2-64"},64
+065,a-65,{"k1":"k1-65", "k2":"k2-65"},65
+066,a-66,{"k1":"k1-66", "k2":"k2-66"},66
+067,a-67,{"k1":"k1-67", "k2":"k2-67"},67
+068,a-68,{"k1":"k1-68", "k2":"k2-68"},68
+069,a-69,{"k1":"k1-69", "k2":"k2-69"},69
+070,a-70,{"k1":"k1-70", "k2":"k2-70"},70
+071,a-71,{"k1":"k1-71", "k2":"k2-71"},71
+072,a-72,{"k1":"k1-72", "k2":"k2-72"},72
+073,a-73,{"k1":"k1-73", "k2":"k2-73"},73
+074,a-74,{"k1":"k1-74", "k2":"k2-74"},74
+075,a-75,{"k1":"k1-75", "k2":"k2-75"},75
+076,a-76,{"k1":"k1-76", "k2":"k2-76"},76
+077,a-77,{"k1":"k1-77", "k2":"k2-77"},77
+078,a-78,{"k1":"k1-78", "k2":"k2-78"},78
+079,a-79,{"k1":"k1-79", "k2":"k2-79"},79
+080,a-80,{"k1":"k1-80", "k2":"k2-80"},80
+081,a-81,{"k1":"k1-81", "k2":"k2-81"},81
+082,a-82,{"k1":"k1-82", "k2":"k2-82"},82
+083,a-83,{"k1":"k1-83", "k2":"k2-83"},83
+084,a-84,{"k1":"k1-84", "k2":"k2-84"},84
+085,a-85,{"k1":"k1-85", "k2":"k2-85"},85
+086,a-86,{"k1":"k1-86", "k2":"k2-86"},86
+087,a-87,{"k1":"k1-87", "k2":"k2-87"},87
+088,a-88,{"k1":"k1-88", "k2":"k2-88"},88
+089,a-89,{"k1":"k1-89", "k2":"k2-89"},89
+090,a-90,{"k1":"k1-90", "k2":"k2-90"},90
+091,a-91,{"k1":"k1-91", "k2":"k2-91"},91
+092,a-92,{"k1":"k1-92", "k2":"k2-92"},92
+093,a-93,{"k1":"k1-93", "k2":"k2-93"},93
+094,a-94,{"k1":"k1-94", "k2":"k2-94"},94
+095,a-95,{"k1":"k1-95", "k2":"k2-95"},95
+096,a-96,{"k1":"k1-96", "k2":"k2-96"},96
+097,a-97,{"k1":"k1-97", "k2":"k2-97"},97
+098,a-98,{"k1":"k1-98", "k2":"k2-98"},98
+099,a-99,{"k1":"k1-99", "k2":"k2-99"},99
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result
new file mode 100644
index 0000000..34369df
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result
@@ -0,0 +1,88 @@
+rk1,rk2,col3
+-------------------------------
+field1-21,field2-21,a-21
+field1-22,field2-22,a-22
+field1-23,field2-23,a-23
+field1-24,field2-24,a-24
+field1-25,field2-25,a-25
+field1-26,field2-26,a-26
+field1-27,field2-27,a-27
+field1-28,field2-28,a-28
+field1-29,field2-29,a-29
+field1-30,field2-30,a-30
+field1-31,field2-31,a-31
+field1-32,field2-32,a-32
+field1-33,field2-33,a-33
+field1-34,field2-34,a-34
+field1-35,field2-35,a-35
+field1-36,field2-36,a-36
+field1-37,field2-37,a-37
+field1-38,field2-38,a-38
+field1-39,field2-39,a-39
+field1-3,field2-3,a-3
+field1-40,field2-40,a-40
+field1-41,field2-41,a-41
+field1-42,field2-42,a-42
+field1-43,field2-43,a-43
+field1-44,field2-44,a-44
+field1-45,field2-45,a-45
+field1-46,field2-46,a-46
+field1-47,field2-47,a-47
+field1-48,field2-48,a-48
+field1-49,field2-49,a-49
+field1-4,field2-4,a-4
+field1-50,field2-50,a-50
+field1-51,field2-51,a-51
+field1-52,field2-52,a-52
+field1-53,field2-53,a-53
+field1-54,field2-54,a-54
+field1-55,field2-55,a-55
+field1-56,field2-56,a-56
+field1-57,field2-57,a-57
+field1-58,field2-58,a-58
+field1-59,field2-59,a-59
+field1-5,field2-5,a-5
+field1-60,field2-60,a-60
+field1-61,field2-61,a-61
+field1-62,field2-62,a-62
+field1-63,field2-63,a-63
+field1-64,field2-64,a-64
+field1-65,field2-65,a-65
+field1-66,field2-66,a-66
+field1-67,field2-67,a-67
+field1-68,field2-68,a-68
+field1-69,field2-69,a-69
+field1-6,field2-6,a-6
+field1-70,field2-70,a-70
+field1-71,field2-71,a-71
+field1-72,field2-72,a-72
+field1-73,field2-73,a-73
+field1-74,field2-74,a-74
+field1-75,field2-75,a-75
+field1-76,field2-76,a-76
+field1-77,field2-77,a-77
+field1-78,field2-78,a-78
+field1-79,field2-79,a-79
+field1-7,field2-7,a-7
+field1-80,field2-80,a-80
+field1-81,field2-81,a-81
+field1-82,field2-82,a-82
+field1-83,field2-83,a-83
+field1-84,field2-84,a-84
+field1-85,field2-85,a-85
+field1-86,field2-86,a-86
+field1-87,field2-87,a-87
+field1-88,field2-88,a-88
+field1-89,field2-89,a-89
+field1-8,field2-8,a-8
+field1-90,field2-90,a-90
+field1-91,field2-91,a-91
+field1-92,field2-92,a-92
+field1-93,field2-93,a-93
+field1-94,field2-94,a-94
+field1-95,field2-95,a-95
+field1-96,field2-96,a-96
+field1-97,field2-97,a-97
+field1-98,field2-98,a-98
+field1-99,field2-99,a-99
+field1-9,field2-9,a-9
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result
new file mode 100644
index 0000000..ef2f27a
--- /dev/null
+++ b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result
@@ -0,0 +1,88 @@
+rk,col1,col2,col3
+-------------------------------
+21,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21
+22,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22
+23,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23
+24,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24
+25,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25
+26,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26
+27,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27
+28,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28
+29,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29
+3,a-3,{"k1":"k1-3", "k2":"k2-3"},b-3
+30,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30
+31,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31
+32,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32
+33,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33
+34,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34
+35,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35
+36,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36
+37,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37
+38,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38
+39,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39
+4,a-4,{"k1":"k1-4", "k2":"k2-4"},b-4
+40,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40
+41,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41
+42,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42
+43,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43
+44,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44
+45,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45
+46,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46
+47,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47
+48,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48
+49,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49
+5,a-5,{"k1":"k1-5", "k2":"k2-5"},b-5
+50,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50
+51,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51
+52,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52
+53,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53
+54,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54
+55,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55
+56,a-56,{"k1":"k1-56", "k2":"k2-56"},b-56
+57,a-57,{"k1":"k1-57", "k2":"k2-57"},b-57
+58,a-58,{"k1":"k1-58", "k2":"k2-58"},b-58
+59,a-59,{"k1":"k1-59", "k2":"k2-59"},b-59
+6,a-6,{"k1":"k1-6", "k2":"k2-6"},b-6
+60,a-60,{"k1":"k1-60", "k2":"k2-60"},b-60
+61,a-61,{"k1":"k1-61", "k2":"k2-61"},b-61
+62,a-62,{"k1":"k1-62", "k2":"k2-62"},b-62
+63,a-63,{"k1":"k1-63", "k2":"k2-63"},b-63
+64,a-64,{"k1":"k1-64", "k2":"k2-64"},b-64
+65,a-65,{"k1":"k1-65", "k2":"k2-65"},b-65
+66,a-66,{"k1":"k1-66", "k2":"k2-66"},b-66
+67,a-67,{"k1":"k1-67", "k2":"k2-67"},b-67
+68,a-68,{"k1":"k1-68", "k2":"k2-68"},b-68
+69,a-69,{"k1":"k1-69", "k2":"k2-69"},b-69
+7,a-7,{"k1":"k1-7", "k2":"k2-7"},b-7
+70,a-70,{"k1":"k1-70", "k2":"k2-70"},b-70
+71,a-71,{"k1":"k1-71", "k2":"k2-71"},b-71
+72,a-72,{"k1":"k1-72", "k2":"k2-72"},b-72
+73,a-73,{"k1":"k1-73", "k2":"k2-73"},b-73
+74,a-74,{"k1":"k1-74", "k2":"k2-74"},b-74
+75,a-75,{"k1":"k1-75", "k2":"k2-75"},b-75
+76,a-76,{"k1":"k1-76", "k2":"k2-76"},b-76
+77,a-77,{"k1":"k1-77", "k2":"k2-77"},b-77
+78,a-78,{"k1":"k1-78", "k2":"k2-78"},b-78
+79,a-79,{"k1":"k1-79", "k2":"k2-79"},b-79
+8,a-8,{"k1":"k1-8", "k2":"k2-8"},b-8
+80,a-80,{"k1":"k1-80", "k2":"k2-80"},b-80
+81,a-81,{"k1":"k1-81", "k2":"k2-81"},b-81
+82,a-82,{"k1":"k1-82", "k2":"k2-82"},b-82
+83,a-83,{"k1":"k1-83", "k2":"k2-83"},b-83
+84,a-84,{"k1":"k1-84", "k2":"k2-84"},b-84
+85,a-85,{"k1":"k1-85", "k2":"k2-85"},b-85
+86,a-86,{"k1":"k1-86", "k2":"k2-86"},b-86
+87,a-87,{"k1":"k1-87", "k2":"k2-87"},b-87
+88,a-88,{"k1":"k1-88", "k2":"k2-88"},b-88
+89,a-89,{"k1":"k1-89", "k2":"k2-89"},b-89
+9,a-9,{"k1":"k1-9", "k2":"k2-9"},b-9
+90,a-90,{"k1":"k1-90", "k2":"k2-90"},b-90
+91,a-91,{"k1":"k1-91", "k2":"k2-91"},b-91
+92,a-92,{"k1":"k1-92", "k2":"k2-92"},b-92
+93,a-93,{"k1":"k1-93", "k2":"k2-93"},b-93
+94,a-94,{"k1":"k1-94", "k2":"k2-94"},b-94
+95,a-95,{"k1":"k1-95", "k2":"k2-95"},b-95
+96,a-96,{"k1":"k1-96", "k2":"k2-96"},b-96
+97,a-97,{"k1":"k1-97", "k2":"k2-97"},b-97
+98,a-98,{"k1":"k1-98", "k2":"k2-98"},b-98
+99,a-99,{"k1":"k1-99", "k2":"k2-99"},b-99
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-dist/src/main/bin/tajo
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo
index d4e99f9..8a9c17a 100755
--- a/tajo-dist/src/main/bin/tajo
+++ b/tajo-dist/src/main/bin/tajo
@@ -226,6 +226,21 @@ HDFS_LIBRARY_PATH="${HADOOP_HOME}/lib/native/"
##############################################################################
##############################################################################
+# Set HBase CLASSPATH
+##############################################################################
+if [ "$HBASE_HOME" != "" ]; then
+ for f in ${HBASE_HOME}/lib/hbase-*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+ for f in ${HBASE_HOME}/lib/htrace-*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+fi
+##############################################################################
+# HBase Configuration End
+##############################################################################
+
+##############################################################################
# Find and Set Hive CLASSPATH
##############################################################################
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-dist/src/main/conf/tajo-env.sh
----------------------------------------------------------------------
diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh
index 92d00bd..064a82f 100755
--- a/tajo-dist/src/main/conf/tajo-env.sh
+++ b/tajo-dist/src/main/conf/tajo-env.sh
@@ -24,6 +24,9 @@
# Hadoop home. Required
# export HADOOP_HOME=
+# HBase home. optional
+# export HBASE_HOME=
+
# The java implementation to use. Required.
# export JAVA_HOME=/usr/java/default
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
index bcc3bce..750e64e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
@@ -44,6 +44,7 @@ import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Set;
import java.util.Stack;
@@ -86,6 +87,12 @@ public class LogicalOptimizer {
}
}
+ public void addRuleAfterToJoinOpt(RewriteRule rewriteRule) {
+ if (rewriteRule != null) {
+ rulesAfterToJoinOpt.addRewriteRule(rewriteRule);
+ }
+ }
+
public LogicalNode optimize(LogicalPlan plan) throws PlanningException {
return optimize(null, plan);
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java
index a697f9f..2e5e159 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java
@@ -25,10 +25,16 @@ import org.apache.tajo.plan.PlanString;
import org.apache.tajo.util.TUtil;
public final class SortNode extends UnaryNode implements Cloneable {
+ public static enum SortPurpose {
+ NORMAL,
+ STORAGE_SPECIFIED
+ }
@Expose private SortSpec [] sortKeys;
+ @Expose private SortPurpose sortPurpose;
public SortNode(int pid) {
super(pid, NodeType.SORT);
+ sortPurpose = SortPurpose.NORMAL;
}
public void setSortSpecs(SortSpec[] sortSpecs) {
@@ -65,8 +71,7 @@ public final class SortNode extends UnaryNode implements Cloneable {
PlanString planStr = new PlanString(this);
StringBuilder sb = new StringBuilder("Sort Keys: ");
for (int i = 0; i < sortKeys.length; i++) {
- sb.append(sortKeys[i].getSortKey().getSimpleName()).append(" ")
- .append(sortKeys[i].isAscending() ? "asc" : "desc");
+ sb.append(sortKeys[i].toString());
if( i < sortKeys.length - 1) {
sb.append(",");
}
@@ -78,8 +83,7 @@ public final class SortNode extends UnaryNode implements Cloneable {
public String toString() {
StringBuilder sb = new StringBuilder("Sort [key= ");
for (int i = 0; i < sortKeys.length; i++) {
- sb.append(sortKeys[i].getSortKey().getQualifiedName()).append(" ")
- .append(sortKeys[i].isAscending() ? "asc" : "desc");
+ sb.append(sortKeys[i].toString());
if(i < sortKeys.length - 1) {
sb.append(",");
}
@@ -91,4 +95,12 @@ public final class SortNode extends UnaryNode implements Cloneable {
return sb.toString()+"\n"
+ getChild().toString();
}
+
+ public SortPurpose getSortPurpose() {
+ return sortPurpose;
+ }
+
+ public void setSortPurpose(SortPurpose sortPurpose) {
+ this.sortPurpose = sortPurpose;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
index 14b24a7..46fda2e 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java
@@ -306,7 +306,7 @@ public class PartitionedTableRewriter implements RewriteRule {
boolean sameVariable =
EvalTreeUtil.findUniqueColumns(orEval.getLeftExpr())
- .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr()));
+ .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr()));
return indexable && sameVariable;
} else {
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
index 6868b6c..a7db188 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
@@ -23,11 +23,9 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.tajo.algebra.*;
import org.apache.tajo.annotation.Nullable;
-import org.apache.tajo.catalog.Column;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.SchemaUtil;
-import org.apache.tajo.catalog.SortSpec;
+import org.apache.tajo.catalog.*;
import org.apache.tajo.catalog.proto.CatalogProtos;
+import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.common.TajoDataTypes.DataType;
import org.apache.tajo.plan.*;
import org.apache.tajo.plan.expr.*;
@@ -37,6 +35,7 @@ import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor;
import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor;
import org.apache.tajo.util.TUtil;
+import java.io.IOException;
import java.util.*;
public class PlannerUtil {
@@ -776,4 +775,109 @@ public class PlannerUtil {
return explains.toString();
}
+
+ public static boolean isFileStorageType(String storageType) {
+ if (storageType.equalsIgnoreCase("hbase")) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public static boolean isFileStorageType(StoreType storageType) {
+ if (storageType== StoreType.HBASE) {
+ return false;
+ } else {
+ return true;
+ }
+ }
+
+ public static StoreType getStoreType(LogicalPlan plan) {
+ LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+ NodeType nodeType = rootNode.getChild().getType();
+ if (nodeType == NodeType.CREATE_TABLE) {
+ return ((CreateTableNode)rootNode.getChild()).getStorageType();
+ } else if (nodeType == NodeType.INSERT) {
+ return ((InsertNode)rootNode.getChild()).getStorageType();
+ } else {
+ return null;
+ }
+ }
+
+ public static String getStoreTableName(LogicalPlan plan) {
+ LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+ NodeType nodeType = rootNode.getChild().getType();
+ if (nodeType == NodeType.CREATE_TABLE) {
+ return ((CreateTableNode)rootNode.getChild()).getTableName();
+ } else if (nodeType == NodeType.INSERT) {
+ return ((InsertNode)rootNode.getChild()).getTableName();
+ } else {
+ return null;
+ }
+ }
+
+ public static TableDesc getTableDesc(CatalogService catalog, LogicalNode node) throws IOException {
+ if (node.getType() == NodeType.CREATE_TABLE) {
+ return createTableDesc((CreateTableNode)node);
+ }
+ String tableName = null;
+ InsertNode insertNode = null;
+ if (node.getType() == NodeType.INSERT) {
+ insertNode = (InsertNode)node;
+ tableName = insertNode.getTableName();
+ } else {
+ return null;
+ }
+
+ if (tableName != null) {
+ String[] tableTokens = tableName.split("\\.");
+ if (tableTokens.length >= 2) {
+ if (catalog.existsTable(tableTokens[0], tableTokens[1])) {
+ return catalog.getTableDesc(tableTokens[0], tableTokens[1]);
+ }
+ }
+ } else {
+ if (insertNode.getPath() != null) {
+ //insert ... location
+ return createTableDesc(insertNode);
+ }
+ }
+ return null;
+ }
+
+ private static TableDesc createTableDesc(CreateTableNode createTableNode) {
+ TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions());
+
+ TableDesc tableDescTobeCreated =
+ new TableDesc(
+ createTableNode.getTableName(),
+ createTableNode.getTableSchema(),
+ meta,
+ createTableNode.getPath());
+
+ tableDescTobeCreated.setExternal(createTableNode.isExternal());
+
+ if (createTableNode.hasPartition()) {
+ tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod());
+ }
+
+ return tableDescTobeCreated;
+ }
+
+ private static TableDesc createTableDesc(InsertNode insertNode) {
+ TableMeta meta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions());
+
+ TableDesc tableDescTobeCreated =
+ new TableDesc(
+ insertNode.getTableName(),
+ insertNode.getTableSchema(),
+ meta,
+ insertNode.getPath());
+
+ if (insertNode.hasPartition()) {
+ tableDescTobeCreated.setPartitionMethod(insertNode.getPartitionMethod());
+ }
+
+ return tableDescTobeCreated;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
index 95e0f30..5356c57 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java
@@ -29,8 +29,10 @@ import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.plan.util.ExprFinder;
import org.apache.tajo.plan.PlanningException;
import org.apache.tajo.plan.algebra.BaseAlgebraVisitor;
+import org.apache.tajo.plan.util.PlannerUtil;
import org.apache.tajo.util.TUtil;
+import java.io.IOException;
import java.util.Set;
import java.util.Stack;
@@ -277,6 +279,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer
context.state.addVerification(String.format("relation \"%s\" does not exist", qualifiedName));
return null;
}
+
if (table.hasPartition()) {
int columnSize = table.getSchema().getColumns().size();
columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size();
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-project/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml
index 5493095..333cda0 100644
--- a/tajo-project/pom.xml
+++ b/tajo-project/pom.xml
@@ -36,6 +36,7 @@
<hadoop.version>2.5.1</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<tajo.version>0.9.1-SNAPSHOT</tajo.version>
+ <hbase.version>0.98.7-hadoop2</hbase.version>
<tajo.root>${project.parent.relativePath}/..</tajo.root>
<extra.source.path>src/main/hadoop-${hadoop.version}</extra.source.path>
</properties>
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml
index 680e520..c126afd 100644
--- a/tajo-storage/pom.xml
+++ b/tajo-storage/pom.xml
@@ -131,6 +131,7 @@
<argument>--proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto</argument>
<argument>--java_out=target/generated-sources/proto</argument>
<argument>src/main/proto/IndexProtos.proto</argument>
+ <argument>src/main/proto/StorageFragmentProtos.proto</argument>
</arguments>
</configuration>
<goals>
@@ -176,6 +177,10 @@
<groupId>org.apache.tajo</groupId>
<artifactId>tajo-catalog-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.tajo</groupId>
+ <artifactId>tajo-plan</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.avro</groupId>
@@ -291,6 +296,12 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</dependency>
@@ -315,6 +326,18 @@
<version>${parquet.format.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${hbase.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
index dff337a..1e2b0f3 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.*;
+import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.proto.CatalogProtos;
@@ -76,9 +77,10 @@ public class CSVFile {
private NonSyncByteArrayOutputStream os = new NonSyncByteArrayOutputStream(BUFFER_SIZE);
private SerializerDeserializer serde;
- public CSVAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path) throws IOException {
- super(conf, schema, meta, path);
- this.fs = path.getFileSystem(conf);
+ public CSVAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId,
+ final Schema schema, final TableMeta meta, final Path workDir) throws IOException {
+ super(conf, taskAttemptId, schema, meta, workDir);
+ this.fs = workDir.getFileSystem(conf);
this.meta = meta;
this.schema = schema;
this.delimiter = StringEscapeUtils.unescapeJava(
@@ -99,7 +101,7 @@ public class CSVFile {
@Override
public void init() throws IOException {
if (!fs.exists(path.getParent())) {
- throw new FileNotFoundException(path.toString());
+ throw new FileNotFoundException(path.getParent().toString());
}
//determine the intermediate file type
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
index 04278e9..117d3da 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java
@@ -18,28 +18,48 @@
package org.apache.tajo.storage;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
+import org.apache.tajo.conf.TajoConf;
import java.io.IOException;
public abstract class FileAppender implements Appender {
+ private static final Log LOG = LogFactory.getLog(FileAppender.class);
+
protected boolean inited = false;
protected final Configuration conf;
protected final TableMeta meta;
protected final Schema schema;
- protected final Path path;
+ protected final Path workDir;
+ protected final QueryUnitAttemptId taskAttemptId;
protected boolean enabledStats;
-
- public FileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) {
+ protected Path path;
+
+ public FileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema,
+ TableMeta meta, Path workDir) {
this.conf = conf;
this.meta = meta;
this.schema = schema;
- this.path = path;
+ this.workDir = workDir;
+ this.taskAttemptId = taskAttemptId;
+
+ try {
+ if (taskAttemptId != null) {
+ this.path = StorageManager.getFileStorageManager((TajoConf) conf).getAppenderFilePath(taskAttemptId, workDir);
+ } else {
+ this.path = workDir;
+ }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
}
public void init() throws IOException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java
index c925cd8..ee24b27 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java
@@ -18,6 +18,7 @@
package org.apache.tajo.storage;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
@@ -26,35 +27,67 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.tajo.TajoConstants;
-import org.apache.tajo.catalog.CatalogUtil;
-import org.apache.tajo.catalog.Schema;
-import org.apache.tajo.catalog.TableDesc;
-import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.catalog.proto.CatalogProtos;
-import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
+import org.apache.tajo.*;
+import org.apache.tajo.catalog.*;
+import org.apache.tajo.catalog.proto.CatalogProtos.StoreType;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.conf.TajoConf;
-import org.apache.tajo.conf.TajoConf.ConfVars;
+import org.apache.tajo.plan.logical.LogicalNode;
+import org.apache.tajo.plan.logical.ScanNode;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.Fragment;
-import org.apache.tajo.storage.fragment.FragmentConvertor;
import org.apache.tajo.util.*;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.net.URI;
+import java.text.NumberFormat;
import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
public class FileStorageManager extends StorageManager {
private final Log LOG = LogFactory.getLog(FileStorageManager.class);
+ static final String OUTPUT_FILE_PREFIX="part-";
+ static final ThreadLocal<NumberFormat> OUTPUT_FILE_FORMAT_SUBQUERY =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(2);
+ return fmt;
+ }
+ };
+ static final ThreadLocal<NumberFormat> OUTPUT_FILE_FORMAT_TASK =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(6);
+ return fmt;
+ }
+ };
+
+ static final ThreadLocal<NumberFormat> OUTPUT_FILE_FORMAT_SEQ =
+ new ThreadLocal<NumberFormat>() {
+ @Override
+ public NumberFormat initialValue() {
+ NumberFormat fmt = NumberFormat.getInstance();
+ fmt.setGroupingUsed(false);
+ fmt.setMinimumIntegerDigits(3);
+ return fmt;
+ }
+ };
+
protected FileSystem fs;
protected Path tableBaseDir;
protected boolean blocksMetadataEnabled;
private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0));
+ public FileStorageManager(StoreType storeType) {
+ super(storeType);
+ }
+
@Override
protected void storageInit() throws IOException {
this.tableBaseDir = TajoConf.getWarehouseDir(conf);
@@ -114,47 +147,10 @@ public class FileStorageManager extends StorageManager {
return new Path(tableBaseDir, tableName);
}
- public Appender getAppender(TableMeta meta, Schema schema, Path path)
+ @VisibleForTesting
+ public Appender getAppender(TableMeta meta, Schema schema, Path filePath)
throws IOException {
- Appender appender;
-
- Class<? extends FileAppender> appenderClass;
-
- String handlerName = meta.getStoreType().name().toLowerCase();
- appenderClass = APPENDER_HANDLER_CACHE.get(handlerName);
- if (appenderClass == null) {
- appenderClass = conf.getClass(
- String.format("tajo.storage.appender-handler.%s.class",
- meta.getStoreType().name().toLowerCase()), null,
- FileAppender.class);
- APPENDER_HANDLER_CACHE.put(handlerName, appenderClass);
- }
-
- if (appenderClass == null) {
- throw new IOException("Unknown Storage Type: " + meta.getStoreType());
- }
-
- appender = newAppenderInstance(appenderClass, conf, meta, schema, path);
-
- return appender;
- }
-
- public TableMeta getTableMeta(Path tablePath) throws IOException {
- TableMeta meta;
-
- FileSystem fs = tablePath.getFileSystem(conf);
- Path tableMetaPath = new Path(tablePath, ".meta");
- if (!fs.exists(tableMetaPath)) {
- throw new FileNotFoundException(".meta file not found in " + tablePath.toString());
- }
-
- FSDataInputStream tableMetaIn = fs.open(tableMetaPath);
-
- CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) org.apache.tajo.util.FileUtil.loadProto(tableMetaIn,
- CatalogProtos.TableProto.getDefaultInstance());
- meta = new TableMeta(tableProto);
-
- return meta;
+ return getAppender(null, null, meta, schema, filePath);
}
public FileFragment[] split(String tableName) throws IOException {
@@ -278,6 +274,23 @@ public class FileStorageManager extends StorageManager {
}
};
+ public Path getAppenderFilePath(QueryUnitAttemptId taskAttemptId, Path workDir) {
+ if (taskAttemptId == null) {
+ // For testcase
+ return workDir;
+ }
+ // The final result of a task will be written in a file named part-ss-nnnnnnn,
+ // where ss is the subquery id associated with this task, and nnnnnn is the task id.
+ Path outFilePath = StorageUtil.concatPath(workDir, TajoConstants.RESULT_DIR_NAME,
+ OUTPUT_FILE_PREFIX +
+ OUTPUT_FILE_FORMAT_SUBQUERY.get().format(taskAttemptId.getQueryUnitId().getExecutionBlockId().getId()) + "-" +
+ OUTPUT_FILE_FORMAT_TASK.get().format(taskAttemptId.getQueryUnitId().getId()) + "-" +
+ OUTPUT_FILE_FORMAT_SEQ.get().format(0));
+ LOG.info("Output File Path: " + outFilePath);
+
+ return outFilePath;
+ }
+
/**
* Proxy PathFilter that accepts a path only if all filters given in the
* constructor do. Used by the listPaths() to apply the built-in
@@ -639,12 +652,12 @@ public class FileStorageManager extends StorageManager {
}
@Override
- public List<Fragment> getSplits(String tableName, TableDesc table) throws IOException {
+ public List<Fragment> getSplits(String tableName, TableDesc table, ScanNode scanNode) throws IOException {
return getSplits(tableName, table.getMeta(), table.getSchema(), table.getPath());
}
@Override
- public void createTable(TableDesc tableDesc) throws IOException {
+ public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException {
if (!tableDesc.isExternal()) {
String [] splitted = CatalogUtil.splitFQTableName(tableDesc.getName());
String databaseName = splitted[0];
@@ -699,4 +712,120 @@ public class FileStorageManager extends StorageManager {
}
}
+ @Override
+ public List<Fragment> getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException {
+ // Listing table data file which is not empty.
+ // If the table is a partitioned table, return file list which has same partition key.
+ FileSystem fs = tableDesc.getPath().getFileSystem(conf);
+
+ List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>();
+ if (fs.exists(tableDesc.getPath())) {
+ getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, currentPage, numFragments,
+ new AtomicInteger(0));
+ }
+
+ List<Fragment> fragments = new ArrayList<Fragment>();
+
+ //In the case of partitioned table, return same partition key data files.
+ int numPartitionColumns = 0;
+ if (tableDesc.hasPartition()) {
+ numPartitionColumns = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size();
+ }
+ String[] previousPartitionPathNames = null;
+ for (FileStatus eachFile: nonZeroLengthFiles) {
+ FileFragment fileFragment = new FileFragment(tableDesc.getName(), eachFile.getPath(), 0, eachFile.getLen(), null);
+
+ if (numPartitionColumns > 0) {
+ // finding partition key;
+ Path filePath = fileFragment.getPath();
+ Path parentPath = filePath;
+ String[] parentPathNames = new String[numPartitionColumns];
+ for (int i = 0; i < numPartitionColumns; i++) {
+ parentPath = parentPath.getParent();
+ parentPathNames[numPartitionColumns - i - 1] = parentPath.getName();
+ }
+
+ // If current partitionKey == previousPartitionKey, add to result.
+ if (previousPartitionPathNames == null) {
+ fragments.add(fileFragment);
+ } else if (previousPartitionPathNames != null && Arrays.equals(previousPartitionPathNames, parentPathNames)) {
+ fragments.add(fileFragment);
+ } else {
+ break;
+ }
+ previousPartitionPathNames = parentPathNames;
+ } else {
+ fragments.add(fileFragment);
+ }
+ }
+
+ return fragments;
+ }
+
+ private void getNonZeroLengthDataFiles(FileSystem fs, Path path, List<FileStatus> result,
+ int startFileIndex, int numResultFiles,
+ AtomicInteger currentFileIndex) throws IOException {
+ if (fs.isDirectory(path)) {
+ FileStatus[] files = fs.listStatus(path, FileStorageManager.hiddenFileFilter);
+ if (files != null && files.length > 0) {
+ for (FileStatus eachFile : files) {
+ if (result.size() >= numResultFiles) {
+ return;
+ }
+ if (eachFile.isDirectory()) {
+ getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles,
+ currentFileIndex);
+ } else if (eachFile.isFile() && eachFile.getLen() > 0) {
+ if (currentFileIndex.get() >= startFileIndex) {
+ result.add(eachFile);
+ }
+ currentFileIndex.incrementAndGet();
+ }
+ }
+ }
+ } else {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ if (fileStatus != null && fileStatus.getLen() > 0) {
+ if (currentFileIndex.get() >= startFileIndex) {
+ result.add(fileStatus);
+ }
+ currentFileIndex.incrementAndGet();
+ if (result.size() >= numResultFiles) {
+ return;
+ }
+ }
+ }
+ }
+
+ @Override
+ public StorageProperty getStorageProperty() {
+ StorageProperty storageProperty = new StorageProperty();
+ storageProperty.setSortedInsert(false);
+ if (storeType == StoreType.RAW) {
+ storageProperty.setSupportsInsertInto(false);
+ } else {
+ storageProperty.setSupportsInsertInto(true);
+ }
+
+ return storageProperty;
+ }
+
+ @Override
+ public void closeStorageManager() {
+ }
+
+ @Override
+ public void beforeInsertOrCATS(LogicalNode node) throws IOException {
+ }
+
+ @Override
+ public void rollbackOutputCommit(LogicalNode node) throws IOException {
+ }
+
+ @Override
+ public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc,
+ Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange)
+ throws IOException {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
index ece31dd..33a9233 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java
@@ -31,7 +31,6 @@ import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.conf.TajoConf.ConfVars;
import org.apache.tajo.util.Pair;
-import org.apache.tajo.storage.StorageManager;
import java.io.IOException;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
index 89e59d0..66b3667 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java
@@ -64,8 +64,9 @@ public class MergeScanner implements Scanner {
long numBytes = 0;
for (Fragment eachFileFragment: rawFragmentList) {
- if (eachFileFragment.getLength() > 0) {
- numBytes += eachFileFragment.getLength();
+ long fragmentLength = StorageManager.getFragmentLength((TajoConf)conf, eachFileFragment);
+ if (fragmentLength > 0) {
+ numBytes += fragmentLength;
fragments.add(eachFileFragment);
}
}
@@ -121,6 +122,9 @@ public class MergeScanner implements Scanner {
@Override
public void reset() throws IOException {
this.iterator = fragments.iterator();
+ if (currentScanner != null) {
+ currentScanner.close();
+ }
this.currentScanner = getNextScanner();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
index de3912a..5a3f32d 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -82,7 +83,6 @@ public class RawFile {
} catch (IllegalArgumentException iae) {
throw new IOException(iae);
}
-
fis = new FileInputStream(file);
channel = fis.getChannel();
filePosition = startOffset = fragment.getStartKey();
@@ -466,8 +466,9 @@ public class RawFile {
private TableStatistics stats;
- public RawFileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException {
- super(conf, schema, meta, path);
+ public RawFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId,
+ Schema schema, TableMeta meta, Path workDir) throws IOException {
+ super(conf, taskAttemptId, schema, meta, workDir);
}
public void init() throws IOException {
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
index 640cae2..efb23eb 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
@@ -313,9 +314,10 @@ public class RowFile {
// statistics
private TableStatistics stats;
- public RowFileAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path)
+ public RowFileAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId,
+ final Schema schema, final TableMeta meta, final Path workDir)
throws IOException {
- super(conf, schema, meta, path);
+ super(conf, taskAttemptId, schema, meta, workDir);
}
public void init() throws IOException {