You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by pb...@apache.org on 2018/11/26 10:39:36 UTC
[07/12] phoenix git commit: PHOENIX-4872: BulkLoad has bug when
loading on single-cell-array-with-offsets table.
PHOENIX-4872: BulkLoad has bug when loading on single-cell-array-with-offsets table.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/e453b772
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/e453b772
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/e453b772
Branch: refs/heads/4.x-cdh5.15
Commit: e453b772fa399ebf1b34d67c11bb7ac22e46f64e
Parents: 76f0748
Author: s.kadam <s....@gus.com>
Authored: Mon Oct 29 21:47:21 2018 +0000
Committer: pboado <pe...@gmail.com>
Committed: Sun Nov 25 22:09:13 2018 +0000
----------------------------------------------------------------------
.../phoenix/end2end/CsvBulkLoadToolIT.java | 51 ++++++++++++++++++++
.../mapreduce/FormatToBytesWritableMapper.java | 10 ++--
2 files changed, 56 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e453b772/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
index 40fe900..7e4226d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
@@ -35,7 +35,10 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.CsvBulkLoadTool;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -446,4 +449,52 @@ public class CsvBulkLoadToolIT extends BaseOwnClusterIT {
rs.close();
stmt.close();
}
+
+ /**
+ * This test case validates the import using CsvBulkLoadTool in
+ * SingleCellArrayWithOffsets table.
+ * PHOENIX-4872
+ */
+
+ @Test
+ public void testImportInSingleCellArrayWithOffsetsTable() throws Exception {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE IMMUTABLE TABLE S.TABLE12 (ID INTEGER NOT NULL PRIMARY KEY," +
+ " CF0.NAME VARCHAR, CF0.T DATE, CF1.T2 DATE, CF2.T3 DATE) " +
+ "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS");
+ PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class);
+ PTable table = phxConn.getTable(new PTableKey(null, "S.TABLE12"));
+
+ assertEquals(PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS,
+ table.getImmutableStorageScheme());
+
+ FileSystem fs = FileSystem.get(getUtility().getConfiguration());
+ FSDataOutputStream outputStream = fs.create(new Path("/tmp/inputSCAWO.csv"));
+ PrintWriter printWriter = new PrintWriter(outputStream);
+ printWriter.println("1,Name 1,1970/01/01,1970/02/01,1970/03/01");
+ printWriter.println("2,Name 2,1970/01/02,1970/02/02,1970/03/02");
+ printWriter.println("3,Name 1,1970/01/01,1970/02/03,1970/03/01");
+ printWriter.println("4,Name 2,1970/01/02,1970/02/04,1970/03/02");
+ printWriter.println("5,Name 1,1970/01/01,1970/02/05,1970/03/01");
+ printWriter.println("6,Name 2,1970/01/02,1970/02/06,1970/03/02");
+ printWriter.close();
+
+ CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool();
+ csvBulkLoadTool.setConf(new Configuration(getUtility().getConfiguration()));
+ csvBulkLoadTool.getConf().set(DATE_FORMAT_ATTRIB,"yyyy/MM/dd");
+ int exitCode = csvBulkLoadTool.run(new String[] {
+ "--input", "/tmp/inputSCAWO.csv",
+ "--table", "table12",
+ "--schema", "s",
+ "--zookeeper", zkQuorum});
+ assertEquals(0, exitCode);
+
+ ResultSet rs = stmt.executeQuery("SELECT COUNT(1) FROM S.TABLE12");
+ assertTrue(rs.next());
+ assertEquals(6, rs.getInt(1));
+
+ rs.close();
+ stmt.close();
+
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e453b772/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
index 360859e..93ab188 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
@@ -242,12 +242,12 @@ public abstract class FormatToBytesWritableMapper<RECORD> extends Mapper<LongWri
columnIndex++;
}
}
- byte[] emptyColumnFamily = SchemaUtil.getEmptyColumnFamily(table);
- byte[] emptyKeyValue = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
- byte[] cfn = Bytes.add(emptyColumnFamily, QueryConstants.NAMESPACE_SEPARATOR_BYTES, emptyKeyValue);
- columnIndexes.put(cfn, new Integer(columnIndex));
- columnIndex++;
}
+ byte[] emptyColumnFamily = SchemaUtil.getEmptyColumnFamily(table);
+ byte[] emptyKeyValue = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
+ byte[] cfn = Bytes.add(emptyColumnFamily, QueryConstants.NAMESPACE_SEPARATOR_BYTES, emptyKeyValue);
+ columnIndexes.put(cfn, new Integer(columnIndex));
+ columnIndex++;
}
}