You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/05/24 17:22:28 UTC
orc git commit: ORC-55. Resynchronize storage-api and orc with the
hive trunk. (omalley reviewed by Sergey)
Repository: orc
Updated Branches:
refs/heads/master 377c63c9d -> 32787e852
ORC-55. Resynchronize storage-api and orc with the hive trunk. (omalley
reviewed by Sergey)
Signed-off-by: Owen O'Malley <om...@apache.org>
Fixes #25
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/32787e85
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/32787e85
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/32787e85
Branch: refs/heads/master
Commit: 32787e8520e3399e2aafe4de115867a832ae654e
Parents: 377c63c
Author: Owen O'Malley <om...@apache.org>
Authored: Fri May 20 16:50:01 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue May 24 10:20:52 2016 -0700
----------------------------------------------------------------------
java/core/pom.xml | 2 +-
.../apache/orc/impl/TestRecordReaderImpl.java | 4 +-
java/mapreduce/pom.xml | 5 --
java/storage-api/pom.xml | 4 +-
.../hadoop/hive/common/type/RandomTypeUtil.java | 95 ++++++++++++++++++++
5 files changed, 100 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/core/pom.xml
----------------------------------------------------------------------
diff --git a/java/core/pom.xml b/java/core/pom.xml
index 85d9ae6..c3efa0b 100644
--- a/java/core/pom.xml
+++ b/java/core/pom.xml
@@ -31,7 +31,7 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>2.0.0.1-SNAPSHOT</version>
+ <version>2.0.2-pre-orc</version>
</dependency>
<!-- inter-project -->
http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java b/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
index a55b378..cdd62ac 100644
--- a/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
+++ b/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java
@@ -1676,10 +1676,8 @@ public class TestRecordReaderImpl {
private void closeMockedRecordReader(DataReader mockedDataReader) throws IOException {
Configuration conf = new Configuration();
- FileSystem fs = FileSystem.getLocal(conf).getRaw();
- fs.delete(workDir, true);
- fs.mkdirs(workDir);
Path path = new Path(workDir, "empty.orc");
+ FileSystem.get(conf).delete(path, true);
Writer writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf)
.setSchema(TypeDescription.createLong()));
writer.close();
http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/mapreduce/pom.xml
----------------------------------------------------------------------
diff --git a/java/mapreduce/pom.xml b/java/mapreduce/pom.xml
index 8792e1c..cbb625f 100644
--- a/java/mapreduce/pom.xml
+++ b/java/mapreduce/pom.xml
@@ -29,11 +29,6 @@
<dependencies>
<dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-storage-api</artifactId>
- <version>2.0.0.1-SNAPSHOT</version>
- </dependency>
- <dependency>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
<version>1.1.0-SNAPSHOT</version>
http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/java/storage-api/pom.xml b/java/storage-api/pom.xml
index 85bd77c..7636446 100644
--- a/java/storage-api/pom.xml
+++ b/java/storage-api/pom.xml
@@ -19,7 +19,9 @@
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>2.0.0.1-SNAPSHOT</version>
+ <!-- remove our custom version of storage-api once we get the changes
+ released as hive 2.0.2 -->
+ <version>2.0.2-pre-orc</version>
<packaging>jar</packaging>
<name>Hive Storage API</name>
http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
----------------------------------------------------------------------
diff --git a/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java b/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
index 3fb0cfd..53a7823 100644
--- a/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
+++ b/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.common.type;
+import java.sql.Date;
import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.ParseException;
@@ -26,6 +27,100 @@ import java.util.concurrent.TimeUnit;
public class RandomTypeUtil {
+ public static String getRandString(Random r) {
+ return getRandString(r, null, r.nextInt(10));
+ }
+
+ public static String getRandString(Random r, String characters, int length) {
+ if (characters == null) {
+ characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+ }
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ if (characters == null) {
+ sb.append((char) (r.nextInt(128)));
+ } else {
+ sb.append(characters.charAt(r.nextInt(characters.length())));
+ }
+ }
+ return sb.toString();
+ }
+
+ public static byte[] getRandBinary(Random r, int len){
+ byte[] bytes = new byte[len];
+ for (int j = 0; j < len; j++){
+ bytes[j] = Byte.valueOf((byte) r.nextInt());
+ }
+ return bytes;
+ }
+
+ private static final String DECIMAL_CHARS = "0123456789";
+
+ public static class HiveDecimalAndPrecisionScale {
+ public HiveDecimal hiveDecimal;
+ public int precision;
+ public int scale;
+
+ HiveDecimalAndPrecisionScale(HiveDecimal hiveDecimal, int precision, int scale) {
+ this.hiveDecimal = hiveDecimal;
+ this.precision = precision;
+ this.scale = scale;
+ }
+ }
+
+ public static HiveDecimalAndPrecisionScale getRandHiveDecimal(Random r) {
+ int precision;
+ int scale;
+ while (true) {
+ StringBuilder sb = new StringBuilder();
+ precision = 1 + r.nextInt(18);
+ scale = 0 + r.nextInt(precision + 1);
+
+ int integerDigits = precision - scale;
+
+ if (r.nextBoolean()) {
+ sb.append("-");
+ }
+
+ if (integerDigits == 0) {
+ sb.append("0");
+ } else {
+ sb.append(getRandString(r, DECIMAL_CHARS, integerDigits));
+ }
+ if (scale != 0) {
+ sb.append(".");
+ sb.append(getRandString(r, DECIMAL_CHARS, scale));
+ }
+
+ HiveDecimal bd = HiveDecimal.create(sb.toString());
+ precision = bd.precision();
+ scale = bd.scale();
+ if (scale > precision) {
+ // Sometimes weird decimals are produced?
+ continue;
+ }
+
+ // For now, punt.
+ precision = HiveDecimal.SYSTEM_DEFAULT_PRECISION;
+ scale = HiveDecimal.SYSTEM_DEFAULT_SCALE;
+ return new HiveDecimalAndPrecisionScale(bd, precision, scale);
+ }
+ }
+
+ public static Date getRandDate(Random r) {
+ String dateStr = String.format("%d-%02d-%02d",
+ Integer.valueOf(1800 + r.nextInt(500)), // year
+ Integer.valueOf(1 + r.nextInt(12)), // month
+ Integer.valueOf(1 + r.nextInt(28))); // day
+ Date dateVal = Date.valueOf(dateStr);
+ return dateVal;
+ }
+
+ /**
+ * TIMESTAMP.
+ */
+
public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1);
public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1);
public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1);