You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/10/26 13:25:29 UTC
carbondata git commit: [CARBONDATA-2981] Support read primitive data
type in CSDK
Repository: carbondata
Updated Branches:
refs/heads/master 71d617955 -> 019f5cd06
[CARBONDATA-2981] Support read primitive data type in CSDK
1.support readNextCarbonRow
2.support read different primitive data type in c code from java side: int double short long string
3.support some data type and convert: date timestamp varchar decimal array<T>
4.remove readNextStringRow
This closes #2792
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/019f5cd0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/019f5cd0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/019f5cd0
Branch: refs/heads/master
Commit: 019f5cd06f09b7006507b6f8a912b4f0a2661357
Parents: 71d6179
Author: xubo245 <xu...@huawei.com>
Authored: Tue Oct 9 17:58:48 2018 +0800
Committer: kunal642 <ku...@gmail.com>
Committed: Fri Oct 26 18:54:59 2018 +0530
----------------------------------------------------------------------
README.md | 1 +
.../core/datastore/row/CarbonRow.java | 1 +
docs/CSDK-guide.md | 197 --------------
docs/csdk-guide.md | 103 +++++++
docs/documentation.md | 2 +-
store/CSDK/CMakeLists.txt | 22 +-
store/CSDK/CarbonReader.cpp | 103 -------
store/CSDK/CarbonReader.h | 105 --------
store/CSDK/main.cpp | 200 --------------
store/CSDK/src/CarbonReader.cpp | 176 ++++++++++++
store/CSDK/src/CarbonReader.h | 112 ++++++++
store/CSDK/src/CarbonRow.cpp | 193 ++++++++++++++
store/CSDK/src/CarbonRow.h | 153 +++++++++++
store/CSDK/test/main.cpp | 266 +++++++++++++++++++
.../carbondata/sdk/file/CarbonReader.java | 30 ---
.../org/apache/carbondata/sdk/file/Field.java | 9 +
.../org/apache/carbondata/sdk/file/RowUtil.java | 146 ++++++++++
.../sdk/file/AvroCarbonWriterTest.java | 3 +-
.../carbondata/sdk/file/CarbonReaderTest.java | 208 +++++++++++++++
19 files changed, 1382 insertions(+), 648 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 87bb71e..d472e13 100644
--- a/README.md
+++ b/README.md
@@ -61,6 +61,7 @@ CarbonData is built using Apache Maven, to [build CarbonData](https://github.com
* [CarbonData Pre-aggregate DataMap](https://github.com/apache/carbondata/blob/master/docs/preaggregate-datamap-guide.md)
* [CarbonData Timeseries DataMap](https://github.com/apache/carbondata/blob/master/docs/timeseries-datamap-guide.md)
* [SDK Guide](https://github.com/apache/carbondata/blob/master/docs/sdk-guide.md)
+* [CSDK Guide](https://github.com/apache/carbondata/blob/master/docs/csdk-guide.md)
* [Performance Tuning](https://github.com/apache/carbondata/blob/master/docs/performance-tuning.md)
* [S3 Storage](https://github.com/apache/carbondata/blob/master/docs/s3-guide.md)
* [Carbon as Spark's Datasource](https://github.com/apache/carbondata/blob/master/docs/carbon-as-spark-datasource-guide.md)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
index 48775d4..1141707 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
@@ -72,6 +72,7 @@ public class CarbonRow implements Serializable {
public Object[] getRawData() {
return rawData;
}
+
public void setRawData(Object[] rawData) {
this.rawData = rawData;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/docs/CSDK-guide.md
----------------------------------------------------------------------
diff --git a/docs/CSDK-guide.md b/docs/CSDK-guide.md
deleted file mode 100644
index c4f4a31..0000000
--- a/docs/CSDK-guide.md
+++ /dev/null
@@ -1,197 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to you under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# CSDK Guide
-
-CarbonData CSDK provides C++ interface to write and read carbon file.
-CSDK use JNI to invoke java SDK in C++ code.
-
-
-# CSDK Reader
-This CSDK reader reads CarbonData file and carbonindex file at a given path.
-External client can make use of this reader to read CarbonData files in C++
-code and without CarbonSession.
-
-
-In the carbon jars package, there exist a carbondata-sdk.jar,
-including SDK reader for CSDK.
-## Quick example
-```
-// 1. init JVM
-JavaVM *jvm;
-JNIEnv *initJVM() {
- JNIEnv *env;
- JavaVMInitArgs vm_args;
- int parNum = 3;
- int res;
- JavaVMOption options[parNum];
-
- options[0].optionString = "-Djava.compiler=NONE";
- options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
- options[2].optionString = "-verbose:jni";
- vm_args.version = JNI_VERSION_1_8;
- vm_args.nOptions = parNum;
- vm_args.options = options;
- vm_args.ignoreUnrecognized = JNI_FALSE;
-
- res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
- if (res < 0) {
- fprintf(stderr, "\nCan't create Java VM\n");
- exit(1);
- }
-
- return env;
-}
-
-// 2. create carbon reader and read data
-// 2.1 read data from local disk
-/**
- * test read data from local disk, without projection
- *
- * @param env jni env
- * @return
- */
-bool readFromLocalWithoutProjection(JNIEnv *env) {
-
- CarbonReader carbonReaderClass;
- carbonReaderClass.builder(env, "../resources/carbondata", "test");
- carbonReaderClass.build();
-
- while (carbonReaderClass.hasNext()) {
- jobjectArray row = carbonReaderClass.readNextRow();
- jsize length = env->GetArrayLength(row);
- int j = 0;
- for (j = 0; j < length; j++) {
- jobject element = env->GetObjectArrayElement(row, j);
- char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
- printf("%s\t", str);
- }
- printf("\n");
- }
- carbonReaderClass.close();
-}
-
-// 2.2 read data from S3
-
-/**
- * read data from S3
- * parameter is ak sk endpoint
- *
- * @param env jni env
- * @param argv argument vector
- * @return
- */
-bool readFromS3(JNIEnv *env, char *argv[]) {
- CarbonReader reader;
-
- char *args[3];
- // "your access key"
- args[0] = argv[1];
- // "your secret key"
- args[1] = argv[2];
- // "your endPoint"
- args[2] = argv[3];
-
- reader.builder(env, "s3a://sdk/WriterOutput", "test");
- reader.withHadoopConf(3, args);
- reader.build();
- printf("\nRead data from S3:\n");
- while (reader.hasNext()) {
- jobjectArray row = reader.readNextRow();
- jsize length = env->GetArrayLength(row);
-
- int j = 0;
- for (j = 0; j < length; j++) {
- jobject element = env->GetObjectArrayElement(row, j);
- char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
- printf("%s\t", str);
- }
- printf("\n");
- }
-
- reader.close();
-}
-
-// 3. destory JVM
- (jvm)->DestroyJavaVM();
-```
-Find example code at main.cpp of CSDK module
-
-## API List
-```
- /**
- * create a CarbonReaderBuilder object for building carbonReader,
- * CarbonReaderBuilder object can configure different parameter
- *
- * @param env JNIEnv
- * @param path data store path
- * @param tableName table name
- * @return CarbonReaderBuilder object
- */
- jobject builder(JNIEnv *env, char *path, char *tableName);
-
- /**
- * Configure the projection column names of carbon reader
- *
- * @param argc argument counter
- * @param argv argument vector
- * @return CarbonReaderBuilder object
- */
- jobject projection(int argc, char *argv[]);
-
- /**
- * build carbon reader with argument vector
- * it support multiple parameter
- * like: key=value
- * for example: fs.s3a.access.key=XXXX, XXXX is user's access key value
- *
- * @param argc argument counter
- * @param argv argument vector
- * @return CarbonReaderBuilder object
- **/
- jobject withHadoopConf(int argc, char *argv[]);
-
- /**
- * build carbonReader object for reading data
- * it support read data from load disk
- *
- * @return carbonReader object
- */
- jobject build();
-
- /**
- * Whether it has next row data
- *
- * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
- */
- jboolean hasNext();
-
- /**
- * read next row from data
- *
- * @return object array of one row
- */
- jobjectArray readNextRow();
-
- /**
- * close the carbon reader
- *
- * @return boolean value
- */
- jboolean close();
-
-```
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/docs/csdk-guide.md
----------------------------------------------------------------------
diff --git a/docs/csdk-guide.md b/docs/csdk-guide.md
new file mode 100644
index 0000000..5598cb6
--- /dev/null
+++ b/docs/csdk-guide.md
@@ -0,0 +1,103 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to you under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# CSDK Guide
+
+CarbonData CSDK provides C++ interface to write and read carbon file.
+CSDK use JNI to invoke java SDK in C++ code.
+
+
+# CSDK Reader
+This CSDK reader reads CarbonData file and carbonindex file at a given path.
+External client can make use of this reader to read CarbonData files in C++
+code and without CarbonSession.
+
+
+In the carbon jars package, there exist a carbondata-sdk.jar,
+including SDK reader for CSDK.
+## Quick example
+
+Please find example code at [main.cpp](https://github.com/apache/carbondata/blob/master/store/CSDK/test/main.cpp) of CSDK module
+
+When users use C++ to read carbon files, users should init JVM first. Then users create
+carbon reader and read data.There are some example code of read data from local disk
+and read data from S3 at main.cpp of CSDK module. Finally, Finally, users need to
+release the memory and destroy JVM.
+
+## API List
+```
+ /**
+ * create a CarbonReaderBuilder object for building carbonReader,
+ * CarbonReaderBuilder object can configure different parameter
+ *
+ * @param env JNIEnv
+ * @param path data store path
+ * @param tableName table name
+ * @return CarbonReaderBuilder object
+ */
+ jobject builder(JNIEnv *env, char *path, char *tableName);
+
+ /**
+ * Configure the projection column names of carbon reader
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return CarbonReaderBuilder object
+ */
+ jobject projection(int argc, char *argv[]);
+
+ /**
+ * build carbon reader with argument vector
+ * it support multiple parameter
+ * like: key=value
+ * for example: fs.s3a.access.key=XXXX, XXXX is user's access key value
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return CarbonReaderBuilder object
+ **/
+ jobject withHadoopConf(int argc, char *argv[]);
+
+ /**
+ * build carbonReader object for reading data
+ * it support read data from load disk
+ *
+ * @return carbonReader object
+ */
+ jobject build();
+
+ /**
+ * Whether it has next row data
+ *
+ * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
+ */
+ jboolean hasNext();
+
+ /**
+ * read next carbonRow from data
+ * @return carbonRow object of one row
+ */
+ jobject readNextRow();
+
+ /**
+ * close the carbon reader
+ *
+ * @return boolean value
+ */
+ jboolean close();
+
+```
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/docs/documentation.md
----------------------------------------------------------------------
diff --git a/docs/documentation.md b/docs/documentation.md
index 1b6726a..4a1d176 100644
--- a/docs/documentation.md
+++ b/docs/documentation.md
@@ -31,7 +31,7 @@ Apache CarbonData is a new big data file format for faster interactive query usi
**CarbonData SQL Language Reference:** CarbonData extends the Spark SQL language and adds several [DDL](./ddl-of-carbondata.md) and [DML](./dml-of-carbondata.md) statements to support operations on it.Refer to the [Reference Manual](./language-manual.md) to understand the supported features and functions.
-**Programming Guides:** You can read our guides about [APIs supported](./sdk-guide.md) to learn how to integrate CarbonData with your applications.
+**Programming Guides:** You can read our guides about [Java APIs supported](./sdk-guide.md) or [C++ APIs supported](./csdk-guide.md) to learn how to integrate CarbonData with your applications.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/store/CSDK/CMakeLists.txt b/store/CSDK/CMakeLists.txt
index c3c57a0..bfda148 100644
--- a/store/CSDK/CMakeLists.txt
+++ b/store/CSDK/CMakeLists.txt
@@ -1,17 +1,17 @@
-cmake_minimum_required (VERSION 2.8)
-project (CJDK)
+cmake_minimum_required(VERSION 2.8)
+project(CJDK)
set(CMAKE_BUILD_TYPE Debug)
-SET (CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
-find_package (JNI REQUIRED)
+find_package(JNI REQUIRED)
-include_directories (${JNI_INCLUDE_DIRS})
+include_directories(${JNI_INCLUDE_DIRS})
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-set (SOURCE_FILES CarbonReader.cpp CarbonReader.h main.cpp)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+set(SOURCE_FILES src/CarbonReader.cpp src/CarbonReader.h test/main.cpp src/CarbonRow.h src/CarbonRow.cpp)
-add_executable (CJDK ${SOURCE_FILES})
-get_filename_component (JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY)
-message (${JAVA_JVM_LIBRARY_DIR})
-target_link_libraries (CJDK ${JAVA_JVM_LIBRARY} )
+add_executable(CJDK ${SOURCE_FILES})
+get_filename_component(JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY)
+message(${JAVA_JVM_LIBRARY_DIR})
+target_link_libraries(CJDK ${JAVA_JVM_LIBRARY})
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/CarbonReader.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/CarbonReader.cpp b/store/CSDK/CarbonReader.cpp
deleted file mode 100644
index 0400957..0000000
--- a/store/CSDK/CarbonReader.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "CarbonReader.h"
-#include <jni.h>
-
-jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) {
-
- jniEnv = env;
- jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
- jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
- "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
- jstring jpath = env->NewStringUTF(path);
- jstring jtableName = env->NewStringUTF(tableName);
- jvalue args[2];
- args[0].l = jpath;
- args[1].l = jtableName;
- carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
- return carbonReaderBuilderObject;
-}
-
-jobject CarbonReader::builder(JNIEnv *env, char *path) {
- jniEnv = env;
- jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
- jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
- "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
- jstring jpath = env->NewStringUTF(path);
- jvalue args[1];
- args[0].l = jpath;
- carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
- return carbonReaderBuilderObject;
-}
-
-jobject CarbonReader::projection(int argc, char *argv[]) {
- jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
- jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "projection",
- "([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
- jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;");
- jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass, NULL);
- for (int i = 0; i < argc; ++i) {
- jstring value = jniEnv->NewStringUTF(argv[i]);
- jniEnv->SetObjectArrayElement(array, i, value);
- }
-
- jvalue args[1];
- args[0].l = array;
- carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args);
- return carbonReaderBuilderObject;
-}
-
-jobject CarbonReader::withHadoopConf(char *key, char *value) {
- jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
- jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "withHadoopConf",
- "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
-
- jvalue args[2];
- args[0].l = jniEnv->NewStringUTF(key);
- args[1].l = jniEnv->NewStringUTF(value);
- carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args);
- return carbonReaderBuilderObject;
-}
-
-jobject CarbonReader::build() {
- jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
- jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build",
- "()Lorg/apache/carbondata/sdk/file/CarbonReader;");
- carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID);
- return carbonReaderObject;
-}
-
-jboolean CarbonReader::hasNext() {
- jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
- jmethodID hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z");
- unsigned char hasNext = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID);
- return hasNext;
-}
-
-jobjectArray CarbonReader::readNextRow() {
- jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
- jmethodID readNextRow2ID = jniEnv->GetMethodID(carbonReader, "readNextStringRow", "()[Ljava/lang/Object;");
- jobjectArray row = (jobjectArray) jniEnv->CallObjectMethod(carbonReaderObject, readNextRow2ID);
- return row;
-}
-
-jboolean CarbonReader::close() {
- jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
- jmethodID closeID = jniEnv->GetMethodID(carbonReader, "close", "()V");
- jniEnv->CallBooleanMethod(carbonReaderObject, closeID);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/CarbonReader.h
----------------------------------------------------------------------
diff --git a/store/CSDK/CarbonReader.h b/store/CSDK/CarbonReader.h
deleted file mode 100644
index 861f704..0000000
--- a/store/CSDK/CarbonReader.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <jni.h>
-
-class CarbonReader {
-public:
- /**
- * jni env
- */
- JNIEnv *jniEnv;
-
- /**
- * carbonReaderBuilder object for building carbonReader
- * it can configure some operation
- */
- jobject carbonReaderBuilderObject;
-
- /**
- * carbonReader object for reading data
- */
- jobject carbonReaderObject;
-
- /**
- * create a CarbonReaderBuilder object for building carbonReader,
- * CarbonReaderBuilder object can configure different parameter
- *
- * @param env JNIEnv
- * @param path data store path
- * @param tableName table name
- * @return CarbonReaderBuilder object
- */
- jobject builder(JNIEnv *env, char *path, char *tableName);
-
- /**
- * create a CarbonReaderBuilder object for building carbonReader,
- * CarbonReaderBuilder object can configure different parameter
- *
- * @param env JNIEnv
- * @param path data store path
- * @return CarbonReaderBuilder object
- * */
- jobject builder(JNIEnv *env, char *path);
-
- /**
- * Configure the projection column names of carbon reader
- *
- * @param argc argument counter
- * @param argv argument vector
- * @return CarbonReaderBuilder object
- */
- jobject projection(int argc, char *argv[]);
-
- /**
- * configure parameter, including ak,sk and endpoint
- *
- * @param key key word
- * @param value value
- * @return CarbonReaderBuilder object
- */
- jobject withHadoopConf(char *key, char *value);
-
- /**
- * build carbonReader object for reading data
- * it support read data from load disk
- *
- * @return carbonReader object
- */
- jobject build();
-
- /**
- * Whether it has next row data
- *
- * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
- */
- jboolean hasNext();
-
- /**
- * read next row from data
- *
- * @return object array of one row
- */
- jobjectArray readNextRow();
-
- /**
- * close the carbon reader
- *
- * @return boolean value
- */
- jboolean close();
-};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/main.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/main.cpp b/store/CSDK/main.cpp
deleted file mode 100644
index a8d1a87..0000000
--- a/store/CSDK/main.cpp
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <stdio.h>
-#include <jni.h>
-#include <stdlib.h>
-#include <iostream>
-#include <unistd.h>
-#include "CarbonReader.h"
-
-using namespace std;
-
-JavaVM *jvm;
-
-/**
- * init jvm
- *
- * @return
- */
-JNIEnv *initJVM() {
- JNIEnv *env;
- JavaVMInitArgs vm_args;
- int parNum = 3;
- int res;
- JavaVMOption options[parNum];
-
- options[0].optionString = "-Djava.compiler=NONE";
- options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
- options[2].optionString = "-verbose:jni";
- vm_args.version = JNI_VERSION_1_8;
- vm_args.nOptions = parNum;
- vm_args.options = options;
- vm_args.ignoreUnrecognized = JNI_FALSE;
-
- res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
- if (res < 0) {
- fprintf(stderr, "\nCan't create Java VM\n");
- exit(1);
- }
-
- return env;
-}
-
-/**
- * test read data from local disk, without projection
- *
- * @param env jni env
- * @return
- */
-bool readFromLocalWithoutProjection(JNIEnv *env) {
-
- CarbonReader carbonReaderClass;
- carbonReaderClass.builder(env, "../resources/carbondata");
- carbonReaderClass.build();
-
- printf("\nRead data from local without projection:\n");
-
- while (carbonReaderClass.hasNext()) {
- jobjectArray row = carbonReaderClass.readNextRow();
- jsize length = env->GetArrayLength(row);
-
- int j = 0;
- for (j = 0; j < length; j++) {
- jobject element = env->GetObjectArrayElement(row, j);
- char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
- printf("%s\t", str);
- }
- printf("\n");
- }
-
- carbonReaderClass.close();
-}
-
-/**
- * test read data from local disk
- *
- * @param env jni env
- * @return
- */
-bool readFromLocal(JNIEnv *env) {
-
- CarbonReader reader;
- reader.builder(env, "../resources/carbondata", "test");
-
- char *argv[11];
- argv[0] = "stringField";
- argv[1] = "shortField";
- argv[2] = "intField";
- argv[3] = "longField";
- argv[4] = "doubleField";
- argv[5] = "boolField";
- argv[6] = "dateField";
- argv[7] = "timeField";
- argv[8] = "decimalField";
- argv[9] = "varcharField";
- argv[10] = "arrayField";
- reader.projection(11, argv);
-
- reader.build();
-
- printf("\nRead data from local:\n");
-
- while (reader.hasNext()) {
- jobjectArray row = reader.readNextRow();
- jsize length = env->GetArrayLength(row);
-
- int j = 0;
- for (j = 0; j < length; j++) {
- jobject element = env->GetObjectArrayElement(row, j);
- char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
- printf("%s\t", str);
- }
- printf("\n");
- }
-
- reader.close();
-}
-
-/**
- * read data from S3
- * parameter is ak sk endpoint
- *
- * @param env jni env
- * @param argv argument vector
- * @return
- */
-bool readFromS3(JNIEnv *env, char *argv[]) {
- CarbonReader reader;
-
- char *args[3];
- // "your access key"
- args[0] = argv[1];
- // "your secret key"
- args[1] = argv[2];
- // "your endPoint"
- args[2] = argv[3];
-
- reader.builder(env, "s3a://sdk/WriterOutput", "test");
- reader.withHadoopConf("fs.s3a.access.key", argv[1]);
- reader.withHadoopConf("fs.s3a.secret.key", argv[2]);
- reader.withHadoopConf("fs.s3a.endpoint", argv[3]);
- reader.build();
- printf("\nRead data from S3:\n");
- while (reader.hasNext()) {
- jobjectArray row = reader.readNextRow();
- jsize length = env->GetArrayLength(row);
-
- int j = 0;
- for (j = 0; j < length; j++) {
- jobject element = env->GetObjectArrayElement(row, j);
- char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
- printf("%s\t", str);
- }
- printf("\n");
- }
-
- reader.close();
-}
-
-/**
- * This a example for C++ interface to read carbon file
- * If you want to test read data fromS3, please input the parameter: ak sk endpoint
- *
- * @param argc argument counter
- * @param argv argument vector
- * @return
- */
-int main(int argc, char *argv[]) {
- // init jvm
- JNIEnv *env;
- env = initJVM();
-
- if (argc > 3) {
- readFromS3(env, argv);
- } else {
- readFromLocalWithoutProjection(env);
- readFromLocal(env);
- }
- cout << "destory jvm\n\n";
- (jvm)->DestroyJavaVM();
-
- cout << "\nfinish destory jvm";
- fprintf(stdout, "Java VM destory.\n");
- return 0;
-}
-
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonReader.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/src/CarbonReader.cpp b/store/CSDK/src/CarbonReader.cpp
new file mode 100644
index 0000000..456e00e
--- /dev/null
+++ b/store/CSDK/src/CarbonReader.cpp
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "CarbonReader.h"
+#include <jni.h>
+#include <mach/mach_types.h>
+#include <stdexcept>
+
+void CarbonReader::builder(JNIEnv *env, char *path, char *tableName) {
+ if (env == NULL) {
+ throw std::runtime_error("JNIEnv parameter can't be NULL.");
+ }
+ if (path == NULL) {
+ throw std::runtime_error("path parameter can't be NULL.");
+ }
+ if (tableName == NULL) {
+ throw std::runtime_error("tableName parameter can't be NULL.");
+ }
+ jniEnv = env;
+ jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+ if (carbonReaderClass == NULL) {
+ throw std::runtime_error("Can't find the class in java: org/apache/carbondata/sdk/file/CarbonReader");
+ }
+ jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+ "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+ if (carbonReaderBuilderID == NULL) {
+ throw std::runtime_error("Can't find the method in java: builder");
+ }
+ jstring jPath = env->NewStringUTF(path);
+ jstring jTableName = env->NewStringUTF(tableName);
+ jvalue args[2];
+ args[0].l = jPath;
+ args[1].l = jTableName;
+ carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
+}
+
+void CarbonReader::builder(JNIEnv *env, char *path) {
+ if (env == NULL) {
+ throw std::runtime_error("JNIEnv parameter can't be NULL.");
+ }
+ if (path == NULL) {
+ throw std::runtime_error("path parameter can't be NULL.");
+ }
+ jniEnv = env;
+ jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+ if (carbonReaderClass == NULL) {
+ throw std::runtime_error("Can't find the class in java: org/apache/carbondata/sdk/file/CarbonReader");
+ }
+ jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+ "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+ if (carbonReaderBuilderID == NULL) {
+ throw std::runtime_error("Can't find the method in java: builder");
+ }
+ jstring jPath = env->NewStringUTF(path);
+ jvalue args[1];
+ args[0].l = jPath;
+ carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
+}
+
+void CarbonReader::projection(int argc, char *argv[]) {
+ if (argc < 0) {
+ throw std::runtime_error("argc parameter can't be negative.");
+ }
+ if (argv == NULL) {
+ throw std::runtime_error("argv parameter can't be NULL.");
+ }
+ jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+ jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "projection",
+ "([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+ if (buildID == NULL) {
+ throw std::runtime_error("Can't find the method in java: projection");
+ }
+ jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;");
+ if (objectArrayClass == NULL) {
+ throw std::runtime_error("Can't find the class in java: java/lang/String");
+ }
+ jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass, NULL);
+ for (int i = 0; i < argc; ++i) {
+ jstring value = jniEnv->NewStringUTF(argv[i]);
+ jniEnv->SetObjectArrayElement(array, i, value);
+ }
+
+ jvalue args[1];
+ args[0].l = array;
+ carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args);
+}
+
+void CarbonReader::withHadoopConf(char *key, char *value) {
+ if (key == NULL) {
+ throw std::runtime_error("key parameter can't be NULL.");
+ }
+ if (value == NULL) {
+ throw std::runtime_error("value parameter can't be NULL.");
+ }
+ jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+ jmethodID id = jniEnv->GetMethodID(carbonReaderBuilderClass, "withHadoopConf",
+ "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+ if (id == NULL) {
+ throw std::runtime_error("Can't find the method in java: withHadoopConf");
+ }
+ jvalue args[2];
+ args[0].l = jniEnv->NewStringUTF(key);
+ args[1].l = jniEnv->NewStringUTF(value);
+ carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, id, args);
+}
+
+jobject CarbonReader::build() {
+ jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+ jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build",
+ "()Lorg/apache/carbondata/sdk/file/CarbonReader;");
+ if (buildID == NULL) {
+ throw std::runtime_error("Can't find the method in java: build");
+ }
+ carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID);
+ if (jniEnv->ExceptionCheck()) {
+ throw jniEnv->ExceptionOccurred();
+ }
+ return carbonReaderObject;
+}
+
+jboolean CarbonReader::hasNext() {
+ if (hasNextID == NULL) {
+ jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+ hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z");
+ if (hasNextID == NULL) {
+ throw std::runtime_error("Can't find the method in java: hasNext");
+ }
+ }
+ jboolean result = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID);
+ if (jniEnv->ExceptionCheck()) {
+ throw jniEnv->ExceptionOccurred();
+ }
+ return result;
+}
+
+jobject CarbonReader::readNextRow() {
+ if (readNextRowID == NULL) {
+ jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+ readNextRowID = jniEnv->GetMethodID(carbonReader, "readNextRow",
+ "()Ljava/lang/Object;");
+ if (readNextRowID == NULL) {
+ throw std::runtime_error("Can't find the method in java: readNextRow");
+ }
+ }
+ jobject result = jniEnv->CallObjectMethod(carbonReaderObject, readNextRowID);
+ if (jniEnv->ExceptionCheck()) {
+ throw jniEnv->ExceptionOccurred();
+ }
+ return result;
+}
+
+void CarbonReader::close() {
+ jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+ jmethodID closeID = jniEnv->GetMethodID(carbonReader, "close", "()V");
+ if (closeID == NULL) {
+ throw std::runtime_error("Can't find the method in java: close");
+ }
+ jniEnv->CallBooleanMethod(carbonReaderObject, closeID);
+ if (jniEnv->ExceptionCheck()) {
+ throw jniEnv->ExceptionOccurred();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonReader.h
----------------------------------------------------------------------
diff --git a/store/CSDK/src/CarbonReader.h b/store/CSDK/src/CarbonReader.h
new file mode 100644
index 0000000..b4841bf
--- /dev/null
+++ b/store/CSDK/src/CarbonReader.h
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <jni.h>
+
+class CarbonReader {
+private:
+ /**
+ * hasNext jmethodID
+ */
+ jmethodID hasNextID = NULL;
+
+ /**
+ * readNextRow jmethodID
+ */
+ jmethodID readNextRowID = NULL;
+
+ /**
+ * carbonReaderBuilder object for building carbonReader
+ * it can configure some operation
+ */
+ jobject carbonReaderBuilderObject;
+
+ /**
+ * carbonReader object for reading data
+ */
+ jobject carbonReaderObject;
+
+public:
+
+ /**
+ * jni env
+ */
+ JNIEnv *jniEnv;
+
+ /**
+ * create a CarbonReaderBuilder object for building carbonReader,
+ * CarbonReaderBuilder object can configure different parameter
+ *
+ * @param env JNIEnv
+ * @param path data store path
+ * @param tableName table name
+ */
+ void builder(JNIEnv *env, char *path, char *tableName);
+
+ /**
+ * create a CarbonReaderBuilder object for building carbonReader,
+ * CarbonReaderBuilder object can configure different parameter
+ *
+ * @param env JNIEnv
+ * @param path data store path
+ * */
+ void builder(JNIEnv *env, char *path);
+
+ /**
+ * Configure the projection column names of carbon reader
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ */
+ void projection(int argc, char *argv[]);
+
+ /**
+ * configure parameter, including ak,sk and endpoint
+ *
+ * @param key key word
+ * @param value value
+ */
+ void withHadoopConf(char *key, char *value);
+
+ /**
+ * build carbonReader object for reading data
+ * it support read data from load disk
+ *
+ * @return carbonReader object
+ */
+ jobject build();
+
+ /**
+ * Whether it has next row data
+ *
+ * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
+ */
+ jboolean hasNext();
+
+ /**
+ * read next carbon Row from data
+ * @return carbon Row object of one row
+ */
+ jobject readNextRow();
+
+ /**
+ * close the carbon reader
+ *
+ * @return boolean value
+ */
+ void close();
+};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonRow.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/src/CarbonRow.cpp b/store/CSDK/src/CarbonRow.cpp
new file mode 100644
index 0000000..1c18ae6
--- /dev/null
+++ b/store/CSDK/src/CarbonRow.cpp
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <jni.h>
+#include <cstring>
+#include <stdexcept>
+#include "CarbonRow.h"
+
+CarbonRow::CarbonRow(JNIEnv *env) {
+ if (env == NULL) {
+ throw std::runtime_error("JNIEnv parameter can't be NULL.");
+ }
+ this->rowUtilClass = env->FindClass("org/apache/carbondata/sdk/file/RowUtil");
+ if (rowUtilClass == NULL) {
+ throw std::runtime_error("Can't find the class in java: org/apache/carbondata/sdk/file/RowUtil");
+ }
+ this->jniEnv = env;
+ getShortId = jniEnv->GetStaticMethodID(rowUtilClass, "getShort",
+ "([Ljava/lang/Object;I)S");
+ if (getShortId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getShort");
+ }
+
+ getIntId = jniEnv->GetStaticMethodID(rowUtilClass, "getInt",
+ "([Ljava/lang/Object;I)I");
+ if (getIntId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getInt");
+ }
+ getLongId = jniEnv->GetStaticMethodID(rowUtilClass, "getLong",
+ "([Ljava/lang/Object;I)J");
+ if (getLongId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getLong");
+ }
+
+ getDoubleId = jniEnv->GetStaticMethodID(rowUtilClass, "getDouble",
+ "([Ljava/lang/Object;I)D");
+ if (getDoubleId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getDouble");
+ }
+
+ getFloatId = jniEnv->GetStaticMethodID(rowUtilClass, "getFloat",
+ "([Ljava/lang/Object;I)F");
+ if (getFloatId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getFloat");
+ }
+
+ getBooleanId = jniEnv->GetStaticMethodID(rowUtilClass, "getBoolean",
+ "([Ljava/lang/Object;I)Z");
+ if (getBooleanId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getBoolean");
+ }
+
+ getStringId = jniEnv->GetStaticMethodID(rowUtilClass, "getString",
+ "([Ljava/lang/Object;I)Ljava/lang/String;");
+ if (getStringId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getString");
+ }
+
+ getDecimalId = jniEnv->GetStaticMethodID(rowUtilClass, "getDecimal",
+ "([Ljava/lang/Object;I)Ljava/lang/String;");
+ if (getDecimalId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getDecimal");
+ }
+
+ getVarcharId = jniEnv->GetStaticMethodID(rowUtilClass, "getVarchar",
+ "([Ljava/lang/Object;I)Ljava/lang/String;");
+ if (getVarcharId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getVarchar");
+ }
+
+ getArrayId = jniEnv->GetStaticMethodID(rowUtilClass, "getArray",
+ "([Ljava/lang/Object;I)[Ljava/lang/Object;");
+ if (getArrayId == NULL) {
+ throw std::runtime_error("Can't find the method in java: getArray");
+ }
+}
+
+void CarbonRow::setCarbonRow(jobject data) {
+ if (data == NULL) {
+ throw std::runtime_error("data parameter can't be NULL.");
+ }
+ this->carbonRow = data;
+}
+
+void CarbonRow::checkOrdinal(int ordinal) {
+ if (ordinal < 0) {
+ throw std::runtime_error("ordinal parameter can't be negative.");
+ }
+}
+
+short CarbonRow::getShort(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ return jniEnv->CallStaticShortMethodA(rowUtilClass, getShortId, args);
+}
+
+int CarbonRow::getInt(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ return jniEnv->CallStaticIntMethodA(rowUtilClass, getIntId, args);
+}
+
+long CarbonRow::getLong(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ return jniEnv->CallStaticLongMethodA(rowUtilClass, getLongId, args);
+}
+
+double CarbonRow::getDouble(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ return jniEnv->CallStaticDoubleMethodA(rowUtilClass, getDoubleId, args);
+}
+
+
+float CarbonRow::getFloat(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ return jniEnv->CallStaticFloatMethodA(rowUtilClass, getFloatId, args);
+}
+
+jboolean CarbonRow::getBoolean(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ return jniEnv->CallStaticBooleanMethodA(rowUtilClass, getBooleanId, args);
+}
+
+char *CarbonRow::getString(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ jobject data = jniEnv->CallStaticObjectMethodA(rowUtilClass, getStringId, args);
+ char *str = (char *) jniEnv->GetStringUTFChars((jstring) data, JNI_FALSE);
+ jniEnv->DeleteLocalRef(data);
+ return str;
+}
+
+char *CarbonRow::getDecimal(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ jobject data = jniEnv->CallStaticObjectMethodA(rowUtilClass, getDecimalId, args);
+ char *str = (char *) jniEnv->GetStringUTFChars((jstring) data, JNI_FALSE);
+ jniEnv->DeleteLocalRef(data);
+ return str;
+}
+
+char *CarbonRow::getVarchar(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ jobject data = jniEnv->CallStaticObjectMethodA(rowUtilClass, getVarcharId, args);
+ char *str = (char *) jniEnv->GetStringUTFChars((jstring) data, JNI_FALSE);
+ jniEnv->DeleteLocalRef(data);
+ return str;
+}
+
+jobjectArray CarbonRow::getArray(int ordinal) {
+ checkOrdinal(ordinal);
+ jvalue args[2];
+ args[0].l = carbonRow;
+ args[1].i = ordinal;
+ return (jobjectArray) jniEnv->CallStaticObjectMethodA(rowUtilClass, getArrayId, args);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/src/CarbonRow.h
----------------------------------------------------------------------
diff --git a/store/CSDK/src/CarbonRow.h b/store/CSDK/src/CarbonRow.h
new file mode 100644
index 0000000..007f8a9
--- /dev/null
+++ b/store/CSDK/src/CarbonRow.h
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <jni.h>
+
+class CarbonRow {
+private:
+ jmethodID getShortId;
+ jmethodID getIntId;
+ jmethodID getLongId;
+ jmethodID getDoubleId;
+ jmethodID getFloatId;
+ jmethodID getBooleanId;
+ jmethodID getStringId;
+ jmethodID getDecimalId;
+ jmethodID getVarcharId;
+ jmethodID getArrayId;
+
+ /**
+ * RowUtil Class for read data from Carbon Row
+ */
+ jclass rowUtilClass;
+
+ /**
+ * carbon row data
+ */
+ jobject carbonRow;
+
+ /**
+ * check ordinal, ordinal can't be negative
+ *
+ * @param ordinal int value, the data index of carbon Row
+ */
+ void checkOrdinal(int ordinal);
+
+public:
+
+ /**
+ * jni env
+ */
+ JNIEnv *jniEnv;
+
+ /**
+ * Constructor and express the carbon row result
+ *
+ * @param env JNI env
+ * @param jo carbon Row object
+ */
+ CarbonRow(JNIEnv *env);
+
+ /**
+ * set carbon row data
+ *
+ * @param data
+ */
+ void setCarbonRow(jobject data);
+
+ /**
+ * get short data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return short data type data
+ */
+ short getShort(int ordinal);
+
+ /**
+ * get int data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return int data type data
+ */
+ int getInt(int ordinal);
+
+ /**
+ * get long data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return long data type data
+ */
+ long getLong(int ordinal);
+
+ /**
+ * get double data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return double data type data
+ */
+ double getDouble(int ordinal);
+
+ /**
+ * get float data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return float data type data
+ */
+ float getFloat(int ordinal);
+
+ /**
+ * get boolean data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return jboolean data type data
+ */
+ jboolean getBoolean(int ordinal);
+
+ /**
+ * get decimal data type data by ordinal
+ * JNI don't support Decimal, so carbon convert decimal to string
+ *
+ * @param ordinal the data index of carbon Row
+ * @return string data type data
+ */
+ char *getDecimal(int ordinal);
+
+ /**
+ * get string data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return string data type data
+ */
+ char *getString(int ordinal);
+
+ /**
+ * get varchar data type data by ordinal
+ * JNI don't support varchar, so carbon convert decimal to string
+ *
+ * @param ordinal the data index of carbon Row
+ * @return string data type data
+ */
+ char *getVarchar(int ordinal);
+
+ /**
+ * get array<T> data type data by ordinal
+ *
+ * @param ordinal the data index of carbon Row
+ * @return jobjectArray data type data
+ */
+ jobjectArray getArray(int ordinal);
+};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/CSDK/test/main.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/test/main.cpp b/store/CSDK/test/main.cpp
new file mode 100644
index 0000000..843155f
--- /dev/null
+++ b/store/CSDK/test/main.cpp
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+#include <jni.h>
+#include <stdlib.h>
+#include <iostream>
+#include <unistd.h>
+#include "../src/CarbonReader.h"
+#include "../src/CarbonRow.h"
+
+using namespace std;
+
+JavaVM *jvm;
+
+/**
+ * init jvm
+ *
+ * @return
+ */
+JNIEnv *initJVM() {
+ JNIEnv *env;
+ JavaVMInitArgs vm_args;
+ int parNum = 3;
+ int res;
+ JavaVMOption options[parNum];
+
+ options[0].optionString = "-Djava.compiler=NONE";
+ options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
+ options[2].optionString = "-verbose:jni";
+ vm_args.version = JNI_VERSION_1_8;
+ vm_args.nOptions = parNum;
+ vm_args.options = options;
+ vm_args.ignoreUnrecognized = JNI_FALSE;
+
+ res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
+ if (res < 0) {
+ fprintf(stderr, "\nCan't create Java VM\n");
+ exit(1);
+ }
+
+ return env;
+}
+
+/**
+ * print array result
+ *
+ * @param env JNIEnv
+ * @param arr array
+ */
+void printArray(JNIEnv *env, jobjectArray arr) {
+ jsize length = env->GetArrayLength(arr);
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(arr, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ env->DeleteLocalRef(arr);
+}
+
+/**
+ * print boolean result
+ *
+ * @param env JNIEnv
+ * @param bool1 boolean value
+ */
+void printBoolean(jboolean bool1) {
+ if (bool1) {
+ printf("true\t");
+ } else {
+ printf("false\t");
+ }
+}
+
+/**
+ * print result of reading data
+ *
+ * @param env JNIEnv
+ * @param reader CarbonReader object
+ */
+void printResult(JNIEnv *env, CarbonReader reader) {
+ CarbonRow carbonRow(env);
+ while (reader.hasNext()) {
+ jobject row = reader.readNextRow();
+ carbonRow.setCarbonRow(row);
+ printf("%s\t", carbonRow.getString(0));
+ printf("%d\t", carbonRow.getInt(1));
+ printf("%ld\t", carbonRow.getLong(2));
+ printf("%s\t", carbonRow.getVarchar(3));
+ printArray(env, carbonRow.getArray(4));
+ printf("%d\t", carbonRow.getShort(5));
+ printf("%d\t", carbonRow.getInt(6));
+ printf("%ld\t", carbonRow.getLong(7));
+ printf("%lf\t", carbonRow.getDouble(8));
+ printBoolean(carbonRow.getBoolean(9));
+ printf("%s\t", carbonRow.getDecimal(10));
+ printf("%f\t", carbonRow.getFloat(11));
+ printf("\n");
+ env->DeleteLocalRef(row);
+ }
+ reader.close();
+}
+
+/**
+ * test read data from local disk, without projection
+ *
+ * @param env jni env
+ * @return
+ */
+bool readFromLocalWithoutProjection(JNIEnv *env) {
+ printf("\nRead data from local without projection:\n");
+
+ CarbonReader carbonReaderClass;
+ try {
+ carbonReaderClass.builder(env, "../../../../resources/carbondata");
+ } catch (runtime_error e) {
+ printf("\nget exception fro builder and throw\n");
+ throw e;
+ }
+ try {
+ carbonReaderClass.build();
+ } catch (jthrowable e) {
+ env->ExceptionDescribe();
+ }
+ printResult(env, carbonReaderClass);
+}
+
+/**
+ * test read data from local disk
+ *
+ * @param env jni env
+ * @return
+ */
+bool readFromLocal(JNIEnv *env) {
+ printf("\nRead data from local:\n");
+
+ CarbonReader reader;
+ reader.builder(env, "../../../../resources/carbondata", "test");
+
+ char *argv[12];
+ argv[0] = "stringField";
+ argv[1] = "shortField";
+ argv[2] = "intField";
+ argv[3] = "longField";
+ argv[4] = "doubleField";
+ argv[5] = "boolField";
+ argv[6] = "dateField";
+ argv[7] = "timeField";
+ argv[8] = "decimalField";
+ argv[9] = "varcharField";
+ argv[10] = "arrayField";
+ argv[11] = "floatField";
+ reader.projection(12, argv);
+
+ try {
+ reader.build();
+ } catch (jthrowable e) {
+ env->ExceptionDescribe();
+ }
+
+ CarbonRow carbonRow(env);
+ while (reader.hasNext()) {
+ jobject row = reader.readNextRow();
+ carbonRow.setCarbonRow(row);
+
+ printf("%s\t", carbonRow.getString(0));
+ printf("%d\t", carbonRow.getShort(1));
+ printf("%d\t", carbonRow.getInt(2));
+ printf("%ld\t", carbonRow.getLong(3));
+ printf("%lf\t", carbonRow.getDouble(4));
+ printBoolean(carbonRow.getBoolean(5));
+ printf("%d\t", carbonRow.getInt(6));
+ printf("%ld\t", carbonRow.getLong(7));
+ printf("%s\t", carbonRow.getDecimal(8));
+ printf("%s\t", carbonRow.getVarchar(9));
+ printArray(env, carbonRow.getArray(10));
+ printf("%f\t", carbonRow.getFloat(11));
+ printf("\n");
+ env->DeleteLocalRef(row);
+ }
+
+ reader.close();
+}
+
+
+bool tryCatchException(JNIEnv *env) {
+ printf("\ntry catch exception and print:\n");
+
+ CarbonReader carbonReaderClass;
+ carbonReaderClass.builder(env, "./carbondata");
+ try {
+ carbonReaderClass.build();
+ } catch (jthrowable e) {
+ env->ExceptionDescribe();
+ }
+ printf("\nfinished handle exception\n");
+}
+/**
+ * read data from S3
+ * parameter is ak sk endpoint
+ *
+ * @param env jni env
+ * @param argv argument vector
+ * @return
+ */
+bool readFromS3(JNIEnv *env, char *argv[]) {
+ printf("\nRead data from S3:\n");
+ CarbonReader reader;
+
+ char *args[3];
+ // "your access key"
+ args[0] = argv[1];
+ // "your secret key"
+ args[1] = argv[2];
+ // "your endPoint"
+ args[2] = argv[3];
+
+ reader.builder(env, "s3a://sdk/WriterOutput/carbondata/", "test");
+ reader.withHadoopConf("fs.s3a.access.key", argv[1]);
+ reader.withHadoopConf("fs.s3a.secret.key", argv[2]);
+ reader.withHadoopConf("fs.s3a.endpoint", argv[3]);
+ reader.build();
+ printResult(env, reader);
+}
+
+/**
+ * This a example for C++ interface to read carbon file
+ * If you want to test read data fromS3, please input the parameter: ak sk endpoint
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return
+ */
+int main(int argc, char *argv[]) {
+ // init jvm
+ JNIEnv *env;
+ env = initJVM();
+
+ if (argc > 3) {
+ readFromS3(env, argv);
+ } else {
+ tryCatchException(env);
+ readFromLocalWithoutProjection(env);
+ readFromLocal(env);
+ }
+ (jvm)->DestroyJavaVM();
+
+ cout << "\nfinish destroy jvm";
+ return 0;
+}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index 932cf85..a381429 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -23,7 +23,6 @@ import java.util.UUID;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonTaskInfo;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
@@ -92,35 +91,6 @@ public class CarbonReader<T> {
}
/**
- * Read and return next string row object
- * limitation: only single dimension Array is supported
- * TODO: support didfferent data type
- */
- public Object[] readNextStringRow() throws IOException, InterruptedException {
- validateReader();
- T t = currentReader.getCurrentValue();
- Object[] objects = (Object[]) t;
- String[] strings = new String[objects.length];
- for (int i = 0; i < objects.length; i++) {
- if (objects[i] instanceof Object[]) {
- Object[] arrayString = (Object[]) objects[i];
- StringBuffer stringBuffer = new StringBuffer();
- stringBuffer.append(String.valueOf(arrayString[0]));
- if (arrayString.length > 1) {
- for (int j = 1; j < arrayString.length; j++) {
- stringBuffer.append(CarbonCommonConstants.ARRAY_SEPARATOR)
- .append(String.valueOf(arrayString[j]));
- }
- }
- strings[i] = stringBuffer.toString();
- } else {
- strings[i] = String.valueOf(objects[i]);
- }
- }
- return strings;
- }
-
- /**
* Return a new {@link CarbonReaderBuilder} instance
*
* @param tablePath table store path
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
index 924835f..065ff80 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
@@ -240,6 +240,15 @@ public class Field {
}
}
+ @Override
+ public String toString() {
+ return "Field{" +
+ "name='" + name + '\'' +
+ ", type=" + type +
+ ", schemaOrdinal=" + schemaOrdinal +
+ '}';
+ }
+
/**
* prepare sub fields for complex types
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java
new file mode 100644
index 0000000..82b3904
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/RowUtil.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.file;
+
+import java.io.Serializable;
+import java.math.BigDecimal;
+
+/**
+ * This row class is used to transfer the row data from one step to other step
+ */
+public class RowUtil implements Serializable {
+
+ public static String getString(Object[] data, int ordinal) {
+ return (String) data[ordinal];
+ }
+
+ /**
+ * get short type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return short data type data
+ */
+ public static short getShort(Object[] data, int ordinal) {
+ return (short) data[ordinal];
+ }
+
+ /**
+ * get int data type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return int data type data
+ */
+ public static int getInt(Object[] data, int ordinal) {
+ return (Integer) data[ordinal];
+ }
+
+ /**
+ * get long data type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return long data type data
+ */
+ public static long getLong(Object[] data, int ordinal) {
+ return (long) data[ordinal];
+ }
+
+ /**
+ * get array data type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return array data type data
+ */
+ public static Object[] getArray(Object[] data, int ordinal) {
+ return (Object[]) data[ordinal];
+ }
+
+ /**
+ * get double data type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return double data type data
+ */
+ public static double getDouble(Object[] data, int ordinal) {
+ return (double) data[ordinal];
+ }
+
+ /**
+ * get boolean data type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return boolean data type data
+ */
+ public static boolean getBoolean(Object[] data, int ordinal) {
+ return (boolean) data[ordinal];
+ }
+
+ /**
+ * get byte data type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return byte data type data
+ */
+ public static Byte getByte(Object[] data, int ordinal) {
+ return (Byte) data[ordinal];
+ }
+
+ /**
+ * get float data type data by ordinal
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return float data type data
+ */
+ public static float getFloat(Object[] data, int ordinal) {
+ return (float) data[ordinal];
+ }
+
+ /**
+ * get varchar data type data by ordinal
+ * This is for CSDK
+ * JNI don't support varchar, so carbon convert varchar to string
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return string data type data
+ */
+ public static String getVarchar(Object[] data, int ordinal) {
+ return (String) data[ordinal];
+ }
+
+ /**
+ * get decimal data type data by ordinal
+ * This is for CSDK
+ * JNI don't support Decimal, so carbon convert decimal to string
+ *
+ * @param data carbon row data
+ * @param ordinal the data index of Row
+ * @return string data type data
+ */
+ public static String getDecimal(Object[] data, int ordinal) {
+ return ((BigDecimal) data[ordinal]).toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java
index f740ae2..09d664f 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/AvroCarbonWriterTest.java
@@ -61,7 +61,8 @@ public class AvroCarbonWriterTest {
}
@After
- public void verifyDMFile() {
+ public void verifyDMFile() throws IOException {
+ FileUtils.deleteDirectory(new File(path));
assert (!TestUtil.verifyMdtFile());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/019f5cd0/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
index c8b2c96..b8eb224 100644
--- a/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
+++ b/store/sdk/src/test/java/org/apache/carbondata/sdk/file/CarbonReaderTest.java
@@ -1529,4 +1529,212 @@ public class CarbonReaderTest extends TestCase {
e.printStackTrace();
}
}
+
+ @Test
+ public void testReadNextRowWithRowUtil() {
+ String path = "./carbondata";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[12];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("intField", DataTypes.INT);
+ fields[3] = new Field("longField", DataTypes.LONG);
+ fields[4] = new Field("doubleField", DataTypes.DOUBLE);
+ fields[5] = new Field("boolField", DataTypes.BOOLEAN);
+ fields[6] = new Field("dateField", DataTypes.DATE);
+ fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2));
+ fields[9] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING));
+ fields[11] = new Field("floatField", DataTypes.FLOAT);
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields))
+ .writtenBy("CarbonReaderTest")
+ .build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ String.valueOf(i),
+ String.valueOf(Long.MAX_VALUE - i),
+ String.valueOf((double) i / 2),
+ String.valueOf(true),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "12.345",
+ "varchar",
+ "Hello#World#From#Carbon",
+ "1.23"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+
+ File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name == null) {
+ return false;
+ }
+ return name.endsWith("carbonindex");
+ }
+ });
+ if (dataFiles == null || dataFiles.length < 1) {
+ throw new RuntimeException("Carbon index file not exists.");
+ }
+ Schema schema = CarbonSchemaReader
+ .readSchemaInIndexFile(dataFiles[0].getAbsolutePath())
+ .asOriginOrder();
+ // Transform the schema
+ int count = 0;
+ for (int i = 0; i < schema.getFields().length; i++) {
+ if (!((schema.getFields())[i].getFieldName().contains("."))) {
+ count++;
+ }
+ }
+ String[] strings = new String[count];
+ int index = 0;
+ for (int i = 0; i < schema.getFields().length; i++) {
+ if (!((schema.getFields())[i].getFieldName().contains("."))) {
+ strings[index] = (schema.getFields())[i].getFieldName();
+ index++;
+ }
+ }
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .projection(strings)
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] data = (Object[]) reader.readNextRow();
+
+ assert (RowUtil.getString(data, 0).equals("robot" + i));
+ assertEquals(RowUtil.getShort(data, 1), i);
+ assertEquals(RowUtil.getInt(data, 2), i);
+ assertEquals(RowUtil.getLong(data, 3), Long.MAX_VALUE - i);
+ assertEquals(RowUtil.getDouble(data, 4), ((double) i) / 2);
+ assert (RowUtil.getBoolean(data, 5));
+ assertEquals(RowUtil.getInt(data, 6), 17957);
+ assert (RowUtil.getDecimal(data, 8).equals("12.35"));
+ assert (RowUtil.getVarchar(data, 9).equals("varchar"));
+
+ Object[] arr = RowUtil.getArray(data, 10);
+ assert (arr[0].equals("Hello"));
+ assert (arr[1].equals("World"));
+ assert (arr[2].equals("From"));
+ assert (arr[3].equals("Carbon"));
+
+ assertEquals(RowUtil.getFloat(data, 11), (float) 1.23);
+ i++;
+ }
+ reader.close();
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ } finally {
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+ }
+
+ @Test
+ public void testReadNextRowWithProjectionAndRowUtil() {
+ String path = "./carbondata";
+ try {
+ FileUtils.deleteDirectory(new File(path));
+
+ Field[] fields = new Field[12];
+ fields[0] = new Field("stringField", DataTypes.STRING);
+ fields[1] = new Field("shortField", DataTypes.SHORT);
+ fields[2] = new Field("intField", DataTypes.INT);
+ fields[3] = new Field("longField", DataTypes.LONG);
+ fields[4] = new Field("doubleField", DataTypes.DOUBLE);
+ fields[5] = new Field("boolField", DataTypes.BOOLEAN);
+ fields[6] = new Field("dateField", DataTypes.DATE);
+ fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
+ fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2));
+ fields[9] = new Field("varcharField", DataTypes.VARCHAR);
+ fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING));
+ fields[11] = new Field("floatField", DataTypes.FLOAT);
+ Map<String, String> map = new HashMap<>();
+ map.put("complex_delimiter_level_1", "#");
+ CarbonWriter writer = CarbonWriter.builder()
+ .outputPath(path)
+ .withLoadOptions(map)
+ .withCsvInput(new Schema(fields))
+ .writtenBy("CarbonReaderTest")
+ .build();
+
+ for (int i = 0; i < 10; i++) {
+ String[] row2 = new String[]{
+ "robot" + (i % 10),
+ String.valueOf(i % 10000),
+ String.valueOf(i),
+ String.valueOf(Long.MAX_VALUE - i),
+ String.valueOf((double) i / 2),
+ String.valueOf(true),
+ "2019-03-02",
+ "2019-02-12 03:03:34",
+ "12.345",
+ "varchar",
+ "Hello#World#From#Carbon",
+ "1.23"
+ };
+ writer.write(row2);
+ }
+ writer.close();
+
+ // Read data
+ CarbonReader reader = CarbonReader
+ .builder(path, "_temp")
+ .build();
+
+ int i = 0;
+ while (reader.hasNext()) {
+ Object[] data = (Object[]) reader.readNextRow();
+
+ assert (RowUtil.getString(data, 0).equals("robot" + i));
+ assertEquals(RowUtil.getInt(data, 1), 17957);
+ assert (RowUtil.getVarchar(data, 3).equals("varchar"));
+ Object[] arr = RowUtil.getArray(data, 4);
+ assert (arr[0].equals("Hello"));
+ assert (arr[1].equals("World"));
+ assert (arr[2].equals("From"));
+ assert (arr[3].equals("Carbon"));
+ assertEquals(RowUtil.getShort(data, 5), i);
+ assertEquals(RowUtil.getInt(data, 6), i);
+ assertEquals(RowUtil.getLong(data, 7), Long.MAX_VALUE - i);
+ assertEquals(RowUtil.getDouble(data, 8), ((double) i) / 2);
+ assert (RowUtil.getBoolean(data, 9));
+ assert (RowUtil.getDecimal(data, 10).equals("12.35"));
+ assertEquals(RowUtil.getFloat(data, 11), (float) 1.23);
+ i++;
+ }
+ reader.close();
+ } catch (Throwable e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ } finally {
+ try {
+ FileUtils.deleteDirectory(new File(path));
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.fail(e.getMessage());
+ }
+ }
+ }
+
}