You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/10/09 15:50:26 UTC
[24/45] carbondata git commit: [CARBONDATA-2952] Provide CarbonReader
C++ interface for SDK
[CARBONDATA-2952] Provide CarbonReader C++ interface for SDK
1.init carbonreader,config data path and tablename
2.config ak sk endpoing for S3
3.configure projection
4.build carbon reader
5.hasNext
6.readNextRow
7.close
This closes #2738
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e9a198ab
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e9a198ab
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e9a198ab
Branch: refs/heads/branch-1.5
Commit: e9a198ab65c7bc9642e8a3186072f39aa2e5f0f6
Parents: d8a51c9
Author: xubo245 <xu...@huawei.com>
Authored: Thu Sep 20 18:35:34 2018 +0800
Committer: kunal642 <ku...@gmail.com>
Committed: Wed Oct 3 17:53:35 2018 +0530
----------------------------------------------------------------------
.gitignore | 1 +
.../core/constants/CarbonCommonConstants.java | 4 +
docs/CSDK-guide.md | 197 ++++++++++++++++++
examples/spark2/pom.xml | 2 +-
pom.xml | 2 +
store/CSDK/CMakeLists.txt | 17 ++
store/CSDK/CarbonReader.cpp | 103 ++++++++++
store/CSDK/CarbonReader.h | 105 ++++++++++
store/CSDK/main.cpp | 200 +++++++++++++++++++
store/sdk/pom.xml | 5 +
.../carbondata/sdk/file/CarbonReader.java | 30 +++
.../sdk/file/CarbonReaderBuilder.java | 16 ++
12 files changed, 681 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 5d66a40..00e4934 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@
.settings
.cache
target/
+store/CSDK/cmake-build-debug/*
.project
.classpath
metastore_db/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index faad0dc..46139c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -507,6 +507,10 @@ public final class CarbonCommonConstants {
*/
public static final String FILE_SEPARATOR = "/";
/**
+ * ARRAY separator
+ */
+ public static final String ARRAY_SEPARATOR = "\001";
+ /**
* MAX_QUERY_EXECUTION_TIME
*/
@CarbonProperty
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/docs/CSDK-guide.md
----------------------------------------------------------------------
diff --git a/docs/CSDK-guide.md b/docs/CSDK-guide.md
new file mode 100644
index 0000000..c4f4a31
--- /dev/null
+++ b/docs/CSDK-guide.md
@@ -0,0 +1,197 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to you under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+# CSDK Guide
+
+CarbonData CSDK provides C++ interface to write and read carbon file.
+CSDK use JNI to invoke java SDK in C++ code.
+
+
+# CSDK Reader
+This CSDK reader reads CarbonData file and carbonindex file at a given path.
+External client can make use of this reader to read CarbonData files in C++
+code and without CarbonSession.
+
+
+In the carbon jars package, there exist a carbondata-sdk.jar,
+including SDK reader for CSDK.
+## Quick example
+```
+// 1. init JVM
+JavaVM *jvm;
+JNIEnv *initJVM() {
+ JNIEnv *env;
+ JavaVMInitArgs vm_args;
+ int parNum = 3;
+ int res;
+ JavaVMOption options[parNum];
+
+ options[0].optionString = "-Djava.compiler=NONE";
+ options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
+ options[2].optionString = "-verbose:jni";
+ vm_args.version = JNI_VERSION_1_8;
+ vm_args.nOptions = parNum;
+ vm_args.options = options;
+ vm_args.ignoreUnrecognized = JNI_FALSE;
+
+ res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
+ if (res < 0) {
+ fprintf(stderr, "\nCan't create Java VM\n");
+ exit(1);
+ }
+
+ return env;
+}
+
+// 2. create carbon reader and read data
+// 2.1 read data from local disk
+/**
+ * test read data from local disk, without projection
+ *
+ * @param env jni env
+ * @return
+ */
+bool readFromLocalWithoutProjection(JNIEnv *env) {
+
+ CarbonReader carbonReaderClass;
+ carbonReaderClass.builder(env, "../resources/carbondata", "test");
+ carbonReaderClass.build();
+
+ while (carbonReaderClass.hasNext()) {
+ jobjectArray row = carbonReaderClass.readNextRow();
+ jsize length = env->GetArrayLength(row);
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(row, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ printf("\n");
+ }
+ carbonReaderClass.close();
+}
+
+// 2.2 read data from S3
+
+/**
+ * read data from S3
+ * parameter is ak sk endpoint
+ *
+ * @param env jni env
+ * @param argv argument vector
+ * @return
+ */
+bool readFromS3(JNIEnv *env, char *argv[]) {
+ CarbonReader reader;
+
+ char *args[3];
+ // "your access key"
+ args[0] = argv[1];
+ // "your secret key"
+ args[1] = argv[2];
+ // "your endPoint"
+ args[2] = argv[3];
+
+ reader.builder(env, "s3a://sdk/WriterOutput", "test");
+ reader.withHadoopConf(3, args);
+ reader.build();
+ printf("\nRead data from S3:\n");
+ while (reader.hasNext()) {
+ jobjectArray row = reader.readNextRow();
+ jsize length = env->GetArrayLength(row);
+
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(row, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ printf("\n");
+ }
+
+ reader.close();
+}
+
+// 3. destory JVM
+ (jvm)->DestroyJavaVM();
+```
+Find example code at main.cpp of CSDK module
+
+## API List
+```
+ /**
+ * create a CarbonReaderBuilder object for building carbonReader,
+ * CarbonReaderBuilder object can configure different parameter
+ *
+ * @param env JNIEnv
+ * @param path data store path
+ * @param tableName table name
+ * @return CarbonReaderBuilder object
+ */
+ jobject builder(JNIEnv *env, char *path, char *tableName);
+
+ /**
+ * Configure the projection column names of carbon reader
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return CarbonReaderBuilder object
+ */
+ jobject projection(int argc, char *argv[]);
+
+ /**
+ * build carbon reader with argument vector
+ * it support multiple parameter
+ * like: key=value
+ * for example: fs.s3a.access.key=XXXX, XXXX is user's access key value
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return CarbonReaderBuilder object
+ **/
+ jobject withHadoopConf(int argc, char *argv[]);
+
+ /**
+ * build carbonReader object for reading data
+ * it support read data from load disk
+ *
+ * @return carbonReader object
+ */
+ jobject build();
+
+ /**
+ * Whether it has next row data
+ *
+ * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
+ */
+ jboolean hasNext();
+
+ /**
+ * read next row from data
+ *
+ * @return object array of one row
+ */
+ jobjectArray readNextRow();
+
+ /**
+ * close the carbon reader
+ *
+ * @return boolean value
+ */
+ jboolean close();
+
+```
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/examples/spark2/pom.xml
----------------------------------------------------------------------
diff --git a/examples/spark2/pom.xml b/examples/spark2/pom.xml
index aa7b7c5..8dbab15 100644
--- a/examples/spark2/pom.xml
+++ b/examples/spark2/pom.xml
@@ -80,7 +80,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.2</version>
+ <version>${httpclient.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7b1d487..3dcce8d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,6 +113,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<snappy.version>1.1.2.6</snappy.version>
<hadoop.version>2.7.2</hadoop.version>
+ <httpclient.version>4.2.5</httpclient.version>
<scala.binary.version>2.11</scala.binary.version>
<scala.version>2.11.8</scala.version>
<hadoop.deps.scope>compile</hadoop.deps.scope>
@@ -483,6 +484,7 @@
<id>hadoop-2.8</id>
<properties>
<hadoop.version>2.8.3</hadoop.version>
+ <httpclient.version>4.5.2</httpclient.version>
</properties>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/store/CSDK/CMakeLists.txt b/store/CSDK/CMakeLists.txt
new file mode 100644
index 0000000..c3c57a0
--- /dev/null
+++ b/store/CSDK/CMakeLists.txt
@@ -0,0 +1,17 @@
+cmake_minimum_required (VERSION 2.8)
+project (CJDK)
+set(CMAKE_BUILD_TYPE Debug)
+SET (CMAKE_INSTALL_RPATH_USE_LINK_PATH true)
+
+find_package (JNI REQUIRED)
+
+include_directories (${JNI_INCLUDE_DIRS})
+
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+set (SOURCE_FILES CarbonReader.cpp CarbonReader.h main.cpp)
+
+add_executable (CJDK ${SOURCE_FILES})
+get_filename_component (JAVA_JVM_LIBRARY_DIR ${JAVA_JVM_LIBRARY} DIRECTORY)
+message (${JAVA_JVM_LIBRARY_DIR})
+target_link_libraries (CJDK ${JAVA_JVM_LIBRARY} )
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/CarbonReader.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/CarbonReader.cpp b/store/CSDK/CarbonReader.cpp
new file mode 100644
index 0000000..0400957
--- /dev/null
+++ b/store/CSDK/CarbonReader.cpp
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "CarbonReader.h"
+#include <jni.h>
+
+jobject CarbonReader::builder(JNIEnv *env, char *path, char *tableName) {
+
+ jniEnv = env;
+ jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+ jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+ "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+ jstring jpath = env->NewStringUTF(path);
+ jstring jtableName = env->NewStringUTF(tableName);
+ jvalue args[2];
+ args[0].l = jpath;
+ args[1].l = jtableName;
+ carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
+ return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::builder(JNIEnv *env, char *path) {
+ jniEnv = env;
+ jclass carbonReaderClass = env->FindClass("org/apache/carbondata/sdk/file/CarbonReader");
+ jmethodID carbonReaderBuilderID = env->GetStaticMethodID(carbonReaderClass, "builder",
+ "(Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+ jstring jpath = env->NewStringUTF(path);
+ jvalue args[1];
+ args[0].l = jpath;
+ carbonReaderBuilderObject = env->CallStaticObjectMethodA(carbonReaderClass, carbonReaderBuilderID, args);
+ return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::projection(int argc, char *argv[]) {
+ jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+ jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "projection",
+ "([Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+ jclass objectArrayClass = jniEnv->FindClass("Ljava/lang/String;");
+ jobjectArray array = jniEnv->NewObjectArray(argc, objectArrayClass, NULL);
+ for (int i = 0; i < argc; ++i) {
+ jstring value = jniEnv->NewStringUTF(argv[i]);
+ jniEnv->SetObjectArrayElement(array, i, value);
+ }
+
+ jvalue args[1];
+ args[0].l = array;
+ carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args);
+ return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::withHadoopConf(char *key, char *value) {
+ jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+ jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "withHadoopConf",
+ "(Ljava/lang/String;Ljava/lang/String;)Lorg/apache/carbondata/sdk/file/CarbonReaderBuilder;");
+
+ jvalue args[2];
+ args[0].l = jniEnv->NewStringUTF(key);
+ args[1].l = jniEnv->NewStringUTF(value);
+ carbonReaderBuilderObject = jniEnv->CallObjectMethodA(carbonReaderBuilderObject, buildID, args);
+ return carbonReaderBuilderObject;
+}
+
+jobject CarbonReader::build() {
+ jclass carbonReaderBuilderClass = jniEnv->GetObjectClass(carbonReaderBuilderObject);
+ jmethodID buildID = jniEnv->GetMethodID(carbonReaderBuilderClass, "build",
+ "()Lorg/apache/carbondata/sdk/file/CarbonReader;");
+ carbonReaderObject = jniEnv->CallObjectMethod(carbonReaderBuilderObject, buildID);
+ return carbonReaderObject;
+}
+
+jboolean CarbonReader::hasNext() {
+ jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+ jmethodID hasNextID = jniEnv->GetMethodID(carbonReader, "hasNext", "()Z");
+ unsigned char hasNext = jniEnv->CallBooleanMethod(carbonReaderObject, hasNextID);
+ return hasNext;
+}
+
+jobjectArray CarbonReader::readNextRow() {
+ jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+ jmethodID readNextRow2ID = jniEnv->GetMethodID(carbonReader, "readNextStringRow", "()[Ljava/lang/Object;");
+ jobjectArray row = (jobjectArray) jniEnv->CallObjectMethod(carbonReaderObject, readNextRow2ID);
+ return row;
+}
+
+jboolean CarbonReader::close() {
+ jclass carbonReader = jniEnv->GetObjectClass(carbonReaderObject);
+ jmethodID closeID = jniEnv->GetMethodID(carbonReader, "close", "()V");
+ jniEnv->CallBooleanMethod(carbonReaderObject, closeID);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/CarbonReader.h
----------------------------------------------------------------------
diff --git a/store/CSDK/CarbonReader.h b/store/CSDK/CarbonReader.h
new file mode 100644
index 0000000..861f704
--- /dev/null
+++ b/store/CSDK/CarbonReader.h
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <jni.h>
+
+class CarbonReader {
+public:
+ /**
+ * jni env
+ */
+ JNIEnv *jniEnv;
+
+ /**
+ * carbonReaderBuilder object for building carbonReader
+ * it can configure some operation
+ */
+ jobject carbonReaderBuilderObject;
+
+ /**
+ * carbonReader object for reading data
+ */
+ jobject carbonReaderObject;
+
+ /**
+ * create a CarbonReaderBuilder object for building carbonReader,
+ * CarbonReaderBuilder object can configure different parameter
+ *
+ * @param env JNIEnv
+ * @param path data store path
+ * @param tableName table name
+ * @return CarbonReaderBuilder object
+ */
+ jobject builder(JNIEnv *env, char *path, char *tableName);
+
+ /**
+ * create a CarbonReaderBuilder object for building carbonReader,
+ * CarbonReaderBuilder object can configure different parameter
+ *
+ * @param env JNIEnv
+ * @param path data store path
+ * @return CarbonReaderBuilder object
+ * */
+ jobject builder(JNIEnv *env, char *path);
+
+ /**
+ * Configure the projection column names of carbon reader
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return CarbonReaderBuilder object
+ */
+ jobject projection(int argc, char *argv[]);
+
+ /**
+ * configure parameter, including ak,sk and endpoint
+ *
+ * @param key key word
+ * @param value value
+ * @return CarbonReaderBuilder object
+ */
+ jobject withHadoopConf(char *key, char *value);
+
+ /**
+ * build carbonReader object for reading data
+ * it support read data from load disk
+ *
+ * @return carbonReader object
+ */
+ jobject build();
+
+ /**
+ * Whether it has next row data
+ *
+ * @return boolean value, if it has next row, return true. if it hasn't next row, return false.
+ */
+ jboolean hasNext();
+
+ /**
+ * read next row from data
+ *
+ * @return object array of one row
+ */
+ jobjectArray readNextRow();
+
+ /**
+ * close the carbon reader
+ *
+ * @return boolean value
+ */
+ jboolean close();
+};
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/CSDK/main.cpp
----------------------------------------------------------------------
diff --git a/store/CSDK/main.cpp b/store/CSDK/main.cpp
new file mode 100644
index 0000000..a8d1a87
--- /dev/null
+++ b/store/CSDK/main.cpp
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+#include <jni.h>
+#include <stdlib.h>
+#include <iostream>
+#include <unistd.h>
+#include "CarbonReader.h"
+
+using namespace std;
+
+JavaVM *jvm;
+
+/**
+ * init jvm
+ *
+ * @return
+ */
+JNIEnv *initJVM() {
+ JNIEnv *env;
+ JavaVMInitArgs vm_args;
+ int parNum = 3;
+ int res;
+ JavaVMOption options[parNum];
+
+ options[0].optionString = "-Djava.compiler=NONE";
+ options[1].optionString = "-Djava.class.path=../../sdk/target/carbondata-sdk.jar";
+ options[2].optionString = "-verbose:jni";
+ vm_args.version = JNI_VERSION_1_8;
+ vm_args.nOptions = parNum;
+ vm_args.options = options;
+ vm_args.ignoreUnrecognized = JNI_FALSE;
+
+ res = JNI_CreateJavaVM(&jvm, (void **) &env, &vm_args);
+ if (res < 0) {
+ fprintf(stderr, "\nCan't create Java VM\n");
+ exit(1);
+ }
+
+ return env;
+}
+
+/**
+ * test read data from local disk, without projection
+ *
+ * @param env jni env
+ * @return
+ */
+bool readFromLocalWithoutProjection(JNIEnv *env) {
+
+ CarbonReader carbonReaderClass;
+ carbonReaderClass.builder(env, "../resources/carbondata");
+ carbonReaderClass.build();
+
+ printf("\nRead data from local without projection:\n");
+
+ while (carbonReaderClass.hasNext()) {
+ jobjectArray row = carbonReaderClass.readNextRow();
+ jsize length = env->GetArrayLength(row);
+
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(row, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ printf("\n");
+ }
+
+ carbonReaderClass.close();
+}
+
+/**
+ * test read data from local disk
+ *
+ * @param env jni env
+ * @return
+ */
+bool readFromLocal(JNIEnv *env) {
+
+ CarbonReader reader;
+ reader.builder(env, "../resources/carbondata", "test");
+
+ char *argv[11];
+ argv[0] = "stringField";
+ argv[1] = "shortField";
+ argv[2] = "intField";
+ argv[3] = "longField";
+ argv[4] = "doubleField";
+ argv[5] = "boolField";
+ argv[6] = "dateField";
+ argv[7] = "timeField";
+ argv[8] = "decimalField";
+ argv[9] = "varcharField";
+ argv[10] = "arrayField";
+ reader.projection(11, argv);
+
+ reader.build();
+
+ printf("\nRead data from local:\n");
+
+ while (reader.hasNext()) {
+ jobjectArray row = reader.readNextRow();
+ jsize length = env->GetArrayLength(row);
+
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(row, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ printf("\n");
+ }
+
+ reader.close();
+}
+
+/**
+ * read data from S3
+ * parameter is ak sk endpoint
+ *
+ * @param env jni env
+ * @param argv argument vector
+ * @return
+ */
+bool readFromS3(JNIEnv *env, char *argv[]) {
+ CarbonReader reader;
+
+ char *args[3];
+ // "your access key"
+ args[0] = argv[1];
+ // "your secret key"
+ args[1] = argv[2];
+ // "your endPoint"
+ args[2] = argv[3];
+
+ reader.builder(env, "s3a://sdk/WriterOutput", "test");
+ reader.withHadoopConf("fs.s3a.access.key", argv[1]);
+ reader.withHadoopConf("fs.s3a.secret.key", argv[2]);
+ reader.withHadoopConf("fs.s3a.endpoint", argv[3]);
+ reader.build();
+ printf("\nRead data from S3:\n");
+ while (reader.hasNext()) {
+ jobjectArray row = reader.readNextRow();
+ jsize length = env->GetArrayLength(row);
+
+ int j = 0;
+ for (j = 0; j < length; j++) {
+ jobject element = env->GetObjectArrayElement(row, j);
+ char *str = (char *) env->GetStringUTFChars((jstring) element, JNI_FALSE);
+ printf("%s\t", str);
+ }
+ printf("\n");
+ }
+
+ reader.close();
+}
+
+/**
+ * This a example for C++ interface to read carbon file
+ * If you want to test read data fromS3, please input the parameter: ak sk endpoint
+ *
+ * @param argc argument counter
+ * @param argv argument vector
+ * @return
+ */
+int main(int argc, char *argv[]) {
+ // init jvm
+ JNIEnv *env;
+ env = initJVM();
+
+ if (argc > 3) {
+ readFromS3(env, argv);
+ } else {
+ readFromLocalWithoutProjection(env);
+ readFromLocal(env);
+ }
+ cout << "destory jvm\n\n";
+ (jvm)->DestroyJavaVM();
+
+ cout << "\nfinish destory jvm";
+ fprintf(stdout, "Java VM destory.\n");
+ return 0;
+}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index ea720a2..2424113 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -39,6 +39,11 @@
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpclient.version}</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
index be809e6..1f1794c 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReader.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonTaskInfo;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
@@ -91,6 +92,35 @@ public class CarbonReader<T> {
}
/**
+ * Read and return next string row object
+ * limitation: only single dimension Array is supported
+ * TODO: support didfferent data type
+ */
+ public Object[] readNextStringRow() throws IOException, InterruptedException {
+ validateReader();
+ T t = currentReader.getCurrentValue();
+ Object[] objects = (Object[]) t;
+ String[] strings = new String[objects.length];
+ for (int i = 0; i < objects.length; i++) {
+ if (objects[i] instanceof Object[]) {
+ Object[] arrayString = (Object[]) objects[i];
+ StringBuffer stringBuffer = new StringBuffer();
+ stringBuffer.append(String.valueOf(arrayString[0]));
+ if (arrayString.length > 1) {
+ for (int j = 1; j < arrayString.length; j++) {
+ stringBuffer.append(CarbonCommonConstants.ARRAY_SEPARATOR)
+ .append(String.valueOf(arrayString[j]));
+ }
+ }
+ strings[i] = stringBuffer.toString();
+ } else {
+ strings[i] = String.valueOf(objects[i]);
+ }
+ }
+ return strings;
+ }
+
+ /**
* Return a new {@link CarbonReaderBuilder} instance
*
* @param tablePath table store path
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e9a198ab/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
index af3480f..151d57c 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonReaderBuilder.java
@@ -102,6 +102,22 @@ public class CarbonReaderBuilder {
}
/**
+ * configure hadoop configuration with key value
+ *
+ * @param key key word
+ * @param value value
+ * @return this object
+ */
+ public CarbonReaderBuilder withHadoopConf(String key, String value) {
+ if (this.hadoopConf == null) {
+ this.hadoopConf = new Configuration();
+
+ }
+ this.hadoopConf.set(key, value);
+ return this;
+ }
+
+ /**
* Build CarbonReader
*
* @param <T>