You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by qi...@apache.org on 2018/08/06 11:04:48 UTC
[1/5] carbondata git commit: [CARBONDATA-2825][CARBONDATA-2828]
CarbonStore and InternalCarbonStore API This closes #2589
Repository: carbondata
Updated Branches:
refs/heads/carbonstore a6027ae11 -> 9f10122af
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStoreFactory.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStoreFactory.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStoreFactory.java
new file mode 100644
index 0000000..fcbd88d
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStoreFactory.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+/**
+ * Factory class to create {@link CarbonStore}
+ */
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public class CarbonStoreFactory {
+ private static final Map<String, CarbonStore> distributedStore = new ConcurrentHashMap<>();
+ private static final Map<String, CarbonStore> localStores = new ConcurrentHashMap<>();
+
+ private CarbonStoreFactory() {
+ }
+
+ public static synchronized CarbonStore getDistributedStore(
+ String storeName, StoreConf storeConf) throws CarbonException {
+ if (distributedStore.containsKey(storeName)) {
+ return distributedStore.get(storeName);
+ }
+
+ // create a new instance
+ try {
+ String className = "org.apache.carbondata.sdk.store.DistributedCarbonStore";
+ CarbonStore store = createCarbonStore(storeConf, className);
+ distributedStore.put(storeName, store);
+ return store;
+ } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException |
+ InstantiationException e) {
+ throw new CarbonException(e);
+ }
+ }
+
+ public static synchronized void removeDistributedStore(String storeName) throws IOException {
+ if (distributedStore.containsKey(storeName)) {
+ distributedStore.get(storeName).close();
+ distributedStore.remove(storeName);
+ }
+ }
+
+ public static synchronized CarbonStore getLocalStore(String storeName, StoreConf storeConf)
+ throws CarbonException {
+ if (localStores.containsKey(storeName)) {
+ return localStores.get(storeName);
+ }
+
+ // create a new instance
+ try {
+ String className = "org.apache.carbondata.store.impl.LocalCarbonStore";
+ CarbonStore store = createCarbonStore(storeConf, className);
+ localStores.put(storeName, store);
+ return store;
+ } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException |
+ InstantiationException e) {
+ throw new CarbonException(e);
+ }
+ }
+
+ private static CarbonStore createCarbonStore(StoreConf storeConf, String className)
+ throws ClassNotFoundException, InstantiationException, IllegalAccessException,
+ InvocationTargetException {
+ Constructor[] constructor = Class.forName(className).getDeclaredConstructors();
+ constructor[0].setAccessible(true);
+ return (CarbonStore) constructor[0].newInstance(storeConf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/DistributedCarbonStore.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/DistributedCarbonStore.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/DistributedCarbonStore.java
new file mode 100644
index 0000000..7834909
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/DistributedCarbonStore.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.datatype.StructType;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.sdk.store.service.ServiceFactory;
+import org.apache.carbondata.sdk.store.service.StoreService;
+
+/**
+ * A CarbonStore that leverage multiple servers via RPC calls (Master and Workers)
+ */
+@InterfaceAudience.User
+public class DistributedCarbonStore implements CarbonStore {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DistributedCarbonStore.class.getCanonicalName());
+
+ protected StoreService storeService;
+
+ public DistributedCarbonStore(StoreConf conf) throws IOException {
+ this.storeService =
+ ServiceFactory.createStoreService(conf.masterHost(), conf.storeServicePort());
+ }
+
+ @Override
+ public void createTable(TableDescriptor descriptor) throws CarbonException {
+ try {
+ storeService.createTable(descriptor);
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ }
+ }
+
+ @Override
+ public void dropTable(TableIdentifier table) throws CarbonException {
+ try {
+ storeService.dropTable(table);
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ }
+ }
+
+ @Override
+ public List<TableDescriptor> listTable() throws CarbonException {
+ return storeService.listTable();
+ }
+
+ @Override
+ public TableDescriptor getDescriptor(TableIdentifier table) throws CarbonException {
+ TableInfo tableInfo = storeService.getTable(table);
+ // TODO: create TableDescriptor from table info
+ return null;
+ }
+
+ @Override
+ public void alterTable(TableIdentifier table, TableDescriptor newTable) throws CarbonException {
+ storeService.alterTable(table, newTable);
+ }
+
+ @Override
+ public void loadData(LoadDescriptor load) throws CarbonException {
+ storeService.loadData(load);
+ }
+
+ @Override
+ public void upsert(Iterator<KeyedRow> row, StructType schema) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void delete(Iterator<PrimaryKey> keys) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<CarbonRow> scan(ScanDescriptor select) throws CarbonException {
+ try {
+ return storeService.scan(select);
+ } catch (Exception e) {
+ System.out.println(e.getMessage());
+ return null;
+ }
+ }
+
+ @Override
+ public Row lookup(PrimaryKey key) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<Row> lookup(TableIdentifier tableIdentifier, String filterExpression)
+ throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/KeyedRow.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/KeyedRow.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/KeyedRow.java
new file mode 100644
index 0000000..bff02e0
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/KeyedRow.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public class KeyedRow {
+ private PrimaryKey key;
+ private CarbonRow row;
+
+ public void setKey(PrimaryKey key) {
+ this.key = key;
+ }
+
+ public void setRow(CarbonRow row) {
+ this.row = row;
+ }
+
+ public PrimaryKey getKey() {
+ return key;
+ }
+
+ public CarbonRow getRow() {
+ return row;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/PrimaryKey.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/PrimaryKey.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/PrimaryKey.java
new file mode 100644
index 0000000..e18c5ad
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/PrimaryKey.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public class PrimaryKey {
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/Row.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/Row.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/Row.java
new file mode 100644
index 0000000..97d2e14
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/Row.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public interface Row {
+ Object[] get();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/conf/StoreConf.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/conf/StoreConf.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/conf/StoreConf.java
new file mode 100644
index 0000000..c6c7cf6
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/conf/StoreConf.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.conf;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.sdk.store.util.StoreUtil;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+public class StoreConf implements Serializable, Writable {
+
+ public static final String SELECT_PROJECTION = "carbon.select.projection";
+ public static final String SELECT_FILTER = "carbon.select.filter";
+ public static final String SELECT_LIMIT = "carbon.select.limit";
+
+ public static final String SELECT_ID = "carbon.select.id";
+
+ public static final String WORKER_HOST = "carbon.worker.host";
+ public static final String WORKER_PORT = "carbon.worker.port";
+ public static final String WORKER_CORE_NUM = "carbon.worker.core.num";
+ public static final String MASTER_HOST = "carbon.master.host";
+ public static final String REGISTRY_PORT = "carbon.master.registry.port";
+ public static final String PRUNE_PORT = "carbon.master.prune.port";
+ public static final String STORE_PORT = "carbon.master.store.port";
+
+ public static final String STORE_TEMP_LOCATION = "carbon.store.temp.location";
+ public static final String STORE_LOCATION = "carbon.store.location";
+ public static final String STORE_NAME = "carbon.store.name";
+
+ public static final String STORE_CONF_FILE = "carbon.store.confFile";
+
+ private Map<String, String> conf = new HashMap<>();
+
+ public StoreConf() {
+ String storeConfFile = System.getProperty(STORE_CONF_FILE);
+ if (storeConfFile != null) {
+ load(storeConfFile);
+ }
+ }
+
+ public StoreConf(String storeName, String storeLocation) {
+ conf.put(STORE_NAME, storeName);
+ conf.put(STORE_LOCATION, storeLocation);
+ }
+
+ public StoreConf(String confFilePath) {
+ load(confFilePath);
+ }
+
+ public StoreConf conf(String key, String value) {
+ conf.put(key, value);
+ return this;
+ }
+
+ public StoreConf conf(String key, int value) {
+ conf.put(key, "" + value);
+ return this;
+ }
+
+ public void load(String filePath) {
+ StoreUtil.loadProperties(filePath, this);
+ }
+
+ public void conf(StoreConf conf) {
+ this.conf.putAll(conf.conf);
+ }
+
+ public Object conf(String key) {
+ return conf.get(key);
+ }
+
+ public String[] projection() {
+ return stringArrayValue(SELECT_PROJECTION);
+ }
+
+ public String filter() {
+ return stringValue(SELECT_FILTER);
+ }
+
+ public int limit() {
+ return intValue(SELECT_LIMIT);
+ }
+
+ public String masterHost() {
+ return stringValue(MASTER_HOST);
+ }
+
+ public int registryServicePort() {
+ return intValue(REGISTRY_PORT);
+ }
+
+ public int pruneServicePort() {
+ return intValue(PRUNE_PORT);
+ }
+
+ public int storeServicePort() {
+ return intValue(STORE_PORT);
+ }
+
+ public String workerHost() {
+ return stringValue(WORKER_HOST);
+ }
+
+ public int workerPort() {
+ return intValue(WORKER_PORT);
+ }
+
+ public int workerCoreNum() {
+ return intValue(WORKER_CORE_NUM);
+ }
+
+ public String storeLocation() {
+ return stringValue(STORE_LOCATION);
+ }
+
+ public String[] storeTempLocation() {
+ return stringArrayValue(STORE_TEMP_LOCATION);
+ }
+
+ public String selectId() {
+ return stringValue(SELECT_ID);
+ }
+
+ public Configuration newHadoopConf() {
+ Configuration hadoopConf = FileFactory.getConfiguration();
+ for (Map.Entry<String, String> entry : conf.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ if (key != null && value != null && key.startsWith("carbon.hadoop.")) {
+ hadoopConf.set(key.substring("carbon.hadoop.".length()), value);
+ }
+ }
+ return hadoopConf;
+ }
+
+ private String stringValue(String key) {
+ Object obj = conf.get(key);
+ if (obj == null) {
+ return null;
+ }
+ return obj.toString();
+ }
+
+ private int intValue(String key) {
+ String value = conf.get(key);
+ if (value == null) {
+ return -1;
+ }
+ return Integer.parseInt(value);
+ }
+
+ private String[] stringArrayValue(String key) {
+ String value = conf.get(key);
+ if (value == null) {
+ return null;
+ }
+ return value.split(",", -1);
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ Set<Map.Entry<String, String>> entries = conf.entrySet();
+ WritableUtils.writeVInt(out, conf.size());
+ for (Map.Entry<String, String> entry : entries) {
+ WritableUtils.writeString(out, entry.getKey());
+ WritableUtils.writeString(out, entry.getValue());
+ }
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ if (conf == null) {
+ conf = new HashMap<>();
+ }
+
+ int size = WritableUtils.readVInt(in);
+ String key, value;
+ for (int i = 0; i < size; i++) {
+ key = WritableUtils.readString(in);
+ value = WritableUtils.readString(in);
+ conf.put(key, value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/LoadDescriptor.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/LoadDescriptor.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/LoadDescriptor.java
new file mode 100644
index 0000000..fc4308e
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/LoadDescriptor.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.descriptor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class LoadDescriptor implements Writable {
+
+ private TableIdentifier table;
+ private String inputPath;
+ private Map<String, String> options;
+ private boolean isOverwrite;
+
+ private LoadDescriptor() {
+ }
+
+ public LoadDescriptor(TableIdentifier table, String inputPath,
+ Map<String, String> options, boolean isOverwrite) {
+ Objects.requireNonNull(table);
+ Objects.requireNonNull(inputPath);
+ this.table = table;
+ this.inputPath = inputPath;
+ this.options = options;
+ this.isOverwrite = isOverwrite;
+ }
+
+ public TableIdentifier getTable() {
+ return table;
+ }
+
+ public void setTable(TableIdentifier table) {
+ this.table = table;
+ }
+
+ public String getInputPath() {
+ return inputPath;
+ }
+
+ public void setInputPath(String inputPath) {
+ this.inputPath = inputPath;
+ }
+
+ public Map<String, String> getOptions() {
+ return options;
+ }
+
+ public void setOptions(Map<String, String> options) {
+ this.options = options;
+ }
+
+ public boolean isOverwrite() {
+ return isOverwrite;
+ }
+
+ public void setOverwrite(boolean overwrite) {
+ isOverwrite = overwrite;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ table.write(out);
+ out.writeUTF(inputPath);
+ out.writeInt(options.size());
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ out.writeUTF(entry.getKey());
+ out.writeUTF(entry.getValue());
+ }
+ out.writeBoolean(isOverwrite);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ table = new TableIdentifier();
+ table.readFields(in);
+ inputPath = in.readUTF();
+ int size = in.readInt();
+ options = new HashMap<>(size);
+ for (int i = 0; i < size; i++) {
+ options.put(in.readUTF(), in.readUTF());
+ }
+ isOverwrite = in.readBoolean();
+ }
+
+ public static class Builder {
+ private LoadDescriptor load;
+ private Map<String, String> options;
+
+ private Builder() {
+ load = new LoadDescriptor();
+ options = new HashMap<>();
+ }
+
+ public Builder table(TableIdentifier tableIdentifier) {
+ load.setTable(tableIdentifier);
+ return this;
+ }
+
+ public Builder overwrite(boolean isOverwrite) {
+ load.setOverwrite(isOverwrite);
+ return this;
+ }
+
+ public Builder inputPath(String inputPath) {
+ load.setInputPath(inputPath);
+ return this;
+ }
+
+ public Builder options(String key, String value) {
+ options.put(key, value);
+ return this;
+ }
+
+ public LoadDescriptor create() {
+ load.setOptions(options);
+ return load;
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/ScanDescriptor.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/ScanDescriptor.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/ScanDescriptor.java
new file mode 100644
index 0000000..f580fc5
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/ScanDescriptor.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.descriptor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.util.ObjectSerializationUtil;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class ScanDescriptor implements Serializable, Writable {
+
+ private TableIdentifier table;
+ private String[] projection;
+ private Expression filter;
+ private long limit = Long.MAX_VALUE;
+
+ private ScanDescriptor() {
+ }
+
+ public ScanDescriptor(TableIdentifier table, String[] projection,
+ Expression filter, long limit) {
+ Objects.requireNonNull(table);
+ Objects.requireNonNull(projection);
+ this.table = table;
+ this.projection = projection;
+ this.filter = filter;
+ this.limit = limit;
+ }
+
+ public TableIdentifier getTableIdentifier() {
+ return table;
+ }
+
+ public void setTableIdentifier(TableIdentifier table) {
+ this.table = table;
+ }
+
+ public String[] getProjection() {
+ return projection;
+ }
+
+ public void setProjection(String[] projection) {
+ this.projection = projection;
+ }
+
+ public Expression getFilter() {
+ return filter;
+ }
+
+ public void setFilter(Expression filter) {
+ this.filter = filter;
+ }
+
+ public long getLimit() {
+ return limit;
+ }
+
+ public void setLimit(long limit) {
+ this.limit = limit;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ table.write(out);
+ out.writeInt(projection.length);
+ for (String s : projection) {
+ out.writeUTF(s);
+ }
+ out.writeBoolean(filter != null);
+ if (filter != null) {
+ out.writeUTF(ObjectSerializationUtil.convertObjectToString(filter));
+ }
+ out.writeLong(limit);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ table = new TableIdentifier();
+ table.readFields(in);
+ int size = in.readInt();
+ projection = new String[size];
+ for (int i = 0; i < size; i++) {
+ projection[i] = in.readUTF();
+ }
+ if (in.readBoolean()) {
+ filter = (Expression) ObjectSerializationUtil.convertStringToObject(in.readUTF());
+ }
+ limit = in.readLong();
+ }
+
+ public static class Builder {
+ private ScanDescriptor select;
+
+ private Builder() {
+ select = new ScanDescriptor();
+ }
+
+ public Builder table(TableIdentifier tableIdentifier) {
+ select.setTableIdentifier(tableIdentifier);
+ return this;
+ }
+
+ public Builder select(String[] columnNames) {
+ select.setProjection(columnNames);
+ return this;
+ }
+
+ public Builder filter(Expression filter) {
+ select.setFilter(filter);
+ return this;
+ }
+
+ public Builder limit(long limit) {
+ select.setLimit(limit);
+ return this;
+ }
+
+ public ScanDescriptor create() {
+ return select;
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableDescriptor.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableDescriptor.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableDescriptor.java
new file mode 100644
index 0000000..bdd5948
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableDescriptor.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.descriptor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.util.ObjectSerializationUtil;
+import org.apache.carbondata.sdk.file.Field;
+import org.apache.carbondata.sdk.file.Schema;
+
+import org.apache.hadoop.io.Writable;
+
+public class TableDescriptor implements Writable {
+
+ private TableIdentifier table;
+ private boolean ifNotExists;
+ private String tablePath;
+ private Schema schema;
+ private Map<String, String> properties;
+ private String comment;
+
+ private TableDescriptor() {
+ }
+
+ public TableDescriptor(TableIdentifier table, Schema schema,
+ Map<String, String> properties, String tablePath, String comment, boolean ifNotExists) {
+ Objects.requireNonNull(table);
+ Objects.requireNonNull(schema);
+ this.table = table;
+ this.ifNotExists = ifNotExists;
+ this.schema = schema;
+ this.properties = properties;
+ this.tablePath = tablePath;
+ this.comment = comment;
+ }
+
+ public boolean isIfNotExists() {
+ return ifNotExists;
+ }
+
+ public void setIfNotExists(boolean ifNotExists) {
+ this.ifNotExists = ifNotExists;
+ }
+
+ public TableIdentifier getTable() {
+ return table;
+ }
+
+ public void setTable(TableIdentifier table) {
+ this.table = table;
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ public void setSchema(Schema schema) {
+ this.schema = schema;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public void setTablePath(String tablePath) {
+ this.tablePath = tablePath;
+ }
+
+ public String getTablePath() {
+ return tablePath;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ table.write(out);
+ out.writeBoolean(ifNotExists);
+ out.writeBoolean(tablePath != null);
+ if (tablePath != null) {
+ out.writeUTF(tablePath);
+ }
+ out.writeUTF(ObjectSerializationUtil.convertObjectToString(schema));
+ out.writeInt(properties.size());
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ out.writeUTF(entry.getKey());
+ out.writeUTF(entry.getValue());
+ }
+ out.writeUTF(comment);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ table = new TableIdentifier();
+ table.readFields(in);
+ ifNotExists = in.readBoolean();
+ if (in.readBoolean()) {
+ tablePath = in.readUTF();
+ }
+ schema = (Schema) ObjectSerializationUtil.convertStringToObject(in.readUTF());
+ int size = in.readInt();
+ properties = new HashMap<>(size);
+ for (int i = 0; i < size; i++) {
+ properties.put(in.readUTF(), in.readUTF());
+ }
+ comment = in.readUTF();
+ }
+
+ public static class Builder {
+
+ private TableDescriptor table;
+ private List<Field> fields;
+ private Map<String, String> tblProperties;
+
+ private Builder() {
+ table = new TableDescriptor();
+ fields = new ArrayList<>();
+ tblProperties = new HashMap<>();
+ }
+
+ public Builder ifNotExists() {
+ table.setIfNotExists(true);
+ return this;
+ }
+
+ public Builder table(TableIdentifier tableId) {
+ table.setTable(tableId);
+ return this;
+ }
+
+ public Builder tablePath(String tablePath) {
+ table.setTablePath(tablePath);
+ return this;
+ }
+
+ public Builder comment(String tableComment) {
+ table.setComment(tableComment);
+ return this;
+ }
+
+ public Builder column(String name, DataType dataType) {
+ fields.add(new Field(name, dataType));
+ return this;
+ }
+
+ public Builder column(String name, DataType dataType, String comment) {
+ Field field = new Field(name, dataType);
+ field.setColumnComment(comment);
+ fields.add(field);
+ return this;
+ }
+
+ public Builder column(String name, DataType dataType, int precision, int scale, String comment)
+ {
+ Field field = new Field(name, dataType);
+ field.setColumnComment(comment);
+ field.setScale(scale);
+ field.setPrecision(precision);
+ fields.add(field);
+ return this;
+ }
+
+ public Builder tblProperties(String key, String value) {
+ tblProperties.put(key, value);
+ return this;
+ }
+
+ public TableDescriptor create() {
+ Field[] fieldArray = new Field[fields.size()];
+ fieldArray = fields.toArray(fieldArray);
+ Schema schema = new Schema(fieldArray);
+ table.setSchema(schema);
+ table.setProperties(tblProperties);
+ return table;
+ }
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableIdentifier.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableIdentifier.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableIdentifier.java
new file mode 100644
index 0000000..c67a748
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/descriptor/TableIdentifier.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.descriptor;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class TableIdentifier implements Serializable, Writable {
+ private String tableName;
+ private String databaseName;
+
+ public TableIdentifier() {
+ }
+
+ public TableIdentifier(String tableName, String databaseName) {
+ this.tableName = tableName;
+ this.databaseName = databaseName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(databaseName);
+ out.writeUTF(tableName);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.databaseName = in.readUTF();
+ this.tableName = in.readUTF();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/CarbonException.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/CarbonException.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/CarbonException.java
new file mode 100644
index 0000000..e4e0b10
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/CarbonException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.exception;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class CarbonException extends Exception {
+
+ public CarbonException() {
+ super();
+ }
+
+ public CarbonException(String message) {
+ super(message);
+ }
+
+ public CarbonException(Exception e) {
+ super(e);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/ExecutionTimeoutException.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/ExecutionTimeoutException.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/ExecutionTimeoutException.java
new file mode 100644
index 0000000..3ddd440
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/ExecutionTimeoutException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.exception;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class ExecutionTimeoutException extends RuntimeException {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/SchedulerException.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/SchedulerException.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/SchedulerException.java
new file mode 100644
index 0000000..8a0309f
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/exception/SchedulerException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.exception;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.User
+@InterfaceStability.Evolving
+public class SchedulerException extends RuntimeException {
+
+ public SchedulerException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/ServiceFactory.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/ServiceFactory.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/ServiceFactory.java
new file mode 100644
index 0000000..925ed91
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/ServiceFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.service;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+
+@InterfaceAudience.Internal
+public class ServiceFactory {
+
+ public static StoreService createStoreService(String host, int port) throws IOException {
+ InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(host), port);
+ return RPC.getProxy(StoreService.class, StoreService.versionID, address, new Configuration());
+ }
+
+ public static void stopStoreService(StoreService service) {
+ RPC.stopProxy(service);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/StoreService.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/StoreService.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/StoreService.java
new file mode 100644
index 0000000..7e54d2f
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/service/StoreService.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.service;
+
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+import org.apache.hadoop.ipc.VersionedProtocol;
+
+@InterfaceAudience.Internal
+public interface StoreService extends VersionedProtocol {
+ long versionID = 1L;
+
+ void createTable(TableDescriptor descriptor) throws CarbonException;
+
+ void dropTable(TableIdentifier table) throws CarbonException;
+
+ TableInfo getTable(TableIdentifier table) throws CarbonException;
+
+ List<TableDescriptor> listTable() throws CarbonException;
+
+ TableDescriptor getDescriptor(TableIdentifier table) throws CarbonException;
+
+ void alterTable(TableIdentifier table, TableDescriptor newTable) throws CarbonException;
+
+ void loadData(LoadDescriptor loadDescriptor) throws CarbonException;
+
+ List<CarbonRow> scan(ScanDescriptor scanDescriptor) throws CarbonException;
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/util/StoreUtil.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/util/StoreUtil.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/util/StoreUtil.java
new file mode 100644
index 0000000..ca5740e
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/util/StoreUtil.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.UnsafeMemoryManager;
+import org.apache.carbondata.core.memory.UnsafeSortMemoryManager;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.BasicConfigurator;
+import org.apache.log4j.PropertyConfigurator;
+
+@InterfaceAudience.Internal
+public class StoreUtil {
+
+ private static LogService LOGGER = LogServiceFactory.getLogService(StoreUtil.class.getName());
+
+ public static void loadProperties(String filePath, StoreConf conf) {
+ InputStream input = null;
+ try {
+ input = new FileInputStream(filePath);
+ Properties prop = new Properties();
+ prop.load(input);
+ for (Map.Entry<Object, Object> entry : prop.entrySet()) {
+ conf.conf(entry.getKey().toString(), entry.getValue().toString());
+ }
+ LOGGER.audit("loaded properties: " + filePath);
+ } catch (IOException ex) {
+ LOGGER.error(ex, "Failed to load properties file " + filePath);
+ } finally {
+ if (input != null) {
+ try {
+ input.close();
+ } catch (IOException e) {
+ LOGGER.error(e);
+ }
+ }
+ }
+ }
+
+ public static void initLog4j(String propertiesFilePath) {
+ BasicConfigurator.configure();
+ PropertyConfigurator.configure(propertiesFilePath);
+ }
+
+ public static byte[] serialize(Object object) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
+ try {
+ ObjectOutputStream oos = new ObjectOutputStream(baos);
+ oos.writeObject(object);
+ } catch (IOException e) {
+ LOGGER.error(e);
+ }
+ return baos.toByteArray();
+ }
+
+ public static Object deserialize(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ try {
+ ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes));
+ return ois.readObject();
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } catch (ClassNotFoundException e) {
+ LOGGER.error(e);
+ }
+ return null;
+ }
+
+ public static void configureCSVInputFormat(Configuration configuration,
+ CarbonLoadModel carbonLoadModel) {
+ CSVInputFormat.setCommentCharacter(configuration, carbonLoadModel.getCommentChar());
+ CSVInputFormat.setCSVDelimiter(configuration, carbonLoadModel.getCsvDelimiter());
+ CSVInputFormat.setSkipEmptyLine(configuration, carbonLoadModel.getSkipEmptyLine());
+ CSVInputFormat.setEscapeCharacter(configuration, carbonLoadModel.getEscapeChar());
+ CSVInputFormat.setMaxColumns(configuration, carbonLoadModel.getMaxColumns());
+ CSVInputFormat.setNumberOfColumns(configuration,
+ "" + carbonLoadModel.getCsvHeaderColumns().length);
+
+ CSVInputFormat.setHeaderExtractionEnabled(
+ configuration,
+ carbonLoadModel.getCsvHeader() == null ||
+ StringUtils.isEmpty(carbonLoadModel.getCsvHeader()));
+
+ CSVInputFormat.setQuoteCharacter(configuration, carbonLoadModel.getQuoteChar());
+
+ CSVInputFormat.setReadBufferSize(
+ configuration,
+ CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.CSV_READ_BUFFER_SIZE,
+ CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT));
+ }
+
+ public static void clearUnsafeMemory(long taskId) {
+ UnsafeMemoryManager.INSTANCE.freeMemoryAll(taskId);
+ UnsafeSortMemoryManager.INSTANCE.freeMemoryAll(taskId);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sql/src/main/java/org/apache/carbondata/horizon/rest/controller/SqlHorizonController.java
----------------------------------------------------------------------
diff --git a/store/sql/src/main/java/org/apache/carbondata/horizon/rest/controller/SqlHorizonController.java b/store/sql/src/main/java/org/apache/carbondata/horizon/rest/controller/SqlHorizonController.java
index 4c3b996..1910e04 100644
--- a/store/sql/src/main/java/org/apache/carbondata/horizon/rest/controller/SqlHorizonController.java
+++ b/store/sql/src/main/java/org/apache/carbondata/horizon/rest/controller/SqlHorizonController.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.horizon.rest.model.validate.RequestValidator;
import org.apache.carbondata.horizon.rest.model.view.SqlRequest;
import org.apache.carbondata.horizon.rest.model.view.SqlResponse;
import org.apache.carbondata.horizon.rest.sql.SparkSqlWrapper;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.store.api.exception.CarbonException;
import org.apache.spark.sql.AnalysisException;
import org.apache.spark.sql.Dataset;
@@ -46,7 +46,7 @@ public class SqlHorizonController {
LogServiceFactory.getLogService(SqlHorizonController.class.getName());
@RequestMapping(value = "/table/sql", produces = MediaType.APPLICATION_JSON_VALUE)
- public ResponseEntity<SqlResponse> sql(@RequestBody SqlRequest request) throws StoreException {
+ public ResponseEntity<SqlResponse> sql(@RequestBody SqlRequest request) throws CarbonException {
RequestValidator.validateSql(request);
List<Row> rows;
Dataset<Row> sqlDataFrame = null;
@@ -56,10 +56,10 @@ public class SqlHorizonController {
rows = sqlDataFrame.collectAsList();
} catch (AnalysisException e) {
LOGGER.error(e);
- throw new StoreException(e.getSimpleMessage());
+ throw new CarbonException(e.getSimpleMessage());
} catch (Exception e) {
LOGGER.error(e);
- throw new StoreException(e.getMessage());
+ throw new CarbonException(e.getMessage());
}
final String[] fieldNames = sqlDataFrame.schema().fieldNames();
Object[][] responseData = new Object[0][];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sql/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
----------------------------------------------------------------------
diff --git a/store/sql/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java b/store/sql/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
index 82e095a..dc82d6d 100644
--- a/store/sql/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
+++ b/store/sql/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
@@ -18,18 +18,18 @@
package org.apache.carbondata.horizon.rest.model.validate;
import org.apache.carbondata.horizon.rest.model.view.SqlRequest;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
import org.apache.commons.lang.StringUtils;
public class RequestValidator {
- public static void validateSql(SqlRequest request) throws StoreException {
+ public static void validateSql(SqlRequest request) throws CarbonException {
if (request == null) {
- throw new StoreException("Select should not be null");
+ throw new CarbonException("Select should not be null");
}
if (StringUtils.isEmpty(request.getSqlStatement())) {
- throw new StoreException("sql statement is invalid");
+ throw new CarbonException("sql statement is invalid");
}
}
}
[3/5] carbondata git commit: [CARBONDATA-2825][CARBONDATA-2828]
CarbonStore and InternalCarbonStore API This closes #2589
Posted by qi...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Master.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Master.java b/store/core/src/main/java/org/apache/carbondata/store/impl/master/Master.java
index 59d4aa3..dbb0116 100644
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Master.java
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/master/Master.java
@@ -19,57 +19,62 @@ package org.apache.carbondata.store.impl.master;
import java.io.IOException;
import java.net.BindException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.impl.rpc.RegistryService;
-import org.apache.carbondata.store.impl.rpc.ServiceFactory;
-import org.apache.carbondata.store.impl.rpc.StoreService;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerRequest;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerResponse;
-import org.apache.carbondata.store.util.StoreUtil;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.service.StoreService;
+import org.apache.carbondata.sdk.store.util.StoreUtil;
+import org.apache.carbondata.store.impl.Schedulable;
+import org.apache.carbondata.store.impl.service.PruneService;
+import org.apache.carbondata.store.impl.service.RegistryService;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerRequest;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
/**
- * Master of CarbonSearch.
- * It provides a Registry service for worker to register.
+ * Master of CarbonStore.
+ * It provides a Registry service and Prune service.
*/
-class Master {
+@InterfaceAudience.Internal
+public class Master {
private static Master instance = null;
private static LogService LOGGER = LogServiceFactory.getLogService(Master.class.getName());
- // worker host address map to EndpointRef
- private StoreConf conf;
+ private StoreConf storeConf;
private Configuration hadoopConf;
private RPC.Server registryServer = null;
+ private RPC.Server pruneServer = null;
+ private RPC.Server storeServer = null;
// mapping of worker IP address to worker instance
Map<String, Schedulable> workers = new ConcurrentHashMap<>();
- private Master(StoreConf conf) {
- this.conf = conf;
- this.hadoopConf = conf.newHadoopConf();
+ public Master(StoreConf storeConf) {
+ this.storeConf = storeConf;
+ this.hadoopConf = storeConf.newHadoopConf();
}
/**
- * start service and listen on port passed in constructor
+ * start registry service
*/
- public void startService() throws IOException {
+ private void startRegistryService() throws IOException {
if (registryServer == null) {
-
BindException exception;
// we will try to create service at worse case 100 times
int numTry = 100;
- String host = conf.masterHost();
- int port = conf.masterPort();
+ String host = storeConf.masterHost();
+ int port = storeConf.registryServicePort();
LOGGER.info("building registry-service on " + host + ":" + port);
RegistryService registryService = new RegistryServiceImpl(this);
@@ -99,11 +104,11 @@ class Master {
}
LOGGER.info("registry-service started");
} else {
- LOGGER.info("Search mode master has already started");
+ LOGGER.info("registry-service has already started");
}
}
- public void stopService() throws InterruptedException {
+ private void stopRegistryService() throws InterruptedException {
if (registryServer != null) {
registryServer.stop();
registryServer.join();
@@ -112,9 +117,110 @@ class Master {
}
/**
+ * start prune service
+ */
+ private void startPruneService() throws IOException {
+ if (pruneServer == null) {
+ BindException exception;
+ // we will try to create service at worse case 100 times
+ int numTry = 100;
+ String host = storeConf.masterHost();
+ int port = storeConf.pruneServicePort();
+ LOGGER.info("building prune-service on " + host + ":" + port);
+
+ PruneService pruneService = new PruneServiceImpl(storeConf, new Scheduler(this));
+ do {
+ try {
+ pruneServer = new RPC.Builder(hadoopConf)
+ .setBindAddress(host)
+ .setPort(port)
+ .setProtocol(PruneService.class)
+ .setInstance(pruneService)
+ .build();
+ pruneServer.start();
+ numTry = 0;
+ exception = null;
+ } catch (BindException e) {
+ // port is occupied, increase the port number and try again
+ exception = e;
+ LOGGER.error(e, "start prune-service failed");
+ port = port + 1;
+ numTry = numTry - 1;
+ }
+ } while (numTry > 0);
+ if (exception != null) {
+ // we have tried many times, but still failed to find an available port
+ throw exception;
+ }
+ LOGGER.info("prune-service started");
+ } else {
+ LOGGER.info("prune-service has already started");
+ }
+ }
+
+ private void stopPruneService() throws InterruptedException {
+ if (pruneServer != null) {
+ pruneServer.stop();
+ pruneServer.join();
+ pruneServer = null;
+ }
+ }
+
+ /**
+ * start store service
+ */
+ private void startStoreService() throws IOException {
+ if (storeServer == null) {
+ BindException exception;
+ // we will try to create service at worse case 100 times
+ int numTry = 100;
+ String host = storeConf.masterHost();
+ int port = storeConf.storeServicePort();
+ LOGGER.info("building store-service on " + host + ":" + port);
+
+ StoreService storeService = new StoreServiceImpl(storeConf);
+ do {
+ try {
+ storeServer = new RPC.Builder(hadoopConf)
+ .setBindAddress(host)
+ .setPort(port)
+ .setProtocol(StoreService.class)
+ .setInstance(storeService)
+ .build();
+
+ storeServer.start();
+ numTry = 0;
+ exception = null;
+ } catch (BindException e) {
+ // port is occupied, increase the port number and try again
+ exception = e;
+ LOGGER.error(e, "start store-service failed");
+ port = port + 1;
+ numTry = numTry - 1;
+ }
+ } while (numTry > 0);
+ if (exception != null) {
+ // we have tried many times, but still failed to find an available port
+ throw exception;
+ }
+ LOGGER.info("store-service started");
+ } else {
+ LOGGER.info("store-service has already started");
+ }
+ }
+
+ private void stopStoreService() throws InterruptedException {
+ if (storeServer != null) {
+ storeServer.stop();
+ storeServer.join();
+ storeServer = null;
+ }
+ }
+
+ /**
* A new searcher is trying to register, add it to the map and connect to this searcher
*/
- public RegisterWorkerResponse addWorker(RegisterWorkerRequest request) throws IOException {
+ RegisterWorkerResponse addWorker(RegisterWorkerRequest request) throws IOException {
LOGGER.info(
"Receive Register request from worker " + request.getHostAddress() + ":" + request.getPort()
+ " with " + request.getCores() + " cores");
@@ -125,9 +231,7 @@ class Master {
"connecting to worker " + request.getHostAddress() + ":" + request.getPort() + ", workerId "
+ workerId);
- StoreService searchService = ServiceFactory.createStoreService(workerAddress, workerPort);
- addWorker(
- new Schedulable(workerId, workerAddress, workerPort, request.getCores(), searchService));
+ addWorker(new Schedulable(workerId, workerAddress, workerPort, request.getCores()));
LOGGER.info("worker " + request + " registered");
return new RegisterWorkerResponse(workerId);
}
@@ -146,7 +250,16 @@ class Master {
return instance;
}
- public static void main(String[] args) throws InterruptedException {
+ public void stopService() throws InterruptedException {
+ stopRegistryService();
+ stopPruneService();
+ }
+
+ public List<String> getAllWorkerAddresses() {
+ return new ArrayList<>(workers.keySet());
+ }
+
+ public static void main(String[] args) throws InterruptedException, IOException {
if (args.length != 2) {
System.err.println("Usage: Master <log4j file> <properties file>");
return;
@@ -155,7 +268,10 @@ class Master {
StoreUtil.initLog4j(args[0]);
StoreConf conf = new StoreConf(args[1]);
Master master = getInstance(conf);
- master.stopService();
+ master.startStoreService();
+ master.startRegistryService();
+ master.startPruneService();
+ Thread.sleep(Long.MAX_VALUE);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/master/PruneServiceImpl.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/master/PruneServiceImpl.java b/store/core/src/main/java/org/apache/carbondata/store/impl/master/PruneServiceImpl.java
new file mode 100644
index 0000000..726c442
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/master/PruneServiceImpl.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.master;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.impl.BlockScanUnit;
+import org.apache.carbondata.store.impl.IndexOperation;
+import org.apache.carbondata.store.impl.MetaOperation;
+import org.apache.carbondata.store.impl.Schedulable;
+import org.apache.carbondata.store.impl.service.PruneService;
+import org.apache.carbondata.store.impl.service.model.PruneRequest;
+import org.apache.carbondata.store.impl.service.model.PruneResponse;
+
+import org.apache.hadoop.ipc.ProtocolSignature;
+
+public class PruneServiceImpl implements PruneService {
+
+ private StoreConf storeConf;
+ private Scheduler scheduler;
+
+ PruneServiceImpl(StoreConf storeConf, Scheduler scheduler) {
+ this.storeConf = storeConf;
+ this.scheduler = scheduler;
+ }
+
+ @Override
+ public PruneResponse prune(PruneRequest request) throws CarbonException {
+ TableInfo tableInfo = MetaOperation.getTable(request.getTable(), storeConf);
+ try {
+ List<CarbonInputSplit> splits =
+ IndexOperation.pruneBlock(tableInfo, request.getFilterExpression());
+ List<ScanUnit> output = splits.stream()
+ .map((Function<CarbonInputSplit, ScanUnit>) inputSplit -> {
+ Schedulable worker;
+ String[] locations;
+ try {
+ locations = inputSplit.getLocations();
+ if (locations.length == 0) {
+ worker = scheduler.pickNexWorker();
+ } else {
+ worker = scheduler.pickWorker(locations[0]);
+ }
+ } catch (IOException e) {
+ // ignore it and pick next worker as no locality
+ worker = scheduler.pickNexWorker();
+ }
+ return new BlockScanUnit(inputSplit, worker);
+ }).collect(Collectors.toList());
+ return new PruneResponse(output);
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return versionID;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+ int clientMethodsHash) throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/master/RegistryServiceImpl.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/master/RegistryServiceImpl.java b/store/core/src/main/java/org/apache/carbondata/store/impl/master/RegistryServiceImpl.java
index 492006b..ce2be99 100644
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/master/RegistryServiceImpl.java
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/master/RegistryServiceImpl.java
@@ -20,14 +20,14 @@ package org.apache.carbondata.store.impl.master;
import java.io.IOException;
import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.store.impl.rpc.RegistryService;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerRequest;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerResponse;
+import org.apache.carbondata.store.impl.service.RegistryService;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerRequest;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerResponse;
import org.apache.hadoop.ipc.ProtocolSignature;
@InterfaceAudience.Internal
-class RegistryServiceImpl implements RegistryService {
+class egistryServiceImpl implements RegistryService {
private Master master;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Schedulable.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Schedulable.java b/store/core/src/main/java/org/apache/carbondata/store/impl/master/Schedulable.java
deleted file mode 100644
index 57d41f0..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Schedulable.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.master;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.store.impl.rpc.StoreService;
-
-@InterfaceAudience.Internal
-public class Schedulable {
-
- private String id;
- private String address;
- private int port;
- private int cores;
- public StoreService service;
- public AtomicInteger workload;
-
- public Schedulable(String id, String address, int port, int cores, StoreService service) {
- this.id = id;
- this.address = address;
- this.port = port;
- this.cores = cores;
- this.service = service;
- this.workload = new AtomicInteger();
- }
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public String getAddress() {
- return address;
- }
-
- public void setAddress(String address) {
- this.address = address;
- }
-
- public int getPort() {
- return port;
- }
-
- public void setPort(int port) {
- this.port = port;
- }
-
- int getCores() {
- return cores;
- }
-
- @Override public String toString() {
- return "Schedulable{" + "id='" + id + '\'' + ", address='" + address + '\'' + ", port=" + port
- + '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Scheduler.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Scheduler.java b/store/core/src/main/java/org/apache/carbondata/store/impl/master/Scheduler.java
index 96a1375..4ede670 100644
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/master/Scheduler.java
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/master/Scheduler.java
@@ -17,69 +17,33 @@
package org.apache.carbondata.store.impl.master;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.exception.SchedulerException;
-import org.apache.carbondata.store.impl.rpc.model.BaseResponse;
-import org.apache.carbondata.store.impl.rpc.model.LoadDataRequest;
-import org.apache.carbondata.store.impl.rpc.model.QueryResponse;
-import org.apache.carbondata.store.impl.rpc.model.Scan;
-import org.apache.carbondata.store.impl.rpc.model.ShutdownRequest;
+import org.apache.carbondata.sdk.store.exception.SchedulerException;
+import org.apache.carbondata.store.impl.Schedulable;
/**
* [[Master]] uses Scheduler to pick a Worker to send request
*/
@InterfaceAudience.Internal
-public class Scheduler {
+class Scheduler {
private static LogService LOGGER = LogServiceFactory.getLogService(Scheduler.class.getName());
private AtomicInteger count = new AtomicInteger(0);
- private ExecutorService executors = Executors.newCachedThreadPool();
- private Master master;
-
- public Scheduler(StoreConf storeConf) throws IOException {
- master = Master.getInstance(storeConf);
- master.startService();
- }
-
- /**
- * Pick a Worker according to the address and workload of the Worker
- * Invoke the RPC and return Future result
- */
- public Future<QueryResponse> sendRequestAsync(
- final Schedulable worker, final Scan scan) {
- LOGGER.info("sending search request to worker " + worker);
- worker.workload.incrementAndGet();
- return executors.submit(new Callable<QueryResponse>() {
- @Override public QueryResponse call() {
- return worker.service.query(scan);
- }
- });
- }
- public BaseResponse sendRequest(final Schedulable worker,
- final LoadDataRequest request) {
+ private Master master;
- LOGGER.info("sending load data request to worker " + worker);
- worker.workload.incrementAndGet();
- return worker.service.loadData(request);
+ Scheduler(Master master) {
+ this.master = master;
}
- public Schedulable pickWorker(String splitAddress) {
+ Schedulable pickWorker(String splitAddress) {
Schedulable worker = master.workers.get(splitAddress);
// no local worker available, choose one worker randomly
if (worker == null) {
@@ -107,31 +71,12 @@ public class Scheduler {
return worker;
}
- public Schedulable pickNexWorker() {
+ Schedulable pickNexWorker() {
if (master.workers.size() == 0) {
throw new SchedulerException("No worker is available");
}
int index = count.getAndIncrement() % master.workers.size();
return new ArrayList<>(master.workers.values()).get(index);
}
-
- public void stopAllWorkers() throws IOException {
- for (Map.Entry<String, Schedulable> entry : master.workers.entrySet()) {
- try {
- entry.getValue().service.shutdown(new ShutdownRequest("user"));
- } catch (Throwable throwable) {
- throw new IOException(throwable);
- }
- master.workers.remove(entry.getKey());
- }
- }
-
- public void stopService() throws InterruptedException {
- master.stopService();
- }
-
- public List<String> getAllWorkerAddresses() {
- return new ArrayList<>(master.workers.keySet());
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/master/StoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/master/StoreServiceImpl.java b/store/core/src/main/java/org/apache/carbondata/store/impl/master/StoreServiceImpl.java
new file mode 100644
index 0000000..5d0fd88
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/master/StoreServiceImpl.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.master;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.sdk.store.service.StoreService;
+import org.apache.carbondata.store.impl.LocalCarbonStore;
+import org.apache.carbondata.store.impl.MetaOperation;
+
+import org.apache.hadoop.ipc.ProtocolSignature;
+
+public class StoreServiceImpl implements StoreService {
+
+ // TODO: simple implementation, load and scan inside master
+ private LocalCarbonStore localStore;
+
+ private MetaOperation metaOperation;
+
+ StoreServiceImpl(StoreConf storeConf) {
+ localStore = new LocalCarbonStore(storeConf);
+ metaOperation = new MetaOperation(storeConf);
+ }
+
+ @Override
+ public void createTable(TableDescriptor descriptor) throws CarbonException {
+ metaOperation.createTable(descriptor);
+ }
+
+ @Override
+ public void dropTable(TableIdentifier identifier) throws CarbonException {
+ metaOperation.dropTable(identifier);
+ }
+
+ @Override
+ public TableInfo getTable(TableIdentifier identifier) throws CarbonException {
+ return metaOperation.getTable(identifier);
+ }
+
+ @Override
+ public List<TableDescriptor> listTable() throws CarbonException {
+ return metaOperation.listTable();
+ }
+
+ @Override
+ public TableDescriptor getDescriptor(TableIdentifier identifier) throws CarbonException {
+ return metaOperation.getDescriptor(identifier);
+ }
+
+ @Override
+ public void alterTable(TableIdentifier identifier, TableDescriptor newTable)
+ throws CarbonException {
+ metaOperation.alterTable(identifier, newTable);
+ }
+
+ @Override
+ public void loadData(LoadDescriptor loadDescriptor) throws CarbonException {
+ localStore.loadData(loadDescriptor);
+ }
+
+ @Override
+ public List<CarbonRow> scan(ScanDescriptor scanDescriptor) throws CarbonException {
+ return localStore.scan(scanDescriptor);
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return versionID;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+ int clientMethodsHash) throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/RegistryService.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/RegistryService.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/RegistryService.java
deleted file mode 100644
index a40f741..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/RegistryService.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerRequest;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerResponse;
-
-import org.apache.hadoop.ipc.VersionedProtocol;
-
-@InterfaceAudience.Internal
-public interface RegistryService extends VersionedProtocol {
- long versionID = 1L;
- RegisterWorkerResponse registerWorker(RegisterWorkerRequest request) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/ServiceFactory.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/ServiceFactory.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/ServiceFactory.java
deleted file mode 100644
index 852f14f..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/ServiceFactory.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC;
-
-@InterfaceAudience.Internal
-public class ServiceFactory {
-
- public static StoreService createStoreService(String host, int port) throws IOException {
- InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(host), port);
- return RPC.getProxy(
- StoreService.class, StoreService.versionID, address, new Configuration());
- }
-
- public static RegistryService createRegistryService(String host, int port) throws IOException {
- InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(host), port);
- return RPC.getProxy(
- RegistryService.class, RegistryService.versionID, address, new Configuration());
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/StoreService.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/StoreService.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/StoreService.java
deleted file mode 100644
index 36702d9..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/StoreService.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.store.impl.rpc.model.BaseResponse;
-import org.apache.carbondata.store.impl.rpc.model.LoadDataRequest;
-import org.apache.carbondata.store.impl.rpc.model.QueryResponse;
-import org.apache.carbondata.store.impl.rpc.model.Scan;
-import org.apache.carbondata.store.impl.rpc.model.ShutdownRequest;
-import org.apache.carbondata.store.impl.rpc.model.ShutdownResponse;
-
-import org.apache.hadoop.ipc.VersionedProtocol;
-
-@InterfaceAudience.Internal
-public interface StoreService extends VersionedProtocol {
-
- long versionID = 1L;
-
- BaseResponse loadData(LoadDataRequest request);
-
- QueryResponse query(Scan scan);
-
- ShutdownResponse shutdown(ShutdownRequest request);
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/BaseResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/BaseResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/BaseResponse.java
deleted file mode 100644
index b11aa69..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/BaseResponse.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-
-import org.apache.hadoop.io.Writable;
-
-@InterfaceAudience.Internal
-public class BaseResponse implements Serializable, Writable {
- private int status;
- private String message;
-
- public BaseResponse() {
- }
-
- public BaseResponse(int status, String message) {
- this.status = status;
- this.message = message;
- }
-
- public int getStatus() {
- return status;
- }
-
- public void setStatus(int status) {
- this.status = status;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- this.message = message;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(status);
- out.writeUTF(message);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- status = in.readInt();
- message = in.readUTF();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/LoadDataRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/LoadDataRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/LoadDataRequest.java
deleted file mode 100644
index e6c8048..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/LoadDataRequest.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.store.util.StoreUtil;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-public class LoadDataRequest implements Serializable, Writable {
-
- private CarbonLoadModel model;
-
- public LoadDataRequest() {
- }
-
- public LoadDataRequest(CarbonLoadModel model) {
- this.model = model;
- }
-
- public CarbonLoadModel getModel() {
- return model;
- }
-
- public void setModel(CarbonLoadModel model) {
- this.model = model;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- WritableUtils.writeCompressedByteArray(out, StoreUtil.serialize(model));
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- byte[] bytes = WritableUtils.readCompressedByteArray(in);
- model = (CarbonLoadModel) StoreUtil.deserialize(bytes);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/QueryResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/QueryResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/QueryResponse.java
deleted file mode 100644
index 304fd0f..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/QueryResponse.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.core.util.ObjectSerializationUtil;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-@InterfaceAudience.Internal
-public class QueryResponse extends BaseResponse implements Serializable, Writable {
- private int queryId;
- private Object[][] rows;
-
- public QueryResponse() {
- super();
- }
-
- public QueryResponse(int queryId, int status, String message, Object[][] rows) {
- super(status, message);
- this.queryId = queryId;
- this.rows = rows;
- }
-
- public int getQueryId() {
- return queryId;
- }
-
-
- public Object[][] getRows() {
- return rows;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- super.write(out);
- out.writeInt(queryId);
- WritableUtils.writeCompressedByteArray(out, ObjectSerializationUtil.serialize(rows));
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- queryId = in.readInt();
- try {
- rows = (Object[][])ObjectSerializationUtil.deserialize(
- WritableUtils.readCompressedByteArray(in));
- } catch (ClassNotFoundException e) {
- throw new IOException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerRequest.java
deleted file mode 100644
index 5f223d6..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerRequest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-
-import org.apache.hadoop.io.Writable;
-
-@InterfaceAudience.Internal
-public class RegisterWorkerRequest implements Serializable, Writable {
- private String hostAddress;
- private int port;
- private int cores;
-
- public RegisterWorkerRequest() {
- }
-
- public RegisterWorkerRequest(String hostAddress, int port, int cores) {
- this.hostAddress = hostAddress;
- this.port = port;
- this.cores = cores;
- }
-
- public String getHostAddress() {
- return hostAddress;
- }
-
- public int getPort() {
- return port;
- }
-
- public int getCores() {
- return cores;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(hostAddress);
- out.writeInt(port);
- out.writeInt(cores);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- hostAddress = in.readUTF();
- port = in.readInt();
- cores = in.readInt();
- }
-
- @Override public String toString() {
- return "RegisterWorkerRequest{" + "hostAddress='" + hostAddress + '\'' + ", port=" + port + '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerResponse.java
deleted file mode 100644
index 16915e9..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/RegisterWorkerResponse.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-
-import org.apache.hadoop.io.Writable;
-
-@InterfaceAudience.Internal
-public class RegisterWorkerResponse implements Serializable, Writable {
-
- private String workerId;
-
- public RegisterWorkerResponse() {
- }
-
- public RegisterWorkerResponse(String workerId) {
- this.workerId = workerId;
- }
-
- public String getWorkerId() {
- return workerId;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(workerId);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- workerId = in.readUTF();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/Scan.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/Scan.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/Scan.java
deleted file mode 100644
index 86767e5..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/Scan.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.util.ObjectSerializationUtil;
-import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
-
-import org.apache.hadoop.io.Writable;
-
-@InterfaceAudience.Internal
-public class Scan implements Serializable, Writable {
- private int requestId;
- private CarbonMultiBlockSplit split;
- private TableInfo tableInfo;
- private String[] projectColumns;
- private Expression filterExpression;
- private long limit;
-
- public Scan() {
- }
-
- public Scan(int requestId, CarbonMultiBlockSplit split,
- TableInfo tableInfo, String[] projectColumns, Expression filterExpression, long limit) {
- this.requestId = requestId;
- this.split = split;
- this.tableInfo = tableInfo;
- this.projectColumns = projectColumns;
- this.filterExpression = filterExpression;
- this.limit = limit;
- }
-
- public int getRequestId() {
- return requestId;
- }
-
- public CarbonMultiBlockSplit getSplit() {
- return split;
- }
-
- public TableInfo getTableInfo() {
- return tableInfo;
- }
-
- public String[] getProjectColumns() {
- return projectColumns;
- }
-
- public Expression getFilterExpression() {
- return filterExpression;
- }
-
- public long getLimit() {
- return limit;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(requestId);
- split.write(out);
- tableInfo.write(out);
- out.writeInt(projectColumns.length);
- for (String projectColumn : projectColumns) {
- out.writeUTF(projectColumn);
- }
- String filter = ObjectSerializationUtil.convertObjectToString(filterExpression);
- out.writeUTF(filter);
- out.writeLong(limit);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- requestId = in.readInt();
- split = new CarbonMultiBlockSplit();
- split.readFields(in);
- tableInfo = new TableInfo();
- tableInfo.readFields(in);
- projectColumns = new String[in.readInt()];
- for (int i = 0; i < projectColumns.length; i++) {
- projectColumns[i] = in.readUTF();
- }
- String filter = in.readUTF();
- filterExpression = (Expression) ObjectSerializationUtil.convertStringToObject(filter);
- limit = in.readLong();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownRequest.java
deleted file mode 100644
index 311963d..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownRequest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-
-import org.apache.hadoop.io.Writable;
-
-@InterfaceAudience.Internal
-public class ShutdownRequest implements Serializable, Writable {
- private String reason;
-
- public ShutdownRequest() {
- }
-
- public ShutdownRequest(String reason) {
- this.reason = reason;
- }
-
- public String getReason() {
- return reason;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeUTF(reason);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- reason = in.readUTF();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownResponse.java
deleted file mode 100644
index 0143f48..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/rpc/model/ShutdownResponse.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.rpc.model;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-
-import org.apache.hadoop.io.Writable;
-
-@InterfaceAudience.Internal
-public class ShutdownResponse implements Serializable, Writable {
- private int status;
- private String message;
-
- public ShutdownResponse() {
- }
-
- public ShutdownResponse(int status, String message) {
- this.status = status;
- this.message = message;
- }
-
- public int getStatus() {
- return status;
- }
-
- public String getMessage() {
- return message;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeInt(status);
- out.writeUTF(message);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- status = in.readInt();
- message = in.readUTF();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/DataService.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/DataService.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/DataService.java
new file mode 100644
index 0000000..670540b
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/DataService.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service;
+
+import java.io.Closeable;
+import java.util.Iterator;
+
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.ResultBatch;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.impl.service.model.BaseResponse;
+import org.apache.carbondata.store.impl.service.model.LoadDataRequest;
+import org.apache.carbondata.store.impl.service.model.ScanRequest;
+import org.apache.carbondata.store.impl.service.model.ScanResponse;
+
+import org.apache.hadoop.ipc.VersionedProtocol;
+
+public interface DataService extends VersionedProtocol, Closeable {
+
+ long versionID = 1L;
+
+ /**
+ * Load data into a Table
+ * @param load descriptor for load operation
+ * @throws CarbonException if any error occurs
+ */
+ BaseResponse loadData(LoadDataRequest load) throws CarbonException;
+
+ /**
+ * Scan a Table and return matched rows
+ * @param scan descriptor for scan operation, including required column, filter, etc
+ * @return matched rows
+ * @throws CarbonException if any error occurs
+ */
+ ScanResponse scan(ScanRequest scan) throws CarbonException;
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/PruneService.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/PruneService.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/PruneService.java
new file mode 100644
index 0000000..d69db96
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/PruneService.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.impl.service.model.PruneRequest;
+import org.apache.carbondata.store.impl.service.model.PruneResponse;
+
+import org.apache.hadoop.ipc.VersionedProtocol;
+
+@InterfaceAudience.Internal
+public interface PruneService extends VersionedProtocol {
+ long versionID = 1L;
+
+ /**
+ * Leveraging index and segment information to skip blocks,
+ * return a list of block eligible for scanning
+ *
+ * @param request prune request containing table identifier
+ * and filter expression
+ * @return prune result containing blocks to scan
+ * @throws CarbonException if any error occurs
+ */
+ PruneResponse prune(PruneRequest request) throws CarbonException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/RegistryService.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/RegistryService.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/RegistryService.java
new file mode 100644
index 0000000..f635050
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/RegistryService.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerRequest;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerResponse;
+
+import org.apache.hadoop.ipc.VersionedProtocol;
+
+@InterfaceAudience.Internal
+public interface RegistryService extends VersionedProtocol {
+ long versionID = 1L;
+ RegisterWorkerResponse registerWorker(RegisterWorkerRequest request) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/ServiceFactory.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/ServiceFactory.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/ServiceFactory.java
new file mode 100644
index 0000000..41daf0f
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/ServiceFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+
+@InterfaceAudience.Internal
+public class ServiceFactory {
+
+ public static RegistryService createRegistryService(String host, int port) throws IOException {
+ InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(host), port);
+ return RPC.getProxy(
+ RegistryService.class, RegistryService.versionID, address, new Configuration());
+ }
+
+ public static PruneService createPruneService(String host, int port) throws IOException {
+ InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(host), port);
+ return RPC.getProxy(
+ PruneService.class, PruneService.versionID, address, new Configuration());
+ }
+
+ public static DataService createDataService(String host, int port) throws IOException {
+ InetSocketAddress address = new InetSocketAddress(InetAddress.getByName(host), port);
+ return RPC.getProxy(
+ DataService.class, DataService.versionID, address, new Configuration());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/BaseResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/BaseResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/BaseResponse.java
new file mode 100644
index 0000000..49478e5
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/BaseResponse.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class BaseResponse implements Serializable, Writable {
+ private int status;
+ private String message;
+
+ public BaseResponse() {
+ }
+
+ public BaseResponse(int status, String message) {
+ this.status = status;
+ this.message = message;
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public void setStatus(int status) {
+ this.status = status;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(status);
+ out.writeUTF(message);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ status = in.readInt();
+ message = in.readUTF();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/LoadDataRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/LoadDataRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/LoadDataRequest.java
new file mode 100644
index 0000000..aa15fa9
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/LoadDataRequest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.carbondata.sdk.store.util.StoreUtil;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+public class LoadDataRequest implements Serializable, Writable {
+
+ private CarbonLoadModel model;
+
+ public LoadDataRequest() {
+ }
+
+ public LoadDataRequest(CarbonLoadModel model) {
+ this.model = model;
+ }
+
+ public CarbonLoadModel getModel() {
+ return model;
+ }
+
+ public void setModel(CarbonLoadModel model) {
+ this.model = model;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ WritableUtils.writeCompressedByteArray(out, StoreUtil.serialize(model));
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ byte[] bytes = WritableUtils.readCompressedByteArray(in);
+ model = (CarbonLoadModel) StoreUtil.deserialize(bytes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneRequest.java
new file mode 100644
index 0000000..ace35f3
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneRequest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.util.ObjectSerializationUtil;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+
+import org.apache.hadoop.io.Writable;
+
+public class PruneRequest implements Serializable, Writable {
+
+ private TableIdentifier table;
+ private Expression filterExpression;
+
+ public PruneRequest() {
+ }
+
+ public PruneRequest(TableIdentifier table, Expression filterExpression) {
+ this.table = table;
+ this.filterExpression = filterExpression;
+ }
+
+ public TableIdentifier getTable() {
+ return table;
+ }
+
+ public Expression getFilterExpression() {
+ return filterExpression;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ table.write(out);
+ out.writeUTF(ObjectSerializationUtil.convertObjectToString(filterExpression));
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ table = new TableIdentifier();
+ table.readFields(in);
+ filterExpression = (Expression) (ObjectSerializationUtil.convertStringToObject(in.readUTF()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneResponse.java
new file mode 100644
index 0000000..111528b
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/PruneResponse.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.impl.BlockScanUnit;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class PruneResponse implements Serializable, Writable {
+
+ private List<ScanUnit> scanUnits;
+
+ public PruneResponse() {
+ }
+
+ public PruneResponse(List<ScanUnit> scanUnits) {
+ this.scanUnits = scanUnits;
+ }
+
+ public List<ScanUnit> getScanUnits() {
+ return scanUnits;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(scanUnits.size());
+ for (ScanUnit scanUnit : scanUnits) {
+ scanUnit.write(out);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int size = in.readInt();
+ scanUnits = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ BlockScanUnit scanUnit = new BlockScanUnit();
+ scanUnit.readFields(in);
+ scanUnits.add(scanUnit);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerRequest.java
new file mode 100644
index 0000000..54e7271
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerRequest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class RegisterWorkerRequest implements Serializable, Writable {
+ private String hostAddress;
+ private int port;
+ private int cores;
+
+ public RegisterWorkerRequest() {
+ }
+
+ public RegisterWorkerRequest(String hostAddress, int port, int cores) {
+ this.hostAddress = hostAddress;
+ this.port = port;
+ this.cores = cores;
+ }
+
+ public String getHostAddress() {
+ return hostAddress;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public int getCores() {
+ return cores;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(hostAddress);
+ out.writeInt(port);
+ out.writeInt(cores);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ hostAddress = in.readUTF();
+ port = in.readInt();
+ cores = in.readInt();
+ }
+
+ @Override public String toString() {
+ return "RegisterWorkerRequest{" + "hostAddress='" + hostAddress + '\'' + ", port=" + port + '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerResponse.java
new file mode 100644
index 0000000..5bbfa67
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/RegisterWorkerResponse.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class RegisterWorkerResponse implements Serializable, Writable {
+
+ private String workerId;
+
+ public RegisterWorkerResponse() {
+ }
+
+ public RegisterWorkerResponse(String workerId) {
+ this.workerId = workerId;
+ }
+
+ public String getWorkerId() {
+ return workerId;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(workerId);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ workerId = in.readUTF();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanRequest.java
new file mode 100644
index 0000000..d7b60ba
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanRequest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.util.ObjectSerializationUtil;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class ScanRequest implements Serializable, Writable {
+ private int requestId;
+ private CarbonMultiBlockSplit split;
+ private TableInfo tableInfo;
+ private String[] projectColumns;
+ private Expression filterExpression;
+ private long limit;
+
+ public ScanRequest() {
+ }
+
+ public ScanRequest(int requestId, CarbonMultiBlockSplit split,
+ TableInfo tableInfo, String[] projectColumns, Expression filterExpression, long limit) {
+ this.requestId = requestId;
+ this.split = split;
+ this.tableInfo = tableInfo;
+ this.projectColumns = projectColumns;
+ this.filterExpression = filterExpression;
+ this.limit = limit;
+ }
+
+ public int getRequestId() {
+ return requestId;
+ }
+
+ public CarbonMultiBlockSplit getSplit() {
+ return split;
+ }
+
+ public TableInfo getTableInfo() {
+ return tableInfo;
+ }
+
+ public String[] getProjectColumns() {
+ return projectColumns;
+ }
+
+ public Expression getFilterExpression() {
+ return filterExpression;
+ }
+
+ public long getLimit() {
+ return limit;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(requestId);
+ split.write(out);
+ tableInfo.write(out);
+ out.writeInt(projectColumns.length);
+ for (String projectColumn : projectColumns) {
+ out.writeUTF(projectColumn);
+ }
+ String filter = ObjectSerializationUtil.convertObjectToString(filterExpression);
+ out.writeUTF(filter);
+ out.writeLong(limit);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ requestId = in.readInt();
+ split = new CarbonMultiBlockSplit();
+ split.readFields(in);
+ tableInfo = new TableInfo();
+ tableInfo.readFields(in);
+ projectColumns = new String[in.readInt()];
+ for (int i = 0; i < projectColumns.length; i++) {
+ projectColumns[i] = in.readUTF();
+ }
+ String filter = in.readUTF();
+ filterExpression = (Expression) ObjectSerializationUtil.convertStringToObject(filter);
+ limit = in.readLong();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanResponse.java
new file mode 100644
index 0000000..c5a14c2
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ScanResponse.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.util.ObjectSerializationUtil;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+@InterfaceAudience.Internal
+public class ScanResponse extends BaseResponse implements Serializable, Writable {
+ private int queryId;
+ private Object[][] rows;
+
+ public ScanResponse() {
+ super();
+ }
+
+ public ScanResponse(int queryId, int status, String message, Object[][] rows) {
+ super(status, message);
+ this.queryId = queryId;
+ this.rows = rows;
+ }
+
+ public int getQueryId() {
+ return queryId;
+ }
+
+
+ public Object[][] getRows() {
+ return rows;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ out.writeInt(queryId);
+ WritableUtils.writeCompressedByteArray(out, ObjectSerializationUtil.serialize(rows));
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ queryId = in.readInt();
+ try {
+ rows = (Object[][])ObjectSerializationUtil.deserialize(
+ WritableUtils.readCompressedByteArray(in));
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownRequest.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownRequest.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownRequest.java
new file mode 100644
index 0000000..d029d0a
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownRequest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class ShutdownRequest implements Serializable, Writable {
+ private String reason;
+
+ public ShutdownRequest() {
+ }
+
+ public ShutdownRequest(String reason) {
+ this.reason = reason;
+ }
+
+ public String getReason() {
+ return reason;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(reason);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ reason = in.readUTF();
+ }
+}
[5/5] carbondata git commit: [CARBONDATA-2825][CARBONDATA-2828]
CarbonStore and InternalCarbonStore API This closes #2589
Posted by qi...@apache.org.
[CARBONDATA-2825][CARBONDATA-2828] CarbonStore and InternalCarbonStore API
This closes #2589
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9f10122a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9f10122a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9f10122a
Branch: refs/heads/carbonstore
Commit: 9f10122af65fcd518bfe8855c4eaa1a423e81caa
Parents: a6027ae
Author: Jacky Li <ja...@qq.com>
Authored: Wed Aug 1 02:16:26 2018 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Mon Aug 6 19:03:12 2018 +0800
----------------------------------------------------------------------
.../common/annotations/InterfaceStability.java | 2 +-
.../core/datastore/row/CarbonRow.java | 30 ++-
.../core/metadata/schema/table/TableInfo.java | 2 +-
.../carbondata/hadoop/CarbonInputSplit.java | 44 +++-
.../hadoop/api/CarbonInputFormat.java | 28 ++-
.../hadoop/api/CarbonTableInputFormat.java | 11 +
.../hadoop/readsupport/CarbonReadSupport.java | 3 +-
.../apache/carbondata/store/WorkerManager.scala | 4 +-
.../org/apache/spark/sql/CarbonSession.scala | 22 +-
.../carbondata/zeppelin/TestCarbonResponse.java | 4 +-
store/conf/store.conf | 4 +-
store/core/pom.xml | 4 +-
.../carbondata/store/api/CarbonStore.java | 32 ---
.../store/api/CarbonStoreFactory.java | 93 --------
.../apache/carbondata/store/api/DataStore.java | 51 ----
.../apache/carbondata/store/api/MetaStore.java | 50 ----
.../apache/carbondata/store/api/SqlStore.java | 34 ---
.../carbondata/store/api/conf/StoreConf.java | 197 ----------------
.../store/api/descriptor/LoadDescriptor.java | 114 ---------
.../store/api/descriptor/SelectDescriptor.java | 111 ---------
.../store/api/descriptor/TableDescriptor.java | 174 --------------
.../store/api/descriptor/TableIdentifier.java | 37 ---
.../exception/ExecutionTimeoutException.java | 22 --
.../store/api/exception/SchedulerException.java | 26 ---
.../store/api/exception/StoreException.java | 33 ---
.../carbondata/store/devapi/DataLoader.java | 49 ++++
.../carbondata/store/devapi/DataScanner.java | 51 ++++
.../store/devapi/InternalCarbonStore.java | 72 ++++++
.../devapi/InternalCarbonStoreFactory.java | 45 ++++
.../apache/carbondata/store/devapi/Pruner.java | 46 ++++
.../carbondata/store/devapi/ResultBatch.java | 47 ++++
.../carbondata/store/devapi/ScanOption.java | 69 ++++++
.../carbondata/store/devapi/ScanUnit.java | 41 ++++
.../apache/carbondata/store/devapi/Scanner.java | 33 +++
.../store/devapi/TransactionalOperation.java | 35 +++
.../carbondata/store/impl/BlockScanUnit.java | 74 ++++++
.../carbondata/store/impl/CarbonStoreBase.java | 177 --------------
.../carbondata/store/impl/DataOperation.java | 95 ++++++++
.../carbondata/store/impl/DataServicePool.java | 41 ++++
.../carbondata/store/impl/DelegatedScanner.java | 57 +++++
.../store/impl/DistributedCarbonStore.java | 232 -------------------
.../carbondata/store/impl/IndexOperation.java | 61 +++++
.../store/impl/IndexedRecordReader.java | 2 +-
.../store/impl/InternalCarbonStoreImpl.java | 110 +++++++++
.../carbondata/store/impl/LocalCarbonStore.java | 95 +++++---
.../carbondata/store/impl/LocalDataScanner.java | 73 ++++++
.../carbondata/store/impl/LocalPruner.java | 58 +++++
.../carbondata/store/impl/MetaOperation.java | 212 +++++++++++++++++
.../carbondata/store/impl/MetaProcessor.java | 170 --------------
.../store/impl/RemoteDataScanner.java | 87 +++++++
.../carbondata/store/impl/RemotePruner.java | 56 +++++
.../store/impl/RowMajorResultBatch.java | 49 ++++
.../carbondata/store/impl/Schedulable.java | 100 ++++++++
.../carbondata/store/impl/master/Master.java | 172 +++++++++++---
.../store/impl/master/PruneServiceImpl.java | 89 +++++++
.../store/impl/master/RegistryServiceImpl.java | 8 +-
.../store/impl/master/Schedulable.java | 76 ------
.../carbondata/store/impl/master/Scheduler.java | 71 +-----
.../store/impl/master/StoreServiceImpl.java | 100 ++++++++
.../store/impl/rpc/RegistryService.java | 32 ---
.../store/impl/rpc/ServiceFactory.java | 43 ----
.../carbondata/store/impl/rpc/StoreService.java | 40 ----
.../store/impl/rpc/model/BaseResponse.java | 69 ------
.../store/impl/rpc/model/LoadDataRequest.java | 60 -----
.../store/impl/rpc/model/QueryResponse.java | 73 ------
.../impl/rpc/model/RegisterWorkerRequest.java | 73 ------
.../impl/rpc/model/RegisterWorkerResponse.java | 54 -----
.../carbondata/store/impl/rpc/model/Scan.java | 108 ---------
.../store/impl/rpc/model/ShutdownRequest.java | 53 -----
.../store/impl/rpc/model/ShutdownResponse.java | 61 -----
.../store/impl/service/DataService.java | 53 +++++
.../store/impl/service/PruneService.java | 41 ++++
.../store/impl/service/RegistryService.java | 32 +++
.../store/impl/service/ServiceFactory.java | 50 ++++
.../store/impl/service/model/BaseResponse.java | 69 ++++++
.../impl/service/model/LoadDataRequest.java | 60 +++++
.../store/impl/service/model/PruneRequest.java | 64 +++++
.../store/impl/service/model/PruneResponse.java | 67 ++++++
.../service/model/RegisterWorkerRequest.java | 73 ++++++
.../service/model/RegisterWorkerResponse.java | 54 +++++
.../store/impl/service/model/ScanRequest.java | 108 +++++++++
.../store/impl/service/model/ScanResponse.java | 73 ++++++
.../impl/service/model/ShutdownRequest.java | 53 +++++
.../impl/service/model/ShutdownResponse.java | 61 +++++
.../store/impl/worker/DataServiceImpl.java | 174 ++++++++++++++
.../store/impl/worker/RequestHandler.java | 166 -------------
.../store/impl/worker/StoreServiceImpl.java | 77 ------
.../carbondata/store/impl/worker/Worker.java | 26 ++-
.../apache/carbondata/store/util/StoreUtil.java | 134 -----------
.../store/DistributedCarbonStoreTest.java | 59 +++--
.../carbondata/store/LocalCarbonStoreTest.java | 39 ++--
.../org/apache/carbondata/store/TestUtil.java | 10 -
.../horizon/rest/client/HorizonClient.java | 10 +-
.../rest/client/impl/SimpleHorizonClient.java | 12 +-
.../horizon/rest/controller/Horizon.java | 3 +-
.../rest/controller/HorizonController.java | 68 +++---
.../rest/model/validate/RequestValidator.java | 38 +--
.../rest/model/view/CreateTableRequest.java | 4 +-
.../horizon/rest/model/view/FieldRequest.java | 2 +-
.../horizon/rest/model/view/LoadRequest.java | 4 +-
.../horizon/rest/model/view/SelectResponse.java | 11 +-
.../apache/carbondata/horizon/HorizonTest.java | 10 +-
store/sdk/pom.xml | 4 +-
.../carbondata/sdk/file/AvroCarbonWriter.java | 50 ++--
.../sdk/file/CarbonWriterBuilder.java | 4 +-
.../org/apache/carbondata/sdk/file/Field.java | 15 +-
.../org/apache/carbondata/sdk/file/Schema.java | 3 +-
.../carbondata/sdk/store/CarbonStore.java | 151 ++++++++++++
.../sdk/store/CarbonStoreFactory.java | 93 ++++++++
.../sdk/store/DistributedCarbonStore.java | 129 +++++++++++
.../apache/carbondata/sdk/store/KeyedRow.java | 45 ++++
.../apache/carbondata/sdk/store/PrimaryKey.java | 26 +++
.../org/apache/carbondata/sdk/store/Row.java | 27 +++
.../carbondata/sdk/store/conf/StoreConf.java | 207 +++++++++++++++++
.../sdk/store/descriptor/LoadDescriptor.java | 149 ++++++++++++
.../sdk/store/descriptor/ScanDescriptor.java | 151 ++++++++++++
.../sdk/store/descriptor/TableDescriptor.java | 214 +++++++++++++++++
.../sdk/store/descriptor/TableIdentifier.java | 63 +++++
.../sdk/store/exception/CarbonException.java | 38 +++
.../exception/ExecutionTimeoutException.java | 27 +++
.../sdk/store/exception/SchedulerException.java | 31 +++
.../sdk/store/service/ServiceFactory.java | 41 ++++
.../sdk/store/service/StoreService.java | 53 +++++
.../carbondata/sdk/store/util/StoreUtil.java | 134 +++++++++++
.../rest/controller/SqlHorizonController.java | 8 +-
.../rest/model/validate/RequestValidator.java | 8 +-
126 files changed, 4912 insertions(+), 3015 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
index 5435028..afd863f 100644
--- a/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
+++ b/common/src/main/java/org/apache/carbondata/common/annotations/InterfaceStability.java
@@ -42,7 +42,7 @@ import org.apache.carbondata.common.annotations.InterfaceAudience.*;
* </ul>
*/
@InterfaceAudience.User
-@org.apache.hadoop.classification.InterfaceStability.Evolving
+@InterfaceStability.Evolving
public class InterfaceStability {
/**
* Can evolve while retaining compatibility for minor release boundaries.;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
index 48775d4..1f1f087 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/CarbonRow.java
@@ -17,13 +17,21 @@
package org.apache.carbondata.core.datastore.row;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.Serializable;
import java.util.Arrays;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+import org.apache.carbondata.core.util.ObjectSerializationUtil;
+
+import org.apache.hadoop.io.WritableUtils;
+
/**
* This row class is used to transfer the row data from one step to other step
*/
-public class CarbonRow implements Serializable {
+public class CarbonRow implements Serializable, Writable {
private Object[] data;
@@ -87,4 +95,24 @@ public class CarbonRow implements Serializable {
public void clearData() {
this.data = null;
}
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ WritableUtils.writeCompressedByteArray(out, ObjectSerializationUtil.serialize(data));
+ WritableUtils.writeCompressedByteArray(out, ObjectSerializationUtil.serialize(rawData));
+ out.writeShort(rangeId);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ try {
+ data = (Object[]) ObjectSerializationUtil.deserialize(
+ WritableUtils.readCompressedByteArray(in));
+ rawData = (Object[]) ObjectSerializationUtil.deserialize(
+ WritableUtils.readCompressedByteArray(in));
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ rangeId = in.readShort();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 46328f7..abe1810 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -44,7 +44,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ParentColumnTable
* Store the information about the table.
* it stores the fact table as well as aggregate table present in the schema
*/
-public class TableInfo implements Serializable, Writable {
+public class TableInfo implements Serializable, Writable, org.apache.hadoop.io.Writable {
private static final LogService LOGGER =
LogServiceFactory.getLogService(TableInfo.class.getName());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 405ff53..dd3e63f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -42,6 +42,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.internal.index.Block;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
@@ -56,6 +57,11 @@ public class CarbonInputSplit extends FileSplit
private Segment segment;
+ // We use this filePath to store the block location instead of the
+ // filePath in FileSplit, because filePath in FileSplit is not Serializable
+ // before Hadoop 3, see HADOOP-13519
+ private String filePath;
+
private String bucketId;
private String blockletId;
@@ -98,6 +104,7 @@ public class CarbonInputSplit extends FileSplit
numberOfBlocklets = 0;
invalidSegments = new ArrayList<>();
version = CarbonProperties.getInstance().getFormatVersion();
+ filePath = "";
}
private CarbonInputSplit(String segmentId, String blockletId, Path path, long start, long length,
@@ -116,6 +123,7 @@ public class CarbonInputSplit extends FileSplit
this.version = version;
this.deleteDeltaFiles = deleteDeltaFiles;
this.dataMapWritePath = dataMapWritePath;
+ this.filePath = path.toString();
}
public CarbonInputSplit(String segmentId, String blockletId, Path path, long start, long length,
@@ -136,6 +144,7 @@ public class CarbonInputSplit extends FileSplit
numberOfBlocklets = 0;
invalidSegments = new ArrayList<>();
version = CarbonProperties.getInstance().getFormatVersion();
+ filePath = path.toString();
}
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations,
@@ -149,6 +158,7 @@ public class CarbonInputSplit extends FileSplit
numberOfBlocklets = 0;
invalidSegments = new ArrayList<>();
version = CarbonProperties.getInstance().getFormatVersion();
+ filePath = path.toString();
}
/**
@@ -252,10 +262,24 @@ public class CarbonInputSplit extends FileSplit
if (dataMapWriterPathExists) {
dataMapWritePath = in.readUTF();
}
+ boolean filePathExists = in.readBoolean();
+ if (filePathExists) {
+ filePath = in.readUTF();
+ } else {
+ filePath = super.getPath().toString();
+ }
}
@Override public void write(DataOutput out) throws IOException {
- super.write(out);
+ if (super.getPath() != null) {
+ super.write(out);
+ } else {
+ // see HADOOP-13519, after Java deserialization, super.filePath is
+ // null, so write our filePath instead
+ Text.writeString(out, filePath);
+ out.writeLong(getStart());
+ out.writeLong(getLength());
+ }
out.writeUTF(segment.toString());
out.writeShort(version.number());
out.writeUTF(bucketId);
@@ -278,6 +302,10 @@ public class CarbonInputSplit extends FileSplit
if (dataMapWritePath != null) {
out.writeUTF(dataMapWritePath);
}
+ out.writeBoolean(filePath != null);
+ if (filePath != null) {
+ out.writeUTF(filePath);
+ }
}
public List<String> getInvalidSegments() {
@@ -398,7 +426,7 @@ public class CarbonInputSplit extends FileSplit
}
@Override public String getBlockPath() {
- return getPath().getName();
+ return filePath.substring(filePath.lastIndexOf("/") + 1);
}
@Override public List<Long> getMatchedBlocklets() {
@@ -444,4 +472,16 @@ public class CarbonInputSplit extends FileSplit
public Blocklet makeBlocklet() {
return new Blocklet(getPath().getName(), blockletId);
}
+
+ public String[] preferredLocations() {
+ if (CarbonProperties.isTaskLocality()) {
+ try {
+ return getLocations();
+ } catch (IOException e) {
+ return new String[0];
+ }
+ } else {
+ return new String[0];
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
index 70c530f..ce0dc72 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
@@ -185,9 +185,8 @@ public abstract class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
/**
* It sets unresolved filter expression.
*
- * @param configuration
- * @para DataMapJob dataMapJob = getDataMapJob(job.getConfiguration());
-m filterExpression
+ * @param configuration Hadoop conf
+ * @param filterExpression filter expression
*/
public static void setFilterPredicates(Configuration configuration, Expression filterExpression) {
if (filterExpression == null) {
@@ -245,6 +244,17 @@ m filterExpression
return configuration.get(COLUMN_PROJECTION);
}
+ public static String[] getProjectionColumns(Configuration configuration) {
+ String projectionString = getColumnProjection(configuration);
+ String[] projectColumns;
+ if (projectionString != null) {
+ projectColumns = projectionString.split(",");
+ } else {
+ projectColumns = new String[]{};
+ }
+ return projectColumns;
+ }
+
public static void setFgDataMapPruning(Configuration configuration, boolean enable) {
configuration.set(FGDATAMAP_PRUNING, String.valueOf(enable));
}
@@ -353,7 +363,7 @@ m filterExpression
*/
@Override public abstract List<InputSplit> getSplits(JobContext job) throws IOException;
- protected Expression getFilterPredicates(Configuration configuration) {
+ public static Expression getFilterPredicates(Configuration configuration) {
try {
String filterExprString = configuration.get(FILTER_PREDICATE);
if (filterExprString == null) {
@@ -524,7 +534,7 @@ m filterExpression
return prunedBlocklets;
}
- private List<ExtendedBlocklet> getPrunedFiles4ExternalFormat(JobContext job,
+ public List<ExtendedBlocklet> getPrunedFiles4ExternalFormat(JobContext job,
CarbonTable carbonTable,
FilterResolverIntf resolver, List<Segment> segmentIds) throws IOException {
ExplainCollector.addPruningInfo(carbonTable.getTableName());
@@ -664,13 +674,7 @@ m filterExpression
CarbonTable carbonTable = getOrCreateCarbonTable(configuration);
// set projection column in the query model
- String projectionString = getColumnProjection(configuration);
- String[] projectColumns;
- if (projectionString != null) {
- projectColumns = projectionString.split(",");
- } else {
- projectColumns = new String[]{};
- }
+ String[] projectColumns = getProjectionColumns(configuration);
QueryModel queryModel = new QueryModelBuilder(carbonTable)
.projectColumns(projectColumns)
.filterExpression(getFilterPredicates(configuration))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 84e36e3..267ba0f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -71,7 +71,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
/**
* InputFormat for reading carbondata files with table level metadata support,
@@ -98,6 +100,15 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
private CarbonTable carbonTable;
private ReadCommittedScope readCommittedScope;
+ public CarbonTableInputFormat() {
+ }
+
+ public CarbonTableInputFormat(Configuration conf) throws IOException {
+ this.carbonTable = getOrCreateCarbonTable(conf);
+ this.readCommittedScope = getReadCommitted(
+ new JobContextImpl(conf, new JobID()), carbonTable.getAbsoluteTableIdentifier());
+ }
+
/**
* Get the cached CarbonTable or create it by TableInfo in `configuration`
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
index c126e95..78a2e7f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.hadoop.readsupport;
import java.io.IOException;
+import java.io.Serializable;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
@@ -24,7 +25,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
/**
* This is the interface to convert data reading from RecordReader to row representation.
*/
-public interface CarbonReadSupport<T> {
+public interface CarbonReadSupport<T> extends Serializable {
/**
* Initialization if needed based on the projected column list
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/integration/spark2/src/main/scala/org/apache/carbondata/store/WorkerManager.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/store/WorkerManager.scala b/integration/spark2/src/main/scala/org/apache/carbondata/store/WorkerManager.scala
index 7fff2e5..c624702 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/store/WorkerManager.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/store/WorkerManager.scala
@@ -25,9 +25,9 @@ import org.apache.spark.sql.SparkSession
import org.apache.carbondata.common.annotations.InterfaceAudience
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.sdk.store.CarbonStoreFactory
+import org.apache.carbondata.sdk.store.conf.StoreConf
import org.apache.carbondata.spark.util.Util
-import org.apache.carbondata.store.api.CarbonStoreFactory
-import org.apache.carbondata.store.api.conf.StoreConf
import org.apache.carbondata.store.impl.worker.Worker
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 6c13955..3fccfef 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -22,9 +22,7 @@ import java.util.concurrent.atomic.AtomicLong
import scala.collection.JavaConverters._
-import org.apache.hadoop.conf.Configuration
-import org.apache.spark.{SparkConf, SparkContext}
-import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
+import org.apache.spark.SparkContext
import org.apache.spark.sql.SparkSession.Builder
import org.apache.spark.sql.catalyst.encoders.RowEncoder
import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression}
@@ -35,18 +33,16 @@ import org.apache.spark.sql.hive.execution.command.CarbonSetCommand
import org.apache.spark.sql.internal.{SessionState, SharedState}
import org.apache.spark.sql.optimizer.CarbonFilters
import org.apache.spark.sql.profiler.{Profiler, SQLStart}
-import org.apache.spark.util.{CarbonReflectionUtils, Utils}
+import org.apache.spark.util.CarbonReflectionUtils
import org.apache.carbondata.common.annotations.InterfaceAudience
import org.apache.carbondata.common.logging.{LogService, LogServiceFactory}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, ThreadLocalSessionInfo}
-import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
+import org.apache.carbondata.sdk.store.{CarbonStore, CarbonStoreFactory}
+import org.apache.carbondata.sdk.store.conf.StoreConf
+import org.apache.carbondata.sdk.store.descriptor.{ScanDescriptor, TableIdentifier}
import org.apache.carbondata.store.WorkerManager
-import org.apache.carbondata.store.api.{CarbonStore, CarbonStoreFactory}
-import org.apache.carbondata.store.api.conf.StoreConf
-import org.apache.carbondata.store.api.descriptor.{SelectDescriptor, TableIdentifier => CTableIdentifier}
-import org.apache.carbondata.streaming.CarbonStreamingQueryListener
/**
* Session implementation for {org.apache.spark.sql.SparkSession}
@@ -204,7 +200,7 @@ class CarbonSession(@transient val sc: SparkContext,
val storeConf = new StoreConf()
storeConf.conf(StoreConf.STORE_LOCATION, CarbonProperties.getStorePath)
storeConf.conf(StoreConf.MASTER_HOST, InetAddress.getLocalHost.getHostAddress)
- storeConf.conf(StoreConf.MASTER_PORT, CarbonProperties.getSearchMasterPort)
+ storeConf.conf(StoreConf.REGISTRY_PORT, CarbonProperties.getSearchMasterPort)
storeConf.conf(StoreConf.WORKER_HOST, InetAddress.getLocalHost.getHostAddress)
storeConf.conf(StoreConf.WORKER_PORT, CarbonProperties.getSearchWorkerPort)
storeConf.conf(StoreConf.WORKER_CORE_NUM, 2)
@@ -241,13 +237,13 @@ class CarbonSession(@transient val sc: SparkContext,
maxRows: Option[Long] = None,
localMaxRows: Option[Long] = None): DataFrame = {
val table = relation.relation.asInstanceOf[CarbonDatasourceHadoopRelation].carbonTable
- val select = new SelectDescriptor(
- new CTableIdentifier(table.getTableName, table.getDatabaseName),
+ val select = new ScanDescriptor(
+ new TableIdentifier(table.getTableName, table.getDatabaseName),
columns.map(_.name).toArray,
if (expr != null) CarbonFilters.transformExpression(expr) else null,
localMaxRows.getOrElse(Long.MaxValue)
)
- val rows = store.select(select).iterator()
+ val rows = store.scan(select).iterator()
val output = new java.util.ArrayList[Row]()
val maxRowCount = maxRows.getOrElse(Long.MaxValue)
var rowCount = 0
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/integration/zeppelin/src/test/java/org/apache/carbondata/zeppelin/TestCarbonResponse.java
----------------------------------------------------------------------
diff --git a/integration/zeppelin/src/test/java/org/apache/carbondata/zeppelin/TestCarbonResponse.java b/integration/zeppelin/src/test/java/org/apache/carbondata/zeppelin/TestCarbonResponse.java
index 4b169f4..1f1ce65 100644
--- a/integration/zeppelin/src/test/java/org/apache/carbondata/zeppelin/TestCarbonResponse.java
+++ b/integration/zeppelin/src/test/java/org/apache/carbondata/zeppelin/TestCarbonResponse.java
@@ -76,7 +76,7 @@ public class TestCarbonResponse {
" \"timestamp\": 1531884083849,\n" +
" \"status\": 500,\n" +
" \"error\": \"Internal Server Error\",\n" +
- " \"exception\": \"org.apache.carbondata.store.api.exception.StoreException\",\n" +
+ " \"exception\": \"org.apache.carbondata.store.api.exception.CarbonException\",\n" +
" \"message\": \"org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException: " +
"Table or view 'sinka6' already exists in database 'default';\",\n" +
" \"path\": \"/table/sql\"\n" +
@@ -84,7 +84,7 @@ public class TestCarbonResponse {
CarbonResponse errorResponse = CarbonResponse.parse(new ByteArrayInputStream(input.getBytes())).get();
assertEquals("org.apache.spark.sql.catalyst.analysis.TableAlreadyExistsException: " +
"Table or view 'sinka6' already exists in database 'default';", errorResponse.getMessage());
- assertEquals("org.apache.carbondata.store.api.exception.StoreException", errorResponse.getException());
+ assertEquals("org.apache.carbondata.store.api.exception.CarbonException", errorResponse.getException());
assertEquals(1531884083849L, errorResponse.getTimestamp());
assertEquals("Internal Server Error", errorResponse.getError());
assertEquals(500, errorResponse.getStatus());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/conf/store.conf
----------------------------------------------------------------------
diff --git a/store/conf/store.conf b/store/conf/store.conf
index 7f18076..9902061 100644
--- a/store/conf/store.conf
+++ b/store/conf/store.conf
@@ -6,5 +6,7 @@ carbon.store.temp.location=/tmp/carbon.store.temp
# worker and master
carbon.master.host=127.0.0.1
-carbon.master.port=10020
+carbon.master.registry.port=10020
+carbon.master.prune.port=10120
+carbon.master.store.port=9020
carbon.store.location=/tmp/carbon.store
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/pom.xml
----------------------------------------------------------------------
diff --git a/store/core/pom.xml b/store/core/pom.xml
index 44d5ab1..c9e498d 100644
--- a/store/core/pom.xml
+++ b/store/core/pom.xml
@@ -48,8 +48,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>1.8</source>
+ <target>1.8</target>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStore.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStore.java b/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStore.java
deleted file mode 100644
index 3525389..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStore.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api;
-
-import java.io.Closeable;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.annotations.InterfaceStability;
-
-/**
- * Public Interface of CarbonStore
- */
-@InterfaceAudience.User
-@InterfaceStability.Unstable
-public interface CarbonStore extends MetaStore, DataStore, Closeable {
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStoreFactory.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStoreFactory.java b/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStoreFactory.java
deleted file mode 100644
index 76ef450..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/CarbonStoreFactory.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.exception.StoreException;
-
-public class CarbonStoreFactory {
- private static Map<String, CarbonStore> distributedStores = new ConcurrentHashMap<>();
- private static Map<String, CarbonStore> localStores = new ConcurrentHashMap<>();
-
- private CarbonStoreFactory() {
- }
-
- public static CarbonStore getDistributedStore(String storeName, StoreConf storeConf)
- throws StoreException {
- if (distributedStores.containsKey(storeName)) {
- return distributedStores.get(storeName);
- }
-
- // create a new instance
- try {
- String className = "org.apache.carbondata.store.impl.DistributedCarbonStore";
- CarbonStore store = createCarbonStore(storeConf, className);
- distributedStores.put(storeName, store);
- return store;
- } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException |
- InstantiationException e) {
- throw new StoreException(e);
- }
- }
-
- public static void removeDistributedStore(String storeName) throws IOException {
- if (distributedStores.containsKey(storeName)) {
- distributedStores.get(storeName).close();
- distributedStores.remove(storeName);
- }
- }
-
- public static CarbonStore getLocalStore(String storeName, StoreConf storeConf)
- throws StoreException {
- if (localStores.containsKey(storeName)) {
- return localStores.get(storeName);
- }
-
- // create a new instance
- try {
- String className = "org.apache.carbondata.store.impl.LocalCarbonStore";
- CarbonStore store = createCarbonStore(storeConf, className);
- localStores.put(storeName, store);
- return store;
- } catch (ClassNotFoundException | IllegalAccessException | InvocationTargetException |
- InstantiationException e) {
- throw new StoreException(e);
- }
- }
-
- public static void removeLocalStore(String storeName) throws IOException {
- if (localStores.containsKey(storeName)) {
- localStores.get(storeName).close();
- localStores.remove(storeName);
- }
- }
-
- private static CarbonStore createCarbonStore(StoreConf storeConf, String className)
- throws ClassNotFoundException, InstantiationException, IllegalAccessException,
- InvocationTargetException {
- Constructor[] constructor = Class.forName(className).getDeclaredConstructors();
- constructor[0].setAccessible(true);
- return (CarbonStore) constructor[0].newInstance(storeConf);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/DataStore.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/DataStore.java b/store/core/src/main/java/org/apache/carbondata/store/api/DataStore.java
deleted file mode 100644
index d35c133..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/DataStore.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.annotations.InterfaceStability;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.store.api.descriptor.LoadDescriptor;
-import org.apache.carbondata.store.api.descriptor.SelectDescriptor;
-import org.apache.carbondata.store.api.exception.StoreException;
-
-/**
- * Public interface to write and read data in CarbonStore
- */
-@InterfaceAudience.User
-@InterfaceStability.Unstable
-public interface DataStore {
-
- /**
- * Load data into a Table
- * @param load descriptor for load operation
- * @throws IOException if network or disk IO error occurs
- */
- void loadData(LoadDescriptor load) throws IOException, StoreException;
-
- /**
- * Scan a Table and return matched rows
- * @param select descriptor for scan operation, including required column, filter, etc
- * @return matched rows
- * @throws IOException if network or disk IO error occurs
- */
- List<CarbonRow> select(SelectDescriptor select) throws IOException, StoreException;
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/MetaStore.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/MetaStore.java b/store/core/src/main/java/org/apache/carbondata/store/api/MetaStore.java
deleted file mode 100644
index dea6873..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/MetaStore.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.annotations.InterfaceStability;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.store.api.descriptor.TableDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
-import org.apache.carbondata.store.api.exception.StoreException;
-
-/**
- * Public interface to manage table in CarbonStore
- */
-@InterfaceAudience.User
-@InterfaceStability.Unstable
-public interface MetaStore {
- /**
- * Create a Table
- * @param table descriptor for create table operation
- * @throws IOException if network or disk IO error occurs
- */
- void createTable(TableDescriptor table) throws IOException, StoreException;
-
- /**
- * Drop a Table, and remove all data in it
- * @param table table identifier
- * @throws IOException if network or disk IO error occurs
- */
- void dropTable(TableIdentifier table) throws IOException;
-
- CarbonTable getTable(TableIdentifier table) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/SqlStore.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/SqlStore.java b/store/core/src/main/java/org/apache/carbondata/store/api/SqlStore.java
deleted file mode 100644
index 3f52eed..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/SqlStore.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-
-public interface SqlStore {
-
- /**
- * Executor a SQL statement
- * @param sqlString SQL statement
- * @return matched rows
- * @throws IOException if network or disk IO error occurs
- */
- List<CarbonRow> sql(String sqlString) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/conf/StoreConf.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/conf/StoreConf.java b/store/core/src/main/java/org/apache/carbondata/store/api/conf/StoreConf.java
deleted file mode 100644
index 5e4bb4a..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/conf/StoreConf.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.conf;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.store.util.StoreUtil;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-public class StoreConf implements Serializable, Writable {
-
- public static final String SELECT_PROJECTION = "carbon.select.projection";
- public static final String SELECT_FILTER = "carbon.select.filter";
- public static final String SELECT_LIMIT = "carbon.select.limit";
-
- public static final String SELECT_ID = "carbon.select.id";
-
- public static final String WORKER_HOST = "carbon.worker.host";
- public static final String WORKER_PORT = "carbon.worker.port";
- public static final String WORKER_CORE_NUM = "carbon.worker.core.num";
- public static final String MASTER_HOST = "carbon.master.host";
- public static final String MASTER_PORT = "carbon.master.port";
-
- public static final String STORE_TEMP_LOCATION = "carbon.store.temp.location";
- public static final String STORE_LOCATION = "carbon.store.location";
- public static final String STORE_NAME = "carbon.store.name";
-
- public static final String STORE_CONF_FILE = "carbon.store.confFile";
-
- private Map<String, String> conf = new HashMap<>();
-
- public StoreConf() {
- String storeConfFile = System.getProperty(STORE_CONF_FILE);
- if (storeConfFile != null) {
- load(storeConfFile);
- }
- }
-
- public StoreConf(String storeName, String storeLocation) {
- conf.put(STORE_NAME, storeName);
- conf.put(STORE_LOCATION, storeLocation);
- }
-
- public StoreConf(String confFilePath) {
- load(confFilePath);
- }
-
- public StoreConf conf(String key, String value) {
- conf.put(key, value);
- return this;
- }
-
- public StoreConf conf(String key, int value) {
- conf.put(key, "" + value);
- return this;
- }
-
- public void load(String filePath) {
- StoreUtil.loadProperties(filePath, this);
- }
-
- public void conf(StoreConf conf) {
- this.conf.putAll(conf.conf);
- }
-
- public Object conf(String key) {
- return conf.get(key);
- }
-
- public String[] projection() {
- return stringArrayValue(SELECT_PROJECTION);
- }
-
- public String filter() {
- return stringValue(SELECT_FILTER);
- }
-
- public int limit() {
- return intValue(SELECT_LIMIT);
- }
-
- public String masterHost() {
- return stringValue(MASTER_HOST);
- }
-
- public int masterPort() {
- return intValue(MASTER_PORT);
- }
-
- public String workerHost() {
- return stringValue(WORKER_HOST);
- }
-
- public int workerPort() {
- return intValue(WORKER_PORT);
- }
-
- public int workerCoreNum() {
- return intValue(WORKER_CORE_NUM);
- }
-
- public String storeLocation() {
- return stringValue(STORE_LOCATION);
- }
-
- public String[] storeTempLocation() {
- return stringArrayValue(STORE_TEMP_LOCATION);
- }
-
- public String selectId() {
- return stringValue(SELECT_ID);
- }
-
- public Configuration newHadoopConf() {
- Configuration hadoopConf = FileFactory.getConfiguration();
- for (Map.Entry<String, String> entry : conf.entrySet()) {
- String key = entry.getKey();
- String value = entry.getValue();
- if (key != null && value != null && key.startsWith("carbon.hadoop.")) {
- hadoopConf.set(key.substring("carbon.hadoop.".length()), value);
- }
- }
- return hadoopConf;
- }
-
- private String stringValue(String key) {
- Object obj = conf.get(key);
- if (obj == null) {
- return null;
- }
- return obj.toString();
- }
-
- private int intValue(String key) {
- String value = conf.get(key);
- if (value == null) {
- return -1;
- }
- return Integer.parseInt(value);
- }
-
- private String[] stringArrayValue(String key) {
- String value = conf.get(key);
- if (value == null) {
- return null;
- }
- return value.split(",", -1);
- }
-
- @Override public void write(DataOutput out) throws IOException {
- Set<Map.Entry<String, String>> entries = conf.entrySet();
- WritableUtils.writeVInt(out, conf.size());
- for (Map.Entry<String, String> entry : entries) {
- WritableUtils.writeString(out, entry.getKey());
- WritableUtils.writeString(out, entry.getValue());
- }
- }
-
- @Override public void readFields(DataInput in) throws IOException {
- if (conf == null) {
- conf = new HashMap<>();
- }
-
- int size = WritableUtils.readVInt(in);
- String key, value;
- for (int i = 0; i < size; i++) {
- key = WritableUtils.readString(in);
- value = WritableUtils.readString(in);
- conf.put(key, value);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/LoadDescriptor.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/LoadDescriptor.java b/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/LoadDescriptor.java
deleted file mode 100644
index c3a4ff7..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/LoadDescriptor.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.descriptor;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-public class LoadDescriptor {
-
- private TableIdentifier table;
- private String inputPath;
- private Map<String, String> options;
- private boolean isOverwrite;
-
- private LoadDescriptor() {
- }
-
- public LoadDescriptor(TableIdentifier table, String inputPath,
- Map<String, String> options, boolean isOverwrite) {
- Objects.requireNonNull(table);
- Objects.requireNonNull(inputPath);
- this.table = table;
- this.inputPath = inputPath;
- this.options = options;
- this.isOverwrite = isOverwrite;
- }
-
- public TableIdentifier getTable() {
- return table;
- }
-
- public void setTable(TableIdentifier table) {
- this.table = table;
- }
-
- public String getInputPath() {
- return inputPath;
- }
-
- public void setInputPath(String inputPath) {
- this.inputPath = inputPath;
- }
-
- public Map<String, String> getOptions() {
- return options;
- }
-
- public void setOptions(Map<String, String> options) {
- this.options = options;
- }
-
- public boolean isOverwrite() {
- return isOverwrite;
- }
-
- public void setOverwrite(boolean overwrite) {
- isOverwrite = overwrite;
- }
-
- public static class Builder {
- private LoadDescriptor load;
- private Map<String, String> options;
-
- private Builder() {
- load = new LoadDescriptor();
- options = new HashMap<>();
- }
-
- public Builder table(TableIdentifier tableIdentifier) {
- load.setTable(tableIdentifier);
- return this;
- }
-
- public Builder overwrite(boolean isOverwrite) {
- load.setOverwrite(isOverwrite);
- return this;
- }
-
- public Builder inputPath(String inputPath) {
- load.setInputPath(inputPath);
- return this;
- }
-
- public Builder options(String key, String value) {
- options.put(key, value);
- return this;
- }
-
- public LoadDescriptor create() {
- load.setOptions(options);
- return load;
- }
- }
-
- public static Builder builder() {
- return new Builder();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/SelectDescriptor.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/SelectDescriptor.java b/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/SelectDescriptor.java
deleted file mode 100644
index c3627a9..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/SelectDescriptor.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.descriptor;
-
-import java.util.Objects;
-
-import org.apache.carbondata.core.scan.expression.Expression;
-
-public class SelectDescriptor {
-
- private TableIdentifier table;
- private String[] projection;
- private Expression filter;
- private long limit;
-
- private SelectDescriptor() {
- }
-
- public SelectDescriptor(TableIdentifier table, String[] projection,
- Expression filter, long limit) {
- Objects.requireNonNull(table);
- Objects.requireNonNull(projection);
- this.table = table;
- this.projection = projection;
- this.filter = filter;
- this.limit = limit;
- }
-
- public TableIdentifier getTable() {
- return table;
- }
-
- public void setTable(TableIdentifier table) {
- this.table = table;
- }
-
- public String[] getProjection() {
- return projection;
- }
-
- public void setProjection(String[] projection) {
- this.projection = projection;
- }
-
- public Expression getFilter() {
- return filter;
- }
-
- public void setFilter(Expression filter) {
- this.filter = filter;
- }
-
- public long getLimit() {
- return limit;
- }
-
- public void setLimit(long limit) {
- this.limit = limit;
- }
-
- public static class Builder {
- private SelectDescriptor select;
-
- private Builder() {
- select = new SelectDescriptor();
- }
-
- public Builder table(TableIdentifier tableIdentifier) {
- select.setTable(tableIdentifier);
- return this;
- }
-
- public Builder select(String... columnNames) {
- select.setProjection(columnNames);
- return this;
- }
-
- public Builder filter(Expression filter) {
- select.setFilter(filter);
- return this;
- }
-
- public Builder limit(long limit) {
- select.setLimit(limit);
- return this;
- }
-
- public SelectDescriptor create() {
- return select;
- }
- }
-
- public static Builder builder() {
- return new Builder();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableDescriptor.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableDescriptor.java b/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableDescriptor.java
deleted file mode 100644
index 2d677a8..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableDescriptor.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.descriptor;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.sdk.file.Field;
-import org.apache.carbondata.sdk.file.Schema;
-
-public class TableDescriptor {
-
- private boolean ifNotExists;
- private TableIdentifier table;
- private String tablePath;
- private Schema schema;
- private Map<String, String> properties;
- private String comment;
-
- private TableDescriptor() {
- }
-
- public TableDescriptor(TableIdentifier table, Schema schema,
- Map<String, String> properties, String tablePath, String comment, boolean ifNotExists) {
- Objects.requireNonNull(table);
- Objects.requireNonNull(schema);
- this.table = table;
- this.ifNotExists = ifNotExists;
- this.schema = schema;
- this.properties = properties;
- this.tablePath = tablePath;
- this.comment = comment;
- }
-
- public boolean isIfNotExists() {
- return ifNotExists;
- }
-
- public void setIfNotExists(boolean ifNotExists) {
- this.ifNotExists = ifNotExists;
- }
-
- public TableIdentifier getTable() {
- return table;
- }
-
- public void setTable(TableIdentifier table) {
- this.table = table;
- }
-
- public Schema getSchema() {
- return schema;
- }
-
- public void setSchema(Schema schema) {
- this.schema = schema;
- }
-
- public Map<String, String> getProperties() {
- return properties;
- }
-
- public void setProperties(Map<String, String> properties) {
- this.properties = properties;
- }
-
- public String getComment() {
- return comment;
- }
-
- public void setComment(String comment) {
- this.comment = comment;
- }
-
- public void setTablePath(String tablePath) {
- this.tablePath = tablePath;
- }
-
- public String getTablePath() {
- return tablePath;
- }
-
- public static class Builder {
-
- private TableDescriptor table;
- private List<Field> fields;
- private Map<String, String> tblProperties;
-
- private Builder() {
- table = new TableDescriptor();
- fields = new ArrayList<>();
- tblProperties = new HashMap<>();
- }
-
- public Builder ifNotExists() {
- table.setIfNotExists(true);
- return this;
- }
-
- public Builder table(TableIdentifier tableId) {
- table.setTable(tableId);
- return this;
- }
-
- public Builder tablePath(String tablePath) {
- table.setTablePath(tablePath);
- return this;
- }
-
- public Builder comment(String tableComment) {
- table.setComment(tableComment);
- return this;
- }
-
- public Builder column(String name, DataType dataType) {
- fields.add(new Field(name, dataType));
- return this;
- }
-
- public Builder column(String name, DataType dataType, String comment) {
- Field field = new Field(name, dataType);
- field.setColumnComment(comment);
- fields.add(field);
- return this;
- }
-
- public Builder column(String name, DataType dataType, int precision, int scale, String comment)
- {
- Field field = new Field(name, dataType);
- field.setColumnComment(comment);
- field.setScale(scale);
- field.setPrecision(precision);
- fields.add(field);
- return this;
- }
-
- public Builder tblProperties(String key, String value) {
- tblProperties.put(key, value);
- return this;
- }
-
- public TableDescriptor create() {
- Field[] fieldArray = new Field[fields.size()];
- fieldArray = fields.toArray(fieldArray);
- Schema schema = new Schema(fieldArray);
- table.setSchema(schema);
- table.setProperties(tblProperties);
- return table;
- }
- }
-
- public static Builder builder() {
- return new Builder();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableIdentifier.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableIdentifier.java b/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableIdentifier.java
deleted file mode 100644
index ab8edf8..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/descriptor/TableIdentifier.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.descriptor;
-
-public class TableIdentifier {
- private String tableName;
- private String databaseName;
-
- public TableIdentifier(String tableName, String databaseName) {
- this.tableName = tableName;
- this.databaseName = databaseName;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public String getDatabaseName() {
- return databaseName;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/exception/ExecutionTimeoutException.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/exception/ExecutionTimeoutException.java b/store/core/src/main/java/org/apache/carbondata/store/api/exception/ExecutionTimeoutException.java
deleted file mode 100644
index 728837d..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/exception/ExecutionTimeoutException.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.exception;
-
-public class ExecutionTimeoutException extends RuntimeException {
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/exception/SchedulerException.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/exception/SchedulerException.java b/store/core/src/main/java/org/apache/carbondata/store/api/exception/SchedulerException.java
deleted file mode 100644
index 28b8a50..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/exception/SchedulerException.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.exception;
-
-public class SchedulerException extends RuntimeException {
-
- public SchedulerException(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/api/exception/StoreException.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/api/exception/StoreException.java b/store/core/src/main/java/org/apache/carbondata/store/api/exception/StoreException.java
deleted file mode 100644
index 315a09b..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/api/exception/StoreException.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.api.exception;
-
-public class StoreException extends Exception {
-
- public StoreException() {
- super();
- }
-
- public StoreException(String message) {
- super(message);
- }
-
- public StoreException(Exception e) {
- super(e);
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/DataLoader.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/DataLoader.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/DataLoader.java
new file mode 100644
index 0000000..4b79ee4
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/DataLoader.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.metadata.datatype.StructType;
+import org.apache.carbondata.sdk.store.Row;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+/**
+ * A Loader is used to load data from files to the table
+ */
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface DataLoader extends TransactionalOperation, Serializable {
+ /**
+ * Trigger the load operation
+ * @throws CarbonException if any error occurs
+ */
+ void load() throws CarbonException;
+
+ /**
+ * Append a batch of rows.
+ * @param rows rows to append
+ * @param schema schema of the input row
+ * @throws CarbonException if any error occurs
+ */
+ void append(Iterator<Row> rows, StructType schema) throws CarbonException;
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/DataScanner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/DataScanner.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/DataScanner.java
new file mode 100644
index 0000000..c6c5628
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/DataScanner.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.io.Serializable;
+import java.util.Iterator;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface DataScanner<T> extends Serializable {
+
+ /**
+ * Perform a scan in a distributed compute framework like Spark, Presto, etc.
+ * Filter/Projection/Limit operation is pushed down to the scan.
+ *
+ * This should be used with {@link Pruner#prune(TableIdentifier, Expression)}
+ * in a distributed compute environment. It enables the framework to
+ * do a parallel scan by creating multiple {@link ScanUnit} and perform
+ * parallel scan in worker, such as Spark executor
+ *
+ * The return result is in batch so that the caller can start next
+ * level of computation before getting all results, such as
+ * implementing a `prefetch` execution model.
+ *
+ * @param input one scan unit
+ * @return scan result, the result is returned in batch
+ * @throws CarbonException if any error occurs
+ */
+ Iterator<? extends ResultBatch<T>> scan(ScanUnit input) throws CarbonException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStore.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStore.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStore.java
new file mode 100644
index 0000000..01c2008
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStore.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.util.Map;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
+import org.apache.carbondata.sdk.store.CarbonStore;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+/**
+ * Internal API for engine integration developers
+ */
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface InternalCarbonStore extends CarbonStore {
+
+ /**
+ * Get CarbonTable object from the store
+ *
+ * @param tableIdentifier table identifier
+ * @return CarbonTable object
+ * @throws CarbonException if any error occurs
+ */
+ CarbonTable getCarbonTable(TableIdentifier tableIdentifier) throws CarbonException;
+
+ /**
+ * Return a new Loader that can be used to load data in distributed compute framework
+ * @param load descriptor for load operation
+ * @return a new Loader
+ * @throws CarbonException if any error occurs
+ */
+ DataLoader newLoader(LoadDescriptor load) throws CarbonException;
+
+ /**
+ * Return a new Scanner that can be used in for parallel scan
+ *
+ * @param tableIdentifier table to scan
+ * @param scanOption options for scan, use {@link ScanOption} for the map key
+ * @param readSupport read support to convert the row to output object
+ * @param <T> the target object type contain in {@link ResultBatch}
+ * @return a new Scanner
+ * @throws CarbonException if any error occurs
+ */
+ <T> Scanner<T> newScanner(
+ TableIdentifier tableIdentifier,
+ ScanDescriptor scanDescriptor,
+ Map<String, String> scanOption,
+ CarbonReadSupport<T> readSupport) throws CarbonException;
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStoreFactory.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStoreFactory.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStoreFactory.java
new file mode 100644
index 0000000..c875aa1
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/InternalCarbonStoreFactory.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.store.impl.InternalCarbonStoreImpl;
+
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public class InternalCarbonStoreFactory {
+
+ private static final Map<String, InternalCarbonStore> stores = new ConcurrentHashMap<>();
+
+ public static synchronized InternalCarbonStore getStore(String storeName, StoreConf conf)
+ throws IOException {
+ InternalCarbonStore store = stores.getOrDefault(storeName, newStore(conf));
+ stores.putIfAbsent(storeName, store);
+ return store;
+ }
+
+ private static InternalCarbonStore newStore(StoreConf conf) throws IOException {
+ return new InternalCarbonStoreImpl(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/Pruner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/Pruner.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/Pruner.java
new file mode 100644
index 0000000..4a1d2e5
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/Pruner.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface Pruner {
+
+ /**
+ * Return an array of ScanUnit which will be the input in
+ * {@link Scanner#scan(ScanUnit)}
+ *
+ * Implementation will leverage index to prune using specified
+ * filter expression
+ *
+ * @param table table identifier
+ * @param filterExpression expression of filter predicate given by user
+ * @return list of ScanUnit which should be passed to
+ * {@link Scanner#scan(ScanUnit)}
+ * @throws CarbonException if any error occurs
+ */
+ List<ScanUnit> prune(TableIdentifier table, Expression filterExpression) throws CarbonException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/ResultBatch.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/ResultBatch.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/ResultBatch.java
new file mode 100644
index 0000000..221e0f6
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/ResultBatch.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface ResultBatch<T> {
+
+ /**
+ * Return true if the result is returned in columnar batch, otherwise is row by row.
+ * By default, it is columnar batch.
+ */
+ default boolean isColumnar() {
+ return true;
+ }
+
+ /**
+ * Return true if there is more elements in this batch.
+ */
+ boolean hasNext();
+
+ /**
+ * Return next item.
+ * If {@link #isColumnar()} return true, there is only one element in this batch
+ * which is {@link ColumnarBatch}, otherwise, this batch return row by row, caller
+ * should call next() until no element left.
+ */
+ T next();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanOption.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanOption.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanOption.java
new file mode 100644
index 0000000..8e5797d
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanOption.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.util.Map;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public class ScanOption {
+
+ /** batch size in number of rows in one {@link ResultBatch} */
+ public static final String BATCH_SIZE = "batchSize";
+
+ /**
+ * set to true if return in row major object in {@link ResultBatch},
+ * otherwise columnar object is returned
+ */
+ public static final String ROW_MAJOR = "rowMajor";
+
+ /**
+ * set to true if enable remote prune by RPC call,
+ * otherwise prune executes in caller's JVM
+ */
+ public static final String REMOTE_PRUNE = "remotePrune";
+
+ /**
+ * set to true if enable operator pushdown like scan and load
+ * otherwise operation executes in caller's JVM
+ */
+ public static final String OP_PUSHDOWN = "operatorPushDown";
+
+ /**
+ * Return true if REMOTE_PRUNE is set, default is false
+ */
+ public static boolean isRemotePrune(Map<String, String> options) {
+ if (options == null) {
+ return false;
+ }
+ return Boolean.valueOf(options.getOrDefault(REMOTE_PRUNE, "false"));
+ }
+
+ /**
+ * Return true if REMOTE_PRUNE is set, default is false
+ */
+ public static boolean isOperatorPushdown(Map<String, String> options) {
+ if (options == null) {
+ return false;
+ }
+ return Boolean.valueOf(options.getOrDefault(OP_PUSHDOWN, "false"));
+ }
+}
\ No newline at end of file
[4/5] carbondata git commit: [CARBONDATA-2825][CARBONDATA-2828]
CarbonStore and InternalCarbonStore API This closes #2589
Posted by qi...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanUnit.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanUnit.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanUnit.java
new file mode 100644
index 0000000..fdcffdf
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/ScanUnit.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+
+/**
+ * An unit for the scanner in Carbon Store
+ */
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface ScanUnit extends Serializable, Writable {
+
+ /**
+ * Return the list of preferred location of this ScanUnit.
+ * The default return value is empty string array, which means this ScanUnit
+ * has no location preference.
+ */
+ default String[] preferredLocations() {
+ return new String[0];
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/Scanner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/Scanner.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/Scanner.java
new file mode 100644
index 0000000..e56eee2
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/Scanner.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+/**
+ * A Scanner is used to scan the table in a distributed compute
+ * engine like Apache Spark
+ */
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface Scanner<T> extends DataScanner<T>, Pruner, Serializable {
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/devapi/TransactionalOperation.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/devapi/TransactionalOperation.java b/store/core/src/main/java/org/apache/carbondata/store/devapi/TransactionalOperation.java
new file mode 100644
index 0000000..29b3bce
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/devapi/TransactionalOperation.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.devapi;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+
+@InterfaceAudience.Developer("Integration")
+@InterfaceStability.Unstable
+public interface TransactionalOperation {
+ /**
+ * commit the transaction when operation succeed
+ */
+ void commit();
+
+ /**
+ * close the transaction when operation failed
+ */
+ void close();
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/BlockScanUnit.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/BlockScanUnit.java b/store/core/src/main/java/org/apache/carbondata/store/impl/BlockScanUnit.java
new file mode 100644
index 0000000..d7c2194
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/BlockScanUnit.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.store.devapi.ScanUnit;
+
+/**
+ * It contains a block to scan, and a destination worker who should scan it
+ */
+@InterfaceAudience.Internal
+public class BlockScanUnit implements ScanUnit {
+
+ // the data block to scan
+ private CarbonInputSplit inputSplit;
+
+ // the worker who should scan this unit
+ private Schedulable schedulable;
+
+ public BlockScanUnit() {
+ }
+
+ public BlockScanUnit(CarbonInputSplit inputSplit, Schedulable schedulable) {
+ this.inputSplit = inputSplit;
+ this.schedulable = schedulable;
+ }
+
+ public CarbonInputSplit getInputSplit() {
+ return inputSplit;
+ }
+
+ public Schedulable getSchedulable() {
+ return schedulable;
+ }
+
+ @Override
+ public String[] preferredLocations() {
+ return inputSplit.preferredLocations();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ inputSplit.write(out);
+ schedulable.write(out);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ inputSplit = new CarbonInputSplit();
+ inputSplit.readFields(in);
+ schedulable = new Schedulable();
+ schedulable.readFields(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/CarbonStoreBase.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/CarbonStoreBase.java b/store/core/src/main/java/org/apache/carbondata/store/impl/CarbonStoreBase.java
deleted file mode 100644
index 7e50102..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/CarbonStoreBase.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.block.Distributable;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.exception.InvalidConfigurationException;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.core.scan.model.QueryModelBuilder;
-import org.apache.carbondata.core.util.CarbonTaskInfo;
-import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
-import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
-import org.apache.carbondata.hadoop.CarbonRecordReader;
-import org.apache.carbondata.hadoop.api.CarbonInputFormat;
-import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
-import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
-import org.apache.carbondata.store.api.CarbonStore;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.descriptor.TableDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
-import org.apache.carbondata.store.api.exception.StoreException;
-import org.apache.carbondata.store.impl.rpc.model.Scan;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-
-/**
- * Provides base functionality of CarbonStore, it contains basic implementation of metadata
- * management, data pruning and data scan logic.
- */
-@InterfaceAudience.Internal
-public abstract class CarbonStoreBase implements CarbonStore {
-
- private static LogService LOGGER =
- LogServiceFactory.getLogService(CarbonStoreBase.class.getCanonicalName());
-
- MetaProcessor metaProcessor;
- private StoreConf storeConf;
-
- CarbonStoreBase(StoreConf storeConf) {
- this.storeConf = storeConf;
- this.metaProcessor = new MetaProcessor(this);
- }
-
- @Override
- public void createTable(TableDescriptor table) throws IOException, StoreException {
- Objects.requireNonNull(table);
- metaProcessor.createTable(table);
- }
-
- @Override
- public void dropTable(TableIdentifier table) throws IOException {
- Objects.requireNonNull(table);
- metaProcessor.dropTable(table);
- }
-
- @Override
- public CarbonTable getTable(TableIdentifier table) throws IOException {
- Objects.requireNonNull(table);
- return metaProcessor.getTable(table);
- }
-
- public String getTablePath(String tableName, String databaseName) {
- Objects.requireNonNull(tableName);
- Objects.requireNonNull(databaseName);
- return String.format("%s/%s", storeConf.storeLocation(), tableName);
- }
-
- /**
- * Prune data by using CarbonInputFormat.getSplit
- * Return a mapping of host address to list of block.
- * This should be invoked in driver side.
- */
- public static List<Distributable> pruneBlock(CarbonTable table, String[] columns,
- Expression filter) throws IOException {
- Objects.requireNonNull(table);
- Objects.requireNonNull(columns);
- JobConf jobConf = new JobConf(new Configuration());
- Job job = new Job(jobConf);
- CarbonTableInputFormat format;
- try {
- format = CarbonInputFormatUtil.createCarbonTableInputFormat(
- job, table, columns, filter, null, null, true);
- } catch (InvalidConfigurationException e) {
- throw new IOException(e.getMessage());
- }
-
- // We will do FG pruning in reader side, so don't do it here
- CarbonInputFormat.setFgDataMapPruning(job.getConfiguration(), false);
- List<InputSplit> splits = format.getSplits(job);
- List<Distributable> blockInfos = new ArrayList<>(splits.size());
- for (InputSplit split : splits) {
- blockInfos.add((Distributable) split);
- }
- return blockInfos;
- }
-
- /**
- * Scan data and return matched rows. This should be invoked in worker side.
- * @param table carbon table
- * @param scan scan parameter
- * @return matched rows
- * @throws IOException if IO error occurs
- */
- public static List<CarbonRow> scan(CarbonTable table, Scan scan) throws IOException {
- CarbonTaskInfo carbonTaskInfo = new CarbonTaskInfo();
- carbonTaskInfo.setTaskId(System.nanoTime());
- ThreadLocalTaskInfo.setCarbonTaskInfo(carbonTaskInfo);
-
- CarbonMultiBlockSplit mbSplit = scan.getSplit();
- long limit = scan.getLimit();
- QueryModel queryModel = createQueryModel(table, scan);
-
- LOGGER.info(String.format("[QueryId:%d] %s, number of block: %d", scan.getRequestId(),
- queryModel.toString(), mbSplit.getAllSplits().size()));
-
- // read all rows by the reader
- List<CarbonRow> rows = new LinkedList<>();
- try (CarbonRecordReader<CarbonRow> reader = new IndexedRecordReader(scan.getRequestId(),
- table, queryModel)) {
- reader.initialize(mbSplit, null);
-
- // loop to read required number of rows.
- // By default, if user does not specify the limit value, limit is Long.MaxValue
- long rowCount = 0;
- while (reader.nextKeyValue() && rowCount < limit) {
- rows.add(reader.getCurrentValue());
- rowCount++;
- }
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- LOGGER.info(String.format("[QueryId:%d] scan completed, return %d rows",
- scan.getRequestId(), rows.size()));
- return rows;
- }
-
- private static QueryModel createQueryModel(CarbonTable table, Scan scan) {
- String[] projectColumns = scan.getProjectColumns();
- Expression filter = null;
- if (scan.getFilterExpression() != null) {
- filter = scan.getFilterExpression();
- }
- return new QueryModelBuilder(table)
- .projectColumns(projectColumns)
- .filterExpression(filter)
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/DataOperation.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/DataOperation.java b/store/core/src/main/java/org/apache/carbondata/store/impl/DataOperation.java
new file mode 100644
index 0000000..fc50a62
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/DataOperation.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.QueryModelBuilder;
+import org.apache.carbondata.core.util.CarbonTaskInfo;
+import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+import org.apache.carbondata.hadoop.CarbonRecordReader;
+import org.apache.carbondata.store.impl.service.model.ScanRequest;
+
+public class DataOperation {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DataOperation.class.getCanonicalName());
+
+ /**
+ * Scan data and return matched rows. This should be invoked in worker side.
+ * @param tableInfo carbon table
+ * @param scan scan parameter
+ * @return matched rows
+ * @throws IOException if IO error occurs
+ */
+ public static List<CarbonRow> scan(TableInfo tableInfo, ScanRequest scan) throws IOException {
+ CarbonTable table = CarbonTable.buildFromTableInfo(tableInfo);
+ CarbonTaskInfo carbonTaskInfo = new CarbonTaskInfo();
+ carbonTaskInfo.setTaskId(System.nanoTime());
+ ThreadLocalTaskInfo.setCarbonTaskInfo(carbonTaskInfo);
+
+ CarbonMultiBlockSplit mbSplit = scan.getSplit();
+ long limit = scan.getLimit();
+ QueryModel queryModel = createQueryModel(table, scan);
+
+ LOGGER.info(String.format("[QueryId:%d] %s, number of block: %d", scan.getRequestId(),
+ queryModel.toString(), mbSplit.getAllSplits().size()));
+
+ // read all rows by the reader
+ List<CarbonRow> rows = new LinkedList<>();
+ try (CarbonRecordReader<CarbonRow> reader = new IndexedRecordReader(scan.getRequestId(),
+ table, queryModel)) {
+ reader.initialize(mbSplit, null);
+
+ // loop to read required number of rows.
+ // By default, if user does not specify the limit value, limit is Long.MaxValue
+ long rowCount = 0;
+ while (reader.nextKeyValue() && rowCount < limit) {
+ rows.add(reader.getCurrentValue());
+ rowCount++;
+ }
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ LOGGER.info(String.format("[QueryId:%d] scan completed, return %d rows",
+ scan.getRequestId(), rows.size()));
+ return rows;
+ }
+
+ private static QueryModel createQueryModel(CarbonTable table, ScanRequest scan) {
+ String[] projectColumns = scan.getProjectColumns();
+ Expression filter = null;
+ if (scan.getFilterExpression() != null) {
+ filter = scan.getFilterExpression();
+ }
+ return new QueryModelBuilder(table)
+ .projectColumns(projectColumns)
+ .filterExpression(filter)
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/DataServicePool.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/DataServicePool.java b/store/core/src/main/java/org/apache/carbondata/store/impl/DataServicePool.java
new file mode 100644
index 0000000..8d99c3e
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/DataServicePool.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.carbondata.store.impl.service.DataService;
+import org.apache.carbondata.store.impl.service.ServiceFactory;
+
+class DataServicePool {
+ private static final Map<Schedulable, DataService> dataServicePool = new ConcurrentHashMap<>();
+
+ private DataServicePool() {
+ }
+
+ static DataService getOrCreateDataService(Schedulable schedulable) throws IOException {
+ DataService service = dataServicePool.getOrDefault(
+ schedulable,
+ ServiceFactory.createDataService(schedulable.getAddress(), schedulable.getPort()));
+ dataServicePool.putIfAbsent(schedulable, service);
+ return service;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/DelegatedScanner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/DelegatedScanner.java b/store/core/src/main/java/org/apache/carbondata/store/impl/DelegatedScanner.java
new file mode 100644
index 0000000..bd40667
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/DelegatedScanner.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.DataScanner;
+import org.apache.carbondata.store.devapi.Pruner;
+import org.apache.carbondata.store.devapi.ResultBatch;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.devapi.Scanner;
+
+public class DelegatedScanner<T> implements Scanner<T> {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DelegatedScanner.class.getCanonicalName());
+
+ private Pruner pruner;
+ private DataScanner<T> scanner;
+
+ public DelegatedScanner(Pruner pruner, DataScanner<T> scanner) {
+ this.pruner = pruner;
+ this.scanner = scanner;
+ }
+
+ @Override
+ public List<ScanUnit> prune(TableIdentifier table, Expression filterExpression)
+ throws CarbonException {
+ return pruner.prune(table, filterExpression);
+ }
+
+ @Override
+ public Iterator<? extends ResultBatch<T>> scan(ScanUnit input) throws CarbonException {
+ return scanner.scan(input);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/DistributedCarbonStore.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/DistributedCarbonStore.java b/store/core/src/main/java/org/apache/carbondata/store/impl/DistributedCarbonStore.java
deleted file mode 100644
index 3667aea..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/DistributedCarbonStore.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.annotations.InterfaceStability;
-import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.block.Distributable;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.processing.loading.model.CarbonLoadModelBuilder;
-import org.apache.carbondata.processing.util.CarbonLoaderUtil;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.descriptor.LoadDescriptor;
-import org.apache.carbondata.store.api.descriptor.SelectDescriptor;
-import org.apache.carbondata.store.api.exception.ExecutionTimeoutException;
-import org.apache.carbondata.store.api.exception.StoreException;
-import org.apache.carbondata.store.impl.master.Schedulable;
-import org.apache.carbondata.store.impl.master.Scheduler;
-import org.apache.carbondata.store.impl.rpc.model.BaseResponse;
-import org.apache.carbondata.store.impl.rpc.model.LoadDataRequest;
-import org.apache.carbondata.store.impl.rpc.model.QueryResponse;
-import org.apache.carbondata.store.impl.rpc.model.Scan;
-
-/**
- * A CarbonStore that leverage multiple servers via RPC calls (Master and Workers)
- */
-@InterfaceAudience.User
-@InterfaceStability.Unstable
-class DistributedCarbonStore extends CarbonStoreBase {
- private static LogService LOGGER =
- LogServiceFactory.getLogService(DistributedCarbonStore.class.getCanonicalName());
- private SegmentTxnManager txnManager;
- private Scheduler scheduler;
- private Random random = new Random();
-
- DistributedCarbonStore(StoreConf storeConf) throws IOException {
- super(storeConf);
- this.scheduler = new Scheduler(storeConf);
- txnManager = SegmentTxnManager.getInstance();
- }
-
- @Override
- public void loadData(LoadDescriptor load) throws IOException, StoreException {
- Objects.requireNonNull(load);
- CarbonTable table = metaProcessor.getTable(load.getTable());
- CarbonLoadModelBuilder builder = new CarbonLoadModelBuilder(table);
- builder.setInputPath(load.getInputPath());
- CarbonLoadModel loadModel;
- try {
- loadModel = builder.build(load.getOptions(), System.currentTimeMillis(), "0");
- } catch (InvalidLoadOptionException e) {
- LOGGER.error(e, "Invalid loadDescriptor options");
- throw new StoreException(e);
- } catch (IOException e) {
- LOGGER.error(e, "Failed to loadDescriptor data");
- throw e;
- }
-
- Schedulable worker = scheduler.pickNexWorker();
- try {
- if (loadModel.getFactTimeStamp() == 0) {
- loadModel.setFactTimeStamp(CarbonUpdateUtil.readCurrentTime());
- }
- txnManager.openSegment(loadModel, load.isOverwrite());
- LoadDataRequest request = new LoadDataRequest(loadModel);
- BaseResponse response = scheduler.sendRequest(worker, request);
- if (Status.SUCCESS.ordinal() == response.getStatus()) {
- txnManager.commitSegment(loadModel);
- } else {
- txnManager.closeSegment(loadModel);
- throw new StoreException(response.getMessage());
- }
- } finally {
- worker.workload.decrementAndGet();
- }
- }
-
- @Override
- public List<CarbonRow> select(SelectDescriptor select) throws IOException, StoreException {
- Objects.requireNonNull(select);
- CarbonTable carbonTable = metaProcessor.getTable(select.getTable());
- return select(
- carbonTable,
- select.getProjection(),
- select.getFilter(),
- select.getLimit(),
- select.getLimit());
- }
-
- /**
- * Execute search by firing RPC call to worker, return the result rows
- *
- * @param table table to search
- * @param columns projection column names
- * @param filter filter expression
- * @param globalLimit max number of rows required in Master
- * @param localLimit max number of rows required in Worker
- * @return CarbonRow
- */
- private List<CarbonRow> select(CarbonTable table, String[] columns, Expression filter,
- long globalLimit, long localLimit) throws IOException {
- Objects.requireNonNull(table);
- Objects.requireNonNull(columns);
- if (globalLimit < 0 || localLimit < 0) {
- throw new IllegalArgumentException("limit should be positive");
- }
-
- int queryId = random.nextInt();
-
- List<CarbonRow> output = new ArrayList<>();
-
- // prune data and get a mapping of worker hostname to list of blocks,
- // then add these blocks to the Scan and fire the RPC call
- List<Distributable> blockInfos = pruneBlock(table, columns, filter);
-
- Map<String, List<Distributable>> nodeBlockMapping =
- CarbonLoaderUtil.nodeBlockMapping(
- blockInfos, -1, scheduler.getAllWorkerAddresses(),
- CarbonLoaderUtil.BlockAssignmentStrategy.BLOCK_NUM_FIRST, null);
-
- Set<Map.Entry<String, List<Distributable>>> entries = nodeBlockMapping.entrySet();
- List<Future<QueryResponse>> futures = new ArrayList<>(entries.size());
- List<Schedulable> workers = new ArrayList<>(entries.size());
- for (Map.Entry<String, List<Distributable>> entry : entries) {
- CarbonMultiBlockSplit split = new CarbonMultiBlockSplit(entry.getValue(), entry.getKey());
- Scan scan =
- new Scan(queryId, split, table.getTableInfo(), columns, filter, localLimit);
-
- // Find an Endpoind and send the request to it
- // This RPC is non-blocking so that we do not need to wait before send to next worker
- Schedulable worker = scheduler.pickWorker(entry.getKey());
- workers.add(worker);
- futures.add(scheduler.sendRequestAsync(worker, scan));
- }
-
- int rowCount = 0;
- int length = futures.size();
- for (int i = 0; i < length; i++) {
- Future<QueryResponse> future = futures.get(i);
- Schedulable worker = workers.get(i);
- if (rowCount < globalLimit) {
- // wait for worker
- QueryResponse response = null;
- try {
- response = future
- .get((long) (CarbonProperties.getInstance().getQueryTimeout()), TimeUnit.SECONDS);
- } catch (ExecutionException | InterruptedException e) {
- throw new IOException("exception in worker: " + e.getMessage());
- } catch (TimeoutException t) {
- throw new ExecutionTimeoutException();
- } finally {
- worker.workload.decrementAndGet();
- }
- LOGGER.info("[QueryId: " + queryId + "] receive search response from worker " + worker);
- rowCount += onSuccess(queryId, response, output, globalLimit);
- }
- }
- return output;
- }
-
- private int onSuccess(int queryId, QueryResponse result, List<CarbonRow> output, long globalLimit)
- throws IOException {
- // in case of RPC success, collect all rows in response message
- if (result.getQueryId() != queryId) {
- throw new IOException(
- "queryId in response does not match request: " + result.getQueryId() + " != " + queryId);
- }
- if (result.getStatus() != Status.SUCCESS.ordinal()) {
- throw new IOException("failure in worker: " + result.getMessage());
- }
- int rowCount = 0;
- Object[][] rows = result.getRows();
- for (Object[] row : rows) {
- output.add(new CarbonRow(row));
- rowCount++;
- if (rowCount >= globalLimit) {
- break;
- }
- }
- LOGGER.info("[QueryId:" + queryId + "] accumulated result size " + rowCount);
- return rowCount;
- }
-
- @Override
- public void close() throws IOException {
- LOGGER.info("Shutting down all workers...");
- scheduler.stopAllWorkers();
- LOGGER.info("All workers are shut down");
- try {
- LOGGER.info("Stopping master...");
- scheduler.stopService();
- LOGGER.info("Master stopped");
- } catch (InterruptedException e) {
- throw new IOException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/IndexOperation.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/IndexOperation.java b/store/core/src/main/java/org/apache/carbondata/store/impl/IndexOperation.java
new file mode 100644
index 0000000..49afb45
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/IndexOperation.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.carbondata.core.exception.InvalidConfigurationException;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.api.CarbonInputFormat;
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+
+public class IndexOperation {
+
+ /**
+ * Prune data by leveraging Carbon's Index
+ */
+ public static List<CarbonInputSplit> pruneBlock(TableInfo tableInfo, Expression filter)
+ throws IOException {
+ Objects.requireNonNull(tableInfo);
+ CarbonTable table = CarbonTable.buildFromTableInfo(tableInfo);
+ JobConf jobConf = new JobConf(new Configuration());
+ Job job = new Job(jobConf);
+ CarbonTableInputFormat format;
+ try {
+ // We just want to do pruning, so passing empty projection columns
+ format = CarbonInputFormatUtil.createCarbonTableInputFormat(
+ job, table, new String[0], filter, null, null, true);
+ } catch (InvalidConfigurationException e) {
+ throw new IOException(e.getMessage());
+ }
+
+ // We will do FG pruning in reader side, so don't do it here
+ CarbonInputFormat.setFgDataMapPruning(job.getConfiguration(), false);
+ return format.getSplits(job);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/IndexedRecordReader.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/IndexedRecordReader.java b/store/core/src/main/java/org/apache/carbondata/store/impl/IndexedRecordReader.java
index 64f0742..c856d18 100644
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/IndexedRecordReader.java
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/IndexedRecordReader.java
@@ -65,7 +65,7 @@ class IndexedRecordReader extends CarbonRecordReader<CarbonRow> {
private int queryId;
private CarbonTable table;
- public IndexedRecordReader(int queryId, CarbonTable table, QueryModel queryModel) {
+ IndexedRecordReader(int queryId, CarbonTable table, QueryModel queryModel) {
super(queryModel, new CarbonRowReadSupport());
this.queryId = queryId;
this.table = table;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/InternalCarbonStoreImpl.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/InternalCarbonStoreImpl.java b/store/core/src/main/java/org/apache/carbondata/store/impl/InternalCarbonStoreImpl.java
new file mode 100644
index 0000000..4821116
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/InternalCarbonStoreImpl.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
+import org.apache.carbondata.sdk.store.DistributedCarbonStore;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.DataLoader;
+import org.apache.carbondata.store.devapi.DataScanner;
+import org.apache.carbondata.store.devapi.InternalCarbonStore;
+import org.apache.carbondata.store.devapi.Pruner;
+import org.apache.carbondata.store.devapi.ScanOption;
+import org.apache.carbondata.store.devapi.Scanner;
+
+/**
+ * This store does prune and scan either remotely by sending RPC to Master/Worker
+ * or in local JVM, depends on parameter passed.
+ */
+@InterfaceAudience.Internal
+public class InternalCarbonStoreImpl extends DistributedCarbonStore implements InternalCarbonStore {
+
+ private Map<TableIdentifier, CarbonTable> tableCache = new ConcurrentHashMap<>();
+ private StoreConf storeConf;
+
+ public InternalCarbonStoreImpl(StoreConf storeConf) throws IOException {
+ super(storeConf);
+ this.storeConf = storeConf;
+ }
+
+ @Override
+ public CarbonTable getCarbonTable(TableIdentifier tableIdentifier)
+ throws CarbonException {
+ Objects.requireNonNull(tableIdentifier);
+ CarbonTable carbonTable = tableCache.getOrDefault(
+ tableIdentifier,
+ CarbonTable.buildFromTableInfo(storeService.getTable(tableIdentifier)));
+ tableCache.putIfAbsent(tableIdentifier, carbonTable);
+ return carbonTable;
+ }
+
+ @Override
+ public DataLoader newLoader(LoadDescriptor load) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * By default, it returns a Scanner that does prune and scan remotely
+ */
+ @Override
+ public <T> Scanner<T> newScanner(TableIdentifier identifier, ScanDescriptor scanDescriptor,
+ Map<String, String> scanOption, CarbonReadSupport<T> readSupport)
+ throws CarbonException {
+ Objects.requireNonNull(identifier);
+ Objects.requireNonNull(scanDescriptor);
+ boolean isRemotePrune;
+ boolean isOpPushdown;
+ if (scanOption == null) {
+ isRemotePrune = true;
+ isOpPushdown = true;
+ } else {
+ isRemotePrune = ScanOption.isRemotePrune(scanOption);
+ isOpPushdown = ScanOption.isOperatorPushdown(scanOption);
+ }
+
+ TableInfo tableInfo = MetaOperation.getTable(identifier, storeConf);
+ Pruner pruner;
+ DataScanner<T> scanner;
+
+ if (isRemotePrune) {
+ pruner = new RemotePruner(storeConf.masterHost(), storeConf.pruneServicePort());
+ } else {
+ pruner = new LocalPruner(storeConf);
+ }
+ if (isOpPushdown) {
+ scanner = new RemoteDataScanner<>(tableInfo, scanDescriptor, scanOption, readSupport);
+ } else {
+ scanner = new LocalDataScanner<>(storeConf, scanDescriptor, scanOption);
+ }
+
+ return new DelegatedScanner<>(pruner, scanner);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/LocalCarbonStore.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/LocalCarbonStore.java b/store/core/src/main/java/org/apache/carbondata/store/impl/LocalCarbonStore.java
index 40b6bdf..bd8fafe 100644
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/LocalCarbonStore.java
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/LocalCarbonStore.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.store.impl;
import java.io.IOException;
import java.util.Date;
+import java.util.Iterator;
import java.util.List;
import java.util.Objects;
@@ -28,11 +29,13 @@ import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.datatype.StructType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.processing.loading.DataLoadExecutor;
@@ -40,12 +43,17 @@ import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
import org.apache.carbondata.processing.loading.csvinput.CSVRecordReaderIterator;
import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
import org.apache.carbondata.processing.loading.model.CarbonLoadModelBuilder;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.descriptor.LoadDescriptor;
-import org.apache.carbondata.store.api.descriptor.SelectDescriptor;
-import org.apache.carbondata.store.api.exception.StoreException;
-import org.apache.carbondata.store.impl.rpc.model.Scan;
-import org.apache.carbondata.store.util.StoreUtil;
+import org.apache.carbondata.sdk.store.CarbonStore;
+import org.apache.carbondata.sdk.store.KeyedRow;
+import org.apache.carbondata.sdk.store.PrimaryKey;
+import org.apache.carbondata.sdk.store.Row;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.sdk.store.util.StoreUtil;
+import org.apache.carbondata.store.impl.service.model.ScanRequest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
@@ -63,7 +71,7 @@ import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
*/
@InterfaceAudience.User
@InterfaceStability.Unstable
-class LocalCarbonStore extends CarbonStoreBase {
+public class LocalCarbonStore extends MetaOperation implements CarbonStore {
private static final LogService LOGGER =
LogServiceFactory.getLogService(LocalCarbonStore.class.getName());
@@ -72,11 +80,11 @@ class LocalCarbonStore extends CarbonStoreBase {
private Configuration configuration;
private SegmentTxnManager txnManager;
- LocalCarbonStore(StoreConf storeConf) {
+ public LocalCarbonStore(StoreConf storeConf) {
this(storeConf, new Configuration());
}
- LocalCarbonStore(StoreConf storeConf, Configuration hadoopConf) {
+ private LocalCarbonStore(StoreConf storeConf, Configuration hadoopConf) {
super(storeConf);
this.storeConf = storeConf;
this.txnManager = SegmentTxnManager.getInstance();
@@ -84,17 +92,20 @@ class LocalCarbonStore extends CarbonStoreBase {
}
@Override
- public void loadData(LoadDescriptor load) throws IOException, StoreException {
+ public void loadData(LoadDescriptor load) throws CarbonException {
Objects.requireNonNull(load);
- CarbonTable table = metaProcessor.getTable(load.getTable());
- CarbonLoadModelBuilder modelBuilder = new CarbonLoadModelBuilder(table);
- modelBuilder.setInputPath(load.getInputPath());
CarbonLoadModel loadModel;
try {
+ TableInfo tableInfo = getTable(load.getTable(), storeConf);
+ CarbonTable table = CarbonTable.buildFromTableInfo(tableInfo);
+ CarbonLoadModelBuilder modelBuilder = new CarbonLoadModelBuilder(table);
+ modelBuilder.setInputPath(load.getInputPath());
loadModel = modelBuilder.build(load.getOptions(), System.currentTimeMillis(), "0");
} catch (InvalidLoadOptionException e) {
LOGGER.error(e, "Invalid loadDescriptor options");
- throw new StoreException(e.getMessage());
+ throw new CarbonException(e);
+ } catch (IOException e) {
+ throw new CarbonException(e);
}
if (loadModel.getFactTimeStamp() == 0) {
@@ -106,12 +117,27 @@ class LocalCarbonStore extends CarbonStoreBase {
loadData(loadModel);
txnManager.commitSegment(loadModel);
} catch (Exception e) {
- txnManager.closeSegment(loadModel);
LOGGER.error(e, "Failed to load data");
- throw new StoreException(e);
+ try {
+ txnManager.closeSegment(loadModel);
+ } catch (IOException ex) {
+ LOGGER.error(ex, "Failed to close segment");
+ // Ignoring the exception
+ }
+ throw new CarbonException(e);
}
}
+ @Override
+ public void upsert(Iterator<KeyedRow> row, StructType schema) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void delete(Iterator<PrimaryKey> keys) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
private void loadData(CarbonLoadModel model) throws Exception {
DataLoadExecutor executor = null;
try {
@@ -146,19 +172,34 @@ class LocalCarbonStore extends CarbonStoreBase {
}
@Override
- public List<CarbonRow> select(SelectDescriptor select) throws IOException {
- Objects.requireNonNull(select);
- CarbonTable table = metaProcessor.getTable(select.getTable());
- List<Distributable> blocks = pruneBlock(table, select.getProjection(), select.getFilter());
- CarbonMultiBlockSplit split = new CarbonMultiBlockSplit(blocks, "");
- Scan scan = new Scan(
- 0, split, table.getTableInfo(), select.getProjection(), select.getFilter(),
- select.getLimit());
- return scan(table, scan);
+ public List<CarbonRow> scan(ScanDescriptor scanDescriptor) throws CarbonException {
+ Objects.requireNonNull(scanDescriptor);
+ try {
+ TableInfo tableInfo = getTable(scanDescriptor.getTableIdentifier(), storeConf);
+ List<CarbonInputSplit> blocks =
+ IndexOperation.pruneBlock(tableInfo, scanDescriptor.getFilter());
+ CarbonMultiBlockSplit split = new CarbonMultiBlockSplit(blocks, new String[0]);
+ ScanRequest scan =
+ new ScanRequest(0, split, tableInfo, scanDescriptor.getProjection(),
+ scanDescriptor.getFilter(), scanDescriptor.getLimit());
+ return DataOperation.scan(tableInfo, scan);
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ }
+
+ @Override
+ public Row lookup(PrimaryKey key) throws CarbonException {
+ throw new UnsupportedOperationException();
}
@Override
- public void close() throws IOException {
+ public List<Row> lookup(TableIdentifier tableIdentifier, String filterExpression)
+ throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+ @Override
+ public void close() throws IOException {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/LocalDataScanner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/LocalDataScanner.java b/store/core/src/main/java/org/apache/carbondata/store/impl/LocalDataScanner.java
new file mode 100644
index 0000000..c3429a4
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/LocalDataScanner.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.DataScanner;
+import org.apache.carbondata.store.devapi.ResultBatch;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.impl.service.model.ScanRequest;
+
+/**
+ * This scanner scans in local JVM
+ * @param <T> scan output
+ */
+public class LocalDataScanner<T> implements DataScanner<T> {
+
+ private StoreConf storeConf;
+ private ScanDescriptor scanDescriptor;
+ private Map<String, String> scanOption;
+
+ LocalDataScanner(StoreConf storeConf, ScanDescriptor scanDescriptor,
+ Map<String, String> scanOption) {
+ this.storeConf = storeConf;
+ this.scanDescriptor = scanDescriptor;
+ this.scanOption = scanOption;
+ }
+
+ @Override
+ public Iterator<? extends ResultBatch<T>> scan(ScanUnit input) throws CarbonException {
+ Objects.requireNonNull(scanDescriptor);
+ try {
+ TableInfo tableInfo = MetaOperation.getTable(scanDescriptor.getTableIdentifier(), storeConf);
+ List<CarbonInputSplit> blocks =
+ IndexOperation.pruneBlock(tableInfo, scanDescriptor.getFilter());
+ CarbonMultiBlockSplit split = new CarbonMultiBlockSplit(blocks, new String[0]);
+ ScanRequest scan =
+ new ScanRequest(0, split, tableInfo, scanDescriptor.getProjection(),
+ scanDescriptor.getFilter(), scanDescriptor.getLimit());
+ List<T> rows = (List<T>) DataOperation.scan(tableInfo, scan);
+ RowMajorResultBatch<T> resultBatch = new RowMajorResultBatch<>(rows);
+ return Collections.singletonList(resultBatch).iterator();
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/LocalPruner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/LocalPruner.java b/store/core/src/main/java/org/apache/carbondata/store/impl/LocalPruner.java
new file mode 100644
index 0000000..734072c
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/LocalPruner.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.Pruner;
+import org.apache.carbondata.store.devapi.ScanUnit;
+
+public class LocalPruner implements Pruner {
+
+ private StoreConf storeConf;
+
+ LocalPruner(StoreConf storeConf) {
+ this.storeConf = storeConf;
+ }
+
+ @Override
+ public List<ScanUnit> prune(TableIdentifier identifier, Expression filterExpression)
+ throws CarbonException {
+ try {
+ TableInfo table = MetaOperation.getTable(identifier, storeConf);
+ List<CarbonInputSplit> splits = IndexOperation.pruneBlock(table, filterExpression);
+ return splits.stream().map(
+ (Function<CarbonInputSplit, ScanUnit>) inputSplit ->
+ // LocalScanner will scan in local JVM, it is not sending RPC to
+ // schedulable (Worker), so it can be null
+ new BlockScanUnit(inputSplit, null)
+ ).collect(Collectors.toList());
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/MetaOperation.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/MetaOperation.java b/store/core/src/main/java/org/apache/carbondata/store/impl/MetaOperation.java
new file mode 100644
index 0000000..3ca1ffe
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/MetaOperation.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.converter.SchemaConverter;
+import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
+import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.core.metadata.schema.table.TableSchema;
+import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.writer.ThriftWriter;
+import org.apache.carbondata.sdk.file.CarbonWriterBuilder;
+import org.apache.carbondata.sdk.file.Field;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+/**
+ * Provides table management.
+ */
+@InterfaceAudience.Internal
+public class MetaOperation {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(MetaOperation.class.getCanonicalName());
+
+ private StoreConf storeConf;
+
+ // mapping of table path to CarbonTable object
+ private static final Map<String, TableInfo> cache = new HashMap<>();
+
+ public MetaOperation(StoreConf storeConf) {
+ this.storeConf = storeConf;
+ }
+
+ public void createTable(TableDescriptor descriptor) throws CarbonException {
+ TableIdentifier table = descriptor.getTable();
+ Field[] fields = descriptor.getSchema().getFields();
+ // sort_columns
+ List<String> sortColumnsList = null;
+ try {
+ sortColumnsList = descriptor.getSchema().prepareSortColumns(descriptor.getProperties());
+ } catch (MalformedCarbonCommandException e) {
+ throw new CarbonException(e.getMessage());
+ }
+ ColumnSchema[] sortColumnsSchemaList = new ColumnSchema[sortColumnsList.size()];
+
+ TableSchemaBuilder builder = TableSchema.builder();
+ CarbonWriterBuilder.buildTableSchema(fields, builder, sortColumnsList, sortColumnsSchemaList);
+
+ TableSchema schema = builder.tableName(table.getTableName())
+ .properties(descriptor.getProperties())
+ .setSortColumns(Arrays.asList(sortColumnsSchemaList))
+ .build();
+
+ SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
+ schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis());
+ schema.getSchemaEvolution().getSchemaEvolutionEntryList().add(schemaEvolutionEntry);
+ schema.setTableName(table.getTableName());
+
+ String tablePath = descriptor.getTablePath();
+ if (tablePath == null) {
+ tablePath = getTablePath(table.getTableName(), table.getDatabaseName());
+ }
+
+ TableInfo tableInfo = CarbonTable.builder()
+ .databaseName(table.getDatabaseName())
+ .tableName(table.getTableName())
+ .tablePath(tablePath)
+ .tableSchema(schema)
+ .isTransactionalTable(true)
+ .buildTableInfo();
+
+ try {
+ createTable(tableInfo, descriptor.isIfNotExists());
+ } catch (IOException e) {
+ LOGGER.error(e, "create tableDescriptor failed");
+ throw new CarbonException(e.getMessage());
+ }
+ }
+
+ private void createTable(TableInfo tableInfo, boolean ifNotExists) throws IOException {
+ AbsoluteTableIdentifier identifier = tableInfo.getOrCreateAbsoluteTableIdentifier();
+ boolean tableExists = FileFactory.isFileExist(identifier.getTablePath());
+ if (tableExists) {
+ if (ifNotExists) {
+ return;
+ } else {
+ throw new IOException(
+ "car't create table " + tableInfo.getDatabaseName() + "." + tableInfo.getFactTable()
+ .getTableName() + ", because it already exists");
+ }
+ }
+
+ SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
+ String databaseName = tableInfo.getDatabaseName();
+ String tableName = tableInfo.getFactTable().getTableName();
+ org.apache.carbondata.format.TableInfo thriftTableInfo =
+ schemaConverter.fromWrapperToExternalTableInfo(tableInfo, databaseName, tableName);
+
+ String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
+ String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
+ FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
+ try {
+ if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
+ boolean isDirCreated = FileFactory.mkdirs(schemaMetadataPath, fileType);
+ if (!isDirCreated) {
+ throw new IOException("Failed to create the metadata directory " + schemaMetadataPath);
+ }
+ }
+ ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
+ thriftWriter.open(FileWriteOperation.OVERWRITE);
+ thriftWriter.write(thriftTableInfo);
+ thriftWriter.close();
+ } catch (IOException e) {
+ LOGGER.error(e, "Failed to handle create table");
+ throw e;
+ }
+ }
+
+ public void dropTable(TableIdentifier table) throws CarbonException {
+ String tablePath = getTablePath(table.getTableName(), table.getDatabaseName());
+ cache.remove(tablePath);
+ try {
+ FileFactory.deleteFile(tablePath);
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ }
+
+ public TableInfo getTable(TableIdentifier table) throws CarbonException {
+ return getTable(table, storeConf);
+ }
+
+ public static TableInfo getTable(TableIdentifier table, StoreConf storeConf)
+ throws CarbonException {
+ String tablePath = getTablePath(table.getTableName(), table.getDatabaseName(), storeConf);
+ if (cache.containsKey(tablePath)) {
+ return cache.get(tablePath);
+ } else {
+ org.apache.carbondata.format.TableInfo formatTableInfo = null;
+ try {
+ formatTableInfo = CarbonUtil.readSchemaFile(CarbonTablePath.getSchemaFilePath(tablePath));
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
+ TableInfo tableInfo = schemaConverter.fromExternalToWrapperTableInfo(
+ formatTableInfo, table.getDatabaseName(), table.getTableName(), tablePath);
+ tableInfo.setTablePath(tablePath);
+ cache.put(tablePath, tableInfo);
+ return tableInfo;
+ }
+ }
+
+ public List<TableDescriptor> listTable() throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ public TableDescriptor getDescriptor(TableIdentifier table) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ public void alterTable(TableIdentifier table, TableDescriptor newTable) throws CarbonException {
+ throw new UnsupportedOperationException();
+ }
+
+ public String getTablePath(String tableName, String databaseName) {
+ Objects.requireNonNull(tableName);
+ Objects.requireNonNull(databaseName);
+ return String.format("%s/%s", storeConf.storeLocation(), tableName);
+ }
+ public static String getTablePath(String tableName, String databaseName, StoreConf storeConf) {
+ Objects.requireNonNull(tableName);
+ Objects.requireNonNull(databaseName);
+ return String.format("%s/%s", storeConf.storeLocation(), tableName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/MetaProcessor.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/MetaProcessor.java b/store/core/src/main/java/org/apache/carbondata/store/impl/MetaProcessor.java
deleted file mode 100644
index 6d03711..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/MetaProcessor.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.fileoperations.FileWriteOperation;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.converter.SchemaConverter;
-import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
-import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.TableInfo;
-import org.apache.carbondata.core.metadata.schema.table.TableSchema;
-import org.apache.carbondata.core.metadata.schema.table.TableSchemaBuilder;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.core.writer.ThriftWriter;
-import org.apache.carbondata.sdk.file.CarbonWriterBuilder;
-import org.apache.carbondata.sdk.file.Field;
-import org.apache.carbondata.store.api.descriptor.TableDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
-import org.apache.carbondata.store.api.exception.StoreException;
-
-class MetaProcessor {
-
- private static LogService LOGGER =
- LogServiceFactory.getLogService(MetaProcessor.class.getCanonicalName());
-
- private CarbonStoreBase store;
-
- MetaProcessor(CarbonStoreBase store) {
- this.store = store;
- }
-
- // mapping of table path to CarbonTable object
- private Map<String, CarbonTable> cache = new HashMap<>();
-
- public void createTable(TableDescriptor descriptor) throws StoreException {
- Field[] fields = descriptor.getSchema().getFields();
- // sort_columns
- List<String> sortColumnsList = null;
- try {
- sortColumnsList = descriptor.getSchema().prepareSortColumns(descriptor.getProperties());
- } catch (MalformedCarbonCommandException e) {
- throw new StoreException(e.getMessage());
- }
- ColumnSchema[] sortColumnsSchemaList = new ColumnSchema[sortColumnsList.size()];
-
- TableSchemaBuilder builder = TableSchema.builder();
- CarbonWriterBuilder.buildTableSchema(fields, builder, sortColumnsList, sortColumnsSchemaList);
-
- TableSchema schema = builder.tableName(descriptor.getTable().getTableName())
- .properties(descriptor.getProperties())
- .setSortColumns(Arrays.asList(sortColumnsSchemaList))
- .build();
-
- SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
- schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis());
- schema.getSchemaEvolution().getSchemaEvolutionEntryList().add(schemaEvolutionEntry);
- schema.setTableName(descriptor.getTable().getTableName());
-
- String tablePath = descriptor.getTablePath();
- if (tablePath == null) {
- tablePath = store.getTablePath(
- descriptor.getTable().getTableName(), descriptor.getTable().getDatabaseName());
- }
-
- TableInfo tableInfo = CarbonTable.builder()
- .databaseName(descriptor.getTable().getDatabaseName())
- .tableName(descriptor.getTable().getTableName())
- .tablePath(tablePath)
- .tableSchema(schema)
- .isTransactionalTable(true)
- .buildTableInfo();
-
- try {
- createTable(tableInfo, descriptor.isIfNotExists());
- } catch (IOException e) {
- LOGGER.error(e, "create tableDescriptor failed");
- throw new StoreException(e.getMessage());
- }
- }
-
- private void createTable(TableInfo tableInfo, boolean ifNotExists) throws IOException {
- AbsoluteTableIdentifier identifier = tableInfo.getOrCreateAbsoluteTableIdentifier();
- boolean tableExists = FileFactory.isFileExist(identifier.getTablePath());
- if (tableExists) {
- if (ifNotExists) {
- return;
- } else {
- throw new IOException(
- "car't create table " + tableInfo.getDatabaseName() + "." + tableInfo.getFactTable()
- .getTableName() + ", because it already exists");
- }
- }
-
- SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
- String databaseName = tableInfo.getDatabaseName();
- String tableName = tableInfo.getFactTable().getTableName();
- org.apache.carbondata.format.TableInfo thriftTableInfo =
- schemaConverter.fromWrapperToExternalTableInfo(tableInfo, databaseName, tableName);
-
- String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
- String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
- FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
- try {
- if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
- boolean isDirCreated = FileFactory.mkdirs(schemaMetadataPath, fileType);
- if (!isDirCreated) {
- throw new IOException("Failed to create the metadata directory " + schemaMetadataPath);
- }
- }
- ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
- thriftWriter.open(FileWriteOperation.OVERWRITE);
- thriftWriter.write(thriftTableInfo);
- thriftWriter.close();
- } catch (IOException e) {
- LOGGER.error(e, "Failed to handle create table");
- throw e;
- }
- }
-
- public void dropTable(TableIdentifier table) throws IOException {
- String tablePath = store.getTablePath(table.getTableName(), table.getDatabaseName());
- cache.remove(tablePath);
- FileFactory.deleteFile(tablePath);
- }
-
- public CarbonTable getTable(TableIdentifier table) throws IOException {
- String tablePath = store.getTablePath(table.getTableName(), table.getDatabaseName());
- if (cache.containsKey(tablePath)) {
- return cache.get(tablePath);
- } else {
- org.apache.carbondata.format.TableInfo formatTableInfo =
- CarbonUtil.readSchemaFile(CarbonTablePath.getSchemaFilePath(tablePath));
- SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
- TableInfo tableInfo = schemaConverter.fromExternalToWrapperTableInfo(
- formatTableInfo, table.getDatabaseName(), table.getTableName(), tablePath);
- tableInfo.setTablePath(tablePath);
- CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
- cache.put(tablePath, carbonTable);
- return carbonTable;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/RemoteDataScanner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/RemoteDataScanner.java b/store/core/src/main/java/org/apache/carbondata/store/impl/RemoteDataScanner.java
new file mode 100644
index 0000000..8125426
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/RemoteDataScanner.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.stream.Collectors;
+
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.DataScanner;
+import org.apache.carbondata.store.devapi.ResultBatch;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.impl.service.DataService;
+import org.apache.carbondata.store.impl.service.model.ScanRequest;
+import org.apache.carbondata.store.impl.service.model.ScanResponse;
+
+/**
+ * This scanner scans in local JVM
+ * @param <T> scan output
+ */
+public class RemoteDataScanner<T> implements DataScanner<T> {
+
+ private TableInfo tableInfo;
+ private ScanDescriptor scanDescriptor;
+ private Map<String, String> scanOption;
+ private CarbonReadSupport<T> readSupport;
+
+ RemoteDataScanner(TableInfo tableInfo, ScanDescriptor scanDescriptor,
+ Map<String, String> scanOption, CarbonReadSupport<T> readSupport) {
+ this.tableInfo = tableInfo;
+ this.scanDescriptor = scanDescriptor;
+ this.scanOption = scanOption;
+ this.readSupport = readSupport;
+ }
+
+ @Override
+ public Iterator<? extends ResultBatch<T>> scan(ScanUnit input) throws CarbonException {
+ List<CarbonInputSplit> toBeScan = new ArrayList<>();
+ if (input instanceof BlockScanUnit) {
+ toBeScan.add(((BlockScanUnit) input).getInputSplit());
+ } else {
+ throw new CarbonException(input.getClass().getName() + " is not supported");
+ }
+ int queryId = new Random().nextInt();
+ CarbonMultiBlockSplit split = new CarbonMultiBlockSplit(toBeScan, input.preferredLocations());
+ try {
+ ScanRequest request = new ScanRequest(queryId, split, tableInfo,
+ scanDescriptor.getProjection(), scanDescriptor.getFilter(), scanDescriptor.getLimit());
+ DataService dataService =
+ DataServicePool.getOrCreateDataService(((BlockScanUnit) input).getSchedulable());
+ ScanResponse response = dataService.scan(request);
+ List<T> rows = Arrays.stream(response.getRows())
+ .map(row -> readSupport.readRow(row))
+ .collect(Collectors.toList());
+
+ return Collections.singletonList(new RowMajorResultBatch<>(rows)).iterator();
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/RemotePruner.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/RemotePruner.java b/store/core/src/main/java/org/apache/carbondata/store/impl/RemotePruner.java
new file mode 100644
index 0000000..5fc485e
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/RemotePruner.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.Pruner;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.impl.service.PruneService;
+import org.apache.carbondata.store.impl.service.ServiceFactory;
+import org.apache.carbondata.store.impl.service.model.PruneRequest;
+import org.apache.carbondata.store.impl.service.model.PruneResponse;
+
+public class RemotePruner implements Pruner {
+
+ private String pruneServiceHost;
+ private int pruneServiePort;
+
+ RemotePruner(String pruneServiceHost, int pruneServiePort) {
+ this.pruneServiceHost = pruneServiceHost;
+ this.pruneServiePort = pruneServiePort;
+ }
+
+ @Override
+ public List<ScanUnit> prune(TableIdentifier table, Expression filterExpression)
+ throws CarbonException {
+ try {
+ PruneRequest request = new PruneRequest(table, filterExpression);
+ PruneService pruneService = ServiceFactory.createPruneService(
+ pruneServiceHost, pruneServiePort);
+ PruneResponse response = pruneService.prune(request);
+ return response.getScanUnits();
+ } catch (IOException e) {
+ throw new CarbonException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/RowMajorResultBatch.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/RowMajorResultBatch.java b/store/core/src/main/java/org/apache/carbondata/store/impl/RowMajorResultBatch.java
new file mode 100644
index 0000000..1999a63
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/RowMajorResultBatch.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.carbondata.store.devapi.ResultBatch;
+
+public class RowMajorResultBatch<T> implements ResultBatch<T> {
+
+ private Iterator<T> iterator;
+
+ RowMajorResultBatch(List<T> rows) {
+ Objects.requireNonNull(rows);
+ this.iterator = rows.iterator();
+ }
+
+ @Override
+ public boolean isColumnar() {
+ return false;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return iterator.hasNext();
+ }
+
+ @Override
+ public T next() {
+ return iterator.next();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/Schedulable.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/Schedulable.java b/store/core/src/main/java/org/apache/carbondata/store/impl/Schedulable.java
new file mode 100644
index 0000000..6e29b48
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/Schedulable.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class Schedulable implements Writable {
+
+ private String id;
+ private String address;
+ private int port;
+ private int cores;
+ public AtomicInteger workload;
+
+ public Schedulable() {
+ }
+
+ public Schedulable(String id, String address, int port, int cores) {
+ this.id = id;
+ this.address = address;
+ this.port = port;
+ this.cores = cores;
+ this.workload = new AtomicInteger();
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public String getAddress() {
+ return address;
+ }
+
+ public void setAddress(String address) {
+ this.address = address;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public void setPort(int port) {
+ this.port = port;
+ }
+
+ public int getCores() {
+ return cores;
+ }
+
+ @Override public String toString() {
+ return "Schedulable{" + "id='" + id + '\'' + ", address='" + address + '\'' + ", port=" + port
+ + '}';
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(id);
+ out.writeUTF(address);
+ out.writeInt(port);
+ out.writeInt(cores);
+ // We are not writing workload since it is only useful for
+ // Scheduler inside the Master. Client of the Master does
+ // not need it
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ id = in.readUTF();
+ address = in.readUTF();
+ port = in.readInt();
+ cores = in.readInt();
+ }
+}
[2/5] carbondata git commit: [CARBONDATA-2825][CARBONDATA-2828]
CarbonStore and InternalCarbonStore API This closes #2589
Posted by qi...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownResponse.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownResponse.java b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownResponse.java
new file mode 100644
index 0000000..2650bbf
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/service/model/ShutdownResponse.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.service.model;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+
+import org.apache.hadoop.io.Writable;
+
+@InterfaceAudience.Internal
+public class ShutdownResponse implements Serializable, Writable {
+ private int status;
+ private String message;
+
+ public ShutdownResponse() {
+ }
+
+ public ShutdownResponse(int status, String message) {
+ this.status = status;
+ this.message = message;
+ }
+
+ public int getStatus() {
+ return status;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(status);
+ out.writeUTF(message);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ status = in.readInt();
+ message = in.readUTF();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/DataServiceImpl.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/DataServiceImpl.java b/store/core/src/main/java/org/apache/carbondata/store/impl/worker/DataServiceImpl.java
new file mode 100644
index 0000000..d1b8f43
--- /dev/null
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/worker/DataServiceImpl.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.store.impl.worker;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.scan.executor.impl.SearchModeDetailQueryExecutor;
+import org.apache.carbondata.core.scan.executor.impl.SearchModeVectorDetailQueryExecutor;
+import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
+import org.apache.carbondata.processing.loading.DataLoadExecutor;
+import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
+import org.apache.carbondata.processing.loading.csvinput.CSVRecordReaderIterator;
+import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
+import org.apache.carbondata.sdk.store.util.StoreUtil;
+import org.apache.carbondata.store.impl.DataOperation;
+import org.apache.carbondata.store.impl.Status;
+import org.apache.carbondata.store.impl.service.DataService;
+import org.apache.carbondata.store.impl.service.model.BaseResponse;
+import org.apache.carbondata.store.impl.service.model.LoadDataRequest;
+import org.apache.carbondata.store.impl.service.model.ScanRequest;
+import org.apache.carbondata.store.impl.service.model.ScanResponse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+
+@InterfaceAudience.Internal
+public class DataServiceImpl implements DataService {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DataServiceImpl.class.getCanonicalName());
+
+ // temp location for loading (writing sort temp files)
+ private String[] storeTempLocation;
+ private Configuration hadoopConf;
+
+ DataServiceImpl(Worker worker) {
+ this.hadoopConf = worker.getHadoopConf();
+ this.storeTempLocation = worker.getConf().storeTempLocation();
+ }
+
+ @Override
+ public BaseResponse loadData(LoadDataRequest request) {
+ DataLoadExecutor executor = null;
+ try {
+ CarbonLoadModel model = request.getModel();
+
+ JobID jobId = CarbonInputFormatUtil.getJobId(new Date(), 0);
+ CarbonInputFormatUtil.createJobTrackerID(new Date());
+ TaskID taskId = new TaskID(jobId, TaskType.MAP, 0);
+ TaskAttemptID taskAttemptId = new TaskAttemptID(taskId, 0);
+ Configuration configuration = new Configuration(hadoopConf);
+ StoreUtil.configureCSVInputFormat(configuration, model);
+ configuration.set(FileInputFormat.INPUT_DIR, model.getFactFilePath());
+ // Set up the attempt context required to use in the output committer.
+ TaskAttemptContext hadoopAttemptContext =
+ new TaskAttemptContextImpl(configuration, taskAttemptId);
+
+ CSVInputFormat format = new CSVInputFormat();
+ List<InputSplit> splits = format.getSplits(hadoopAttemptContext);
+
+ CarbonIterator<Object[]>[] readerIterators = new CSVRecordReaderIterator[splits.size()];
+ for (int index = 0; index < splits.size(); index++) {
+ readerIterators[index] = new CSVRecordReaderIterator(
+ format.createRecordReader(splits.get(index), hadoopAttemptContext), splits.get(index),
+ hadoopAttemptContext);
+ }
+
+ executor = new DataLoadExecutor();
+ executor.execute(model, storeTempLocation, readerIterators);
+
+ return new BaseResponse(Status.SUCCESS.ordinal(), "");
+ } catch (IOException e) {
+ LOGGER.error(e, "Failed to handle load data");
+ return new BaseResponse(Status.FAILURE.ordinal(), e.getMessage());
+ } catch (InterruptedException e) {
+ LOGGER.error(e, "Interrupted handle load data ");
+ return new BaseResponse(Status.FAILURE.ordinal(), e.getMessage());
+ } catch (Exception e) {
+ LOGGER.error(e, "Failed to execute load data ");
+ return new BaseResponse(Status.FAILURE.ordinal(), e.getMessage());
+ } finally {
+ if (executor != null) {
+ executor.close();
+ StoreUtil.clearUnsafeMemory(ThreadLocalTaskInfo.getCarbonTaskInfo().getTaskId());
+ }
+ }
+ }
+
+ @Override
+ public ScanResponse scan(ScanRequest scan) {
+ try {
+ LOGGER.info(String.format("[QueryId:%d] receive search request", scan.getRequestId()));
+ List<CarbonRow> rows = DataOperation.scan(scan.getTableInfo(), scan);
+ LOGGER.info(String.format("[QueryId:%d] sending success response", scan.getRequestId()));
+ return createSuccessResponse(scan, rows);
+ } catch (IOException e) {
+ LOGGER.error(e);
+ LOGGER.info(String.format("[QueryId:%d] sending failure response", scan.getRequestId()));
+ return createFailureResponse(scan, e);
+ }
+ }
+
+ /**
+ * create a failure response
+ */
+ private ScanResponse createFailureResponse(ScanRequest scan, Throwable throwable) {
+ return new ScanResponse(scan.getRequestId(), Status.FAILURE.ordinal(),
+ throwable.getMessage(), new Object[0][]);
+ }
+
+ /**
+ * create a success response with result rows
+ */
+ private ScanResponse createSuccessResponse(ScanRequest scan, List<CarbonRow> rows) {
+ Iterator<CarbonRow> itor = rows.iterator();
+ Object[][] output = new Object[rows.size()][];
+ int i = 0;
+ while (itor.hasNext()) {
+ output[i++] = itor.next().getData();
+ }
+ return new ScanResponse(scan.getRequestId(), Status.SUCCESS.ordinal(), "", output);
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
+ return versionID;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
+ int clientMethodsHash) throws IOException {
+ return null;
+ }
+
+ @Override
+ public void close() throws IOException {
+ LOGGER.info("Shutting down worker...");
+ SearchModeDetailQueryExecutor.shutdownThreadPool();
+ SearchModeVectorDetailQueryExecutor.shutdownThreadPool();
+ LOGGER.info("Worker shut down");
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/RequestHandler.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/RequestHandler.java b/store/core/src/main/java/org/apache/carbondata/store/impl/worker/RequestHandler.java
deleted file mode 100644
index fd13b20..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/RequestHandler.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.worker;
-
-import java.io.IOException;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.row.CarbonRow;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.scan.executor.impl.SearchModeDetailQueryExecutor;
-import org.apache.carbondata.core.scan.executor.impl.SearchModeVectorDetailQueryExecutor;
-import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
-import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
-import org.apache.carbondata.processing.loading.DataLoadExecutor;
-import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
-import org.apache.carbondata.processing.loading.csvinput.CSVRecordReaderIterator;
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.impl.CarbonStoreBase;
-import org.apache.carbondata.store.impl.Status;
-import org.apache.carbondata.store.impl.rpc.model.BaseResponse;
-import org.apache.carbondata.store.impl.rpc.model.LoadDataRequest;
-import org.apache.carbondata.store.impl.rpc.model.QueryResponse;
-import org.apache.carbondata.store.impl.rpc.model.Scan;
-import org.apache.carbondata.store.impl.rpc.model.ShutdownRequest;
-import org.apache.carbondata.store.impl.rpc.model.ShutdownResponse;
-import org.apache.carbondata.store.util.StoreUtil;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.JobID;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskID;
-import org.apache.hadoop.mapreduce.TaskType;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
-
-/**
- * It handles request from master.
- */
-@InterfaceAudience.Internal
-class RequestHandler {
-
- private StoreConf storeConf;
- private Configuration hadoopConf;
-
- RequestHandler(StoreConf conf, Configuration hadoopConf) {
- this.storeConf = conf;
- this.hadoopConf = hadoopConf;
- }
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(RequestHandler.class.getName());
-
- QueryResponse handleScan(Scan scan) {
- try {
- LOGGER.info(String.format("[QueryId:%d] receive search request", scan.getRequestId()));
- CarbonTable table = CarbonTable.buildFromTableInfo(scan.getTableInfo());
- List<CarbonRow> rows = CarbonStoreBase.scan(table, scan);
- LOGGER.info(String.format("[QueryId:%d] sending success response", scan.getRequestId()));
- return createSuccessResponse(scan, rows);
- } catch (IOException e) {
- LOGGER.error(e);
- LOGGER.info(String.format("[QueryId:%d] sending failure response", scan.getRequestId()));
- return createFailureResponse(scan, e);
- }
- }
-
- ShutdownResponse handleShutdown(ShutdownRequest request) {
- LOGGER.info("Shutting down worker...");
- SearchModeDetailQueryExecutor.shutdownThreadPool();
- SearchModeVectorDetailQueryExecutor.shutdownThreadPool();
- LOGGER.info("Worker shut down");
- return new ShutdownResponse(Status.SUCCESS.ordinal(), "");
- }
-
- /**
- * create a failure response
- */
- private QueryResponse createFailureResponse(Scan scan, Throwable throwable) {
- return new QueryResponse(scan.getRequestId(), Status.FAILURE.ordinal(),
- throwable.getMessage(), new Object[0][]);
- }
-
- /**
- * create a success response with result rows
- */
- private QueryResponse createSuccessResponse(Scan scan, List<CarbonRow> rows) {
- Iterator<CarbonRow> itor = rows.iterator();
- Object[][] output = new Object[rows.size()][];
- int i = 0;
- while (itor.hasNext()) {
- output[i++] = itor.next().getData();
- }
- return new QueryResponse(scan.getRequestId(), Status.SUCCESS.ordinal(), "", output);
- }
-
- public BaseResponse handleLoadData(LoadDataRequest request) {
- DataLoadExecutor executor = null;
- try {
- CarbonLoadModel model = request.getModel();
-
- JobID jobId = CarbonInputFormatUtil.getJobId(new Date(), 0);
- CarbonInputFormatUtil.createJobTrackerID(new Date());
- TaskID taskId = new TaskID(jobId, TaskType.MAP, 0);
- TaskAttemptID taskAttemptId = new TaskAttemptID(taskId, 0);
- Configuration configuration = new Configuration(hadoopConf);
- StoreUtil.configureCSVInputFormat(configuration, model);
- configuration.set(FileInputFormat.INPUT_DIR, model.getFactFilePath());
- // Set up the attempt context required to use in the output committer.
- TaskAttemptContext hadoopAttemptContext =
- new TaskAttemptContextImpl(configuration, taskAttemptId);
-
- CSVInputFormat format = new CSVInputFormat();
- List<InputSplit> splits = format.getSplits(hadoopAttemptContext);
-
- CarbonIterator<Object[]>[] readerIterators = new CSVRecordReaderIterator[splits.size()];
- for (int index = 0; index < splits.size(); index++) {
- readerIterators[index] = new CSVRecordReaderIterator(
- format.createRecordReader(splits.get(index), hadoopAttemptContext), splits.get(index),
- hadoopAttemptContext);
- }
-
- executor = new DataLoadExecutor();
- executor.execute(model, storeConf.storeTempLocation(), readerIterators);
-
- return new BaseResponse(Status.SUCCESS.ordinal(), "");
- } catch (IOException e) {
- LOGGER.error(e, "Failed to handle load data");
- return new BaseResponse(Status.FAILURE.ordinal(), e.getMessage());
- } catch (InterruptedException e) {
- LOGGER.error(e, "Interrupted handle load data ");
- return new BaseResponse(Status.FAILURE.ordinal(), e.getMessage());
- } catch (Exception e) {
- LOGGER.error(e, "Failed to execute load data ");
- return new BaseResponse(Status.FAILURE.ordinal(), e.getMessage());
- } finally {
- if (executor != null) {
- executor.close();
- StoreUtil.clearUnsafeMemory(ThreadLocalTaskInfo.getCarbonTaskInfo().getTaskId());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/StoreServiceImpl.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/StoreServiceImpl.java b/store/core/src/main/java/org/apache/carbondata/store/impl/worker/StoreServiceImpl.java
deleted file mode 100644
index 26f252c..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/StoreServiceImpl.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.impl.worker;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.store.impl.rpc.StoreService;
-import org.apache.carbondata.store.impl.rpc.model.BaseResponse;
-import org.apache.carbondata.store.impl.rpc.model.LoadDataRequest;
-import org.apache.carbondata.store.impl.rpc.model.QueryResponse;
-import org.apache.carbondata.store.impl.rpc.model.Scan;
-import org.apache.carbondata.store.impl.rpc.model.ShutdownRequest;
-import org.apache.carbondata.store.impl.rpc.model.ShutdownResponse;
-
-import org.apache.hadoop.ipc.ProtocolSignature;
-
-@InterfaceAudience.Internal
-public class StoreServiceImpl implements StoreService {
-
- private Worker worker;
- RequestHandler handler;
-
- public StoreServiceImpl(Worker worker) {
- this.worker = worker;
- this.handler = new RequestHandler(worker.getConf(), worker.getHadoopConf());
- }
-
- @Override
- public BaseResponse loadData(LoadDataRequest request) {
- return handler.handleLoadData(request);
- }
-
- @Override
- public QueryResponse query(Scan scan) {
- return handler.handleScan(scan);
- }
-
- @Override
- public ShutdownResponse shutdown(ShutdownRequest request) {
- return handler.handleShutdown(request);
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion) throws IOException {
- return versionID;
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol, long clientVersion,
- int clientMethodsHash) throws IOException {
- return null;
- }
-
- public Worker getWorker() {
- return worker;
- }
-
- public void setWorker(Worker worker) {
- this.worker = worker;
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/Worker.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/Worker.java b/store/core/src/main/java/org/apache/carbondata/store/impl/worker/Worker.java
index a360e36..e3546d9 100644
--- a/store/core/src/main/java/org/apache/carbondata/store/impl/worker/Worker.java
+++ b/store/core/src/main/java/org/apache/carbondata/store/impl/worker/Worker.java
@@ -23,13 +23,13 @@ import java.net.BindException;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.impl.rpc.RegistryService;
-import org.apache.carbondata.store.impl.rpc.ServiceFactory;
-import org.apache.carbondata.store.impl.rpc.StoreService;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerRequest;
-import org.apache.carbondata.store.impl.rpc.model.RegisterWorkerResponse;
-import org.apache.carbondata.store.util.StoreUtil;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.util.StoreUtil;
+import org.apache.carbondata.store.impl.service.DataService;
+import org.apache.carbondata.store.impl.service.RegistryService;
+import org.apache.carbondata.store.impl.service.ServiceFactory;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerRequest;
+import org.apache.carbondata.store.impl.service.model.RegisterWorkerResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
@@ -66,15 +66,15 @@ public class Worker {
int coreNum = conf.workerCoreNum();
String host = conf.workerHost();
int port = conf.workerPort();
- StoreService queryService = new StoreServiceImpl(this);
+ DataService dataService = new DataServiceImpl(this);
do {
try {
server = new RPC.Builder(hadoopConf)
.setNumHandlers(coreNum)
.setBindAddress(host)
.setPort(port)
- .setProtocol(StoreService.class)
- .setInstance(queryService)
+ .setProtocol(DataService.class)
+ .setInstance(dataService)
.build();
server.start();
@@ -116,9 +116,11 @@ public class Worker {
}
private void registerToMaster() throws IOException {
- LOGGER.info("trying to register to master " + conf.masterHost() + ":" + conf.masterPort());
+ LOGGER.info("trying to register to master " +
+ conf.masterHost() + ":" + conf.registryServicePort());
if (registry == null) {
- registry = ServiceFactory.createRegistryService(conf.masterHost(), conf.masterPort());
+ registry = ServiceFactory.createRegistryService(
+ conf.masterHost(), conf.registryServicePort());
}
RegisterWorkerRequest request =
new RegisterWorkerRequest(conf.workerHost(), conf.workerPort(), conf.workerCoreNum());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/main/java/org/apache/carbondata/store/util/StoreUtil.java
----------------------------------------------------------------------
diff --git a/store/core/src/main/java/org/apache/carbondata/store/util/StoreUtil.java b/store/core/src/main/java/org/apache/carbondata/store/util/StoreUtil.java
deleted file mode 100644
index 775669f..0000000
--- a/store/core/src/main/java/org/apache/carbondata/store/util/StoreUtil.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.store.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.memory.UnsafeMemoryManager;
-import org.apache.carbondata.core.memory.UnsafeSortMemoryManager;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.processing.loading.csvinput.CSVInputFormat;
-import org.apache.carbondata.processing.loading.model.CarbonLoadModel;
-import org.apache.carbondata.store.api.conf.StoreConf;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.log4j.BasicConfigurator;
-import org.apache.log4j.PropertyConfigurator;
-
-@InterfaceAudience.Internal
-public class StoreUtil {
-
- private static LogService LOGGER = LogServiceFactory.getLogService(StoreUtil.class.getName());
-
- public static void loadProperties(String filePath, StoreConf conf) {
- InputStream input = null;
- try {
- input = new FileInputStream(filePath);
- Properties prop = new Properties();
- prop.load(input);
- for (Map.Entry<Object, Object> entry : prop.entrySet()) {
- conf.conf(entry.getKey().toString(), entry.getValue().toString());
- }
- LOGGER.audit("loaded properties: " + filePath);
- } catch (IOException ex) {
- LOGGER.error(ex, "Failed to load properties file " + filePath);
- } finally {
- if (input != null) {
- try {
- input.close();
- } catch (IOException e) {
- LOGGER.error(e);
- }
- }
- }
- }
-
- public static void initLog4j(String propertiesFilePath) {
- BasicConfigurator.configure();
- PropertyConfigurator.configure(propertiesFilePath);
- }
-
- public static byte[] serialize(Object object) {
- ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
- try {
- ObjectOutputStream oos = new ObjectOutputStream(baos);
- oos.writeObject(object);
- } catch (IOException e) {
- LOGGER.error(e);
- }
- return baos.toByteArray();
- }
-
- public static Object deserialize(byte[] bytes) {
- if (bytes == null) {
- return null;
- }
- try {
- ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes));
- return ois.readObject();
- } catch (IOException e) {
- LOGGER.error(e);
- } catch (ClassNotFoundException e) {
- LOGGER.error(e);
- }
- return null;
- }
-
- public static void configureCSVInputFormat(Configuration configuration,
- CarbonLoadModel carbonLoadModel) {
- CSVInputFormat.setCommentCharacter(configuration, carbonLoadModel.getCommentChar());
- CSVInputFormat.setCSVDelimiter(configuration, carbonLoadModel.getCsvDelimiter());
- CSVInputFormat.setSkipEmptyLine(configuration, carbonLoadModel.getSkipEmptyLine());
- CSVInputFormat.setEscapeCharacter(configuration, carbonLoadModel.getEscapeChar());
- CSVInputFormat.setMaxColumns(configuration, carbonLoadModel.getMaxColumns());
- CSVInputFormat.setNumberOfColumns(configuration,
- "" + carbonLoadModel.getCsvHeaderColumns().length);
-
- CSVInputFormat.setHeaderExtractionEnabled(
- configuration,
- carbonLoadModel.getCsvHeader() == null ||
- StringUtils.isEmpty(carbonLoadModel.getCsvHeader()));
-
- CSVInputFormat.setQuoteCharacter(configuration, carbonLoadModel.getQuoteChar());
-
- CSVInputFormat.setReadBufferSize(
- configuration,
- CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CSV_READ_BUFFER_SIZE,
- CarbonCommonConstants.CSV_READ_BUFFER_SIZE_DEFAULT));
- }
-
- public static void clearUnsafeMemory(long taskId) {
- UnsafeMemoryManager.INSTANCE.freeMemoryAll(taskId);
- UnsafeSortMemoryManager.INSTANCE.freeMemoryAll(taskId);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/test/java/org/apache/carbondata/store/DistributedCarbonStoreTest.java
----------------------------------------------------------------------
diff --git a/store/core/src/test/java/org/apache/carbondata/store/DistributedCarbonStoreTest.java b/store/core/src/test/java/org/apache/carbondata/store/DistributedCarbonStoreTest.java
index 2448660..7f80a33 100644
--- a/store/core/src/test/java/org/apache/carbondata/store/DistributedCarbonStoreTest.java
+++ b/store/core/src/test/java/org/apache/carbondata/store/DistributedCarbonStoreTest.java
@@ -27,14 +27,15 @@ import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
-import org.apache.carbondata.store.api.CarbonStore;
-import org.apache.carbondata.store.api.CarbonStoreFactory;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.descriptor.LoadDescriptor;
-import org.apache.carbondata.store.api.descriptor.SelectDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.sdk.store.CarbonStore;
+import org.apache.carbondata.sdk.store.CarbonStoreFactory;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.store.impl.master.Master;
import org.apache.carbondata.store.impl.worker.Worker;
import org.junit.After;
@@ -50,24 +51,36 @@ public class DistributedCarbonStoreTest {
private static CarbonStore store;
@BeforeClass
- public static void beforeAll() throws IOException, StoreException {
+ public static void beforeAll() throws IOException, CarbonException, InterruptedException {
projectFolder = new File(DistributedCarbonStoreTest.class.getResource("/").getPath() +
"../../../../").getCanonicalPath();
+
String confFile = projectFolder + "/store/conf/store.conf";
StoreConf storeConf = new StoreConf(confFile);
- store = CarbonStoreFactory.getDistributedStore("DistributedCarbonStoreTest", storeConf);
- projectFolder = new File(LocalCarbonStoreTest.class.getResource("/").getPath() + "../../../../")
- .getCanonicalPath();
+ new Thread(() -> {
+ try {
+ Master.main(new String[]{"", confFile});
+ } catch (InterruptedException | IOException e) {
+ throw new RuntimeException("failed to start master");
+ }
+ }).start();
+ Thread.sleep(1000);
// start worker
Worker worker = new Worker(storeConf);
worker.start();
+
+ Thread.sleep(1000);
+
+ store = CarbonStoreFactory.getDistributedStore("DistributedCarbonStoreTest", storeConf);
}
@AfterClass
public static void afterAll() throws IOException {
- store.close();
+ if (store != null) {
+ store.close();
+ }
}
@Before
@@ -81,13 +94,13 @@ public class DistributedCarbonStoreTest {
}
@Test
- public void testSelect() throws IOException, StoreException {
+ public void testSelect() throws CarbonException {
TableIdentifier tableIdentifier = new TableIdentifier("table_1", "default");
store.dropTable(tableIdentifier);
- TableDescriptor table = TableDescriptor
+ TableDescriptor descriptor = TableDescriptor
.builder()
- .ifNotExists()
.table(tableIdentifier)
+ .ifNotExists()
.comment("first table")
.column("shortField", DataTypes.SHORT, "short field")
.column("intField", DataTypes.INT, "int field")
@@ -101,7 +114,7 @@ public class DistributedCarbonStoreTest {
.column("floatField", DataTypes.DOUBLE, "float field")
.tblProperties(CarbonCommonConstants.SORT_COLUMNS, "intField")
.create();
- store.createTable(table);
+ store.createTable(descriptor);
// load one segment
LoadDescriptor load = LoadDescriptor
@@ -114,26 +127,26 @@ public class DistributedCarbonStoreTest {
store.loadData(load);
// select row
- SelectDescriptor select = SelectDescriptor
+ ScanDescriptor select = ScanDescriptor
.builder()
.table(tableIdentifier)
- .select("intField", "stringField")
+ .select(new String[]{"intField", "stringField"})
.limit(5)
.create();
- List<CarbonRow> result = store.select(select);
+ List<CarbonRow> result = store.scan(select);
Assert.assertEquals(5, result.size());
// select row with filter
- SelectDescriptor select2 = SelectDescriptor
+ ScanDescriptor select2 = ScanDescriptor
.builder()
.table(tableIdentifier)
- .select("intField", "stringField")
+ .select(new String[]{"intField", "stringField"})
.filter(new EqualToExpression(
new ColumnExpression("intField", DataTypes.INT),
new LiteralExpression(11, DataTypes.INT)))
.limit(5)
.create();
- List<CarbonRow> result2 = store.select(select2);
+ List<CarbonRow> result2 = store.scan(select2);
Assert.assertEquals(1, result2.size());
store.dropTable(tableIdentifier);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
----------------------------------------------------------------------
diff --git a/store/core/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java b/store/core/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
index 420c8cf..8cf7e14 100644
--- a/store/core/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
+++ b/store/core/src/test/java/org/apache/carbondata/store/LocalCarbonStoreTest.java
@@ -22,20 +22,19 @@ import java.io.IOException;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.row.CarbonRow;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
-import org.apache.carbondata.store.api.CarbonStore;
-import org.apache.carbondata.store.api.CarbonStoreFactory;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.descriptor.LoadDescriptor;
-import org.apache.carbondata.store.api.descriptor.SelectDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.sdk.store.CarbonStore;
+import org.apache.carbondata.sdk.store.CarbonStoreFactory;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
import org.junit.After;
import org.junit.AfterClass;
@@ -50,7 +49,7 @@ public class LocalCarbonStoreTest {
private static CarbonStore store;
@BeforeClass
- public static void setup() throws IOException, StoreException {
+ public static void setup() throws IOException, CarbonException {
StoreConf conf = new StoreConf("test", "./");
conf.conf(StoreConf.STORE_TEMP_LOCATION, "./temp");
store = CarbonStoreFactory.getLocalStore("LocalCarbonStoreTest", conf);
@@ -74,13 +73,13 @@ public class LocalCarbonStoreTest {
}
@Test
- public void testWriteAndReadFiles() throws IOException, StoreException {
+ public void testWriteAndReadFiles() throws IOException, CarbonException {
TableIdentifier tableIdentifier = new TableIdentifier("table_1", "default");
store.dropTable(tableIdentifier);
- TableDescriptor table = TableDescriptor
+ TableDescriptor descriptor = TableDescriptor
.builder()
- .ifNotExists()
.table(tableIdentifier)
+ .ifNotExists()
.comment("first table")
.column("shortField", DataTypes.SHORT, "short field")
.column("intField", DataTypes.INT, "int field")
@@ -94,7 +93,7 @@ public class LocalCarbonStoreTest {
.column("floatField", DataTypes.DOUBLE, "float field")
.tblProperties(CarbonCommonConstants.SORT_COLUMNS, "intField")
.create();
- store.createTable(table);
+ store.createTable(descriptor);
// load one segment
LoadDescriptor load = LoadDescriptor
@@ -107,26 +106,26 @@ public class LocalCarbonStoreTest {
store.loadData(load);
// select row
- SelectDescriptor select = SelectDescriptor
+ ScanDescriptor select = ScanDescriptor
.builder()
.table(tableIdentifier)
- .select("intField", "stringField")
+ .select(new String[]{"intField", "stringField"})
.limit(5)
.create();
- List<CarbonRow> result = store.select(select);
+ List<CarbonRow> result = store.scan(select);
Assert.assertEquals(5, result.size());
// select row with filter
- SelectDescriptor select2 = SelectDescriptor
+ ScanDescriptor select2 = ScanDescriptor
.builder()
.table(tableIdentifier)
- .select("intField", "stringField")
+ .select(new String[]{"intField", "stringField"})
.filter(new EqualToExpression(
new ColumnExpression("intField", DataTypes.INT),
new LiteralExpression(11, DataTypes.INT)))
.limit(5)
.create();
- List<CarbonRow> result2 = store.select(select2);
+ List<CarbonRow> result2 = store.scan(select2);
Assert.assertEquals(1, result2.size());
store.dropTable(tableIdentifier);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/core/src/test/java/org/apache/carbondata/store/TestUtil.java
----------------------------------------------------------------------
diff --git a/store/core/src/test/java/org/apache/carbondata/store/TestUtil.java b/store/core/src/test/java/org/apache/carbondata/store/TestUtil.java
index f73591c..dffc8a7 100644
--- a/store/core/src/test/java/org/apache/carbondata/store/TestUtil.java
+++ b/store/core/src/test/java/org/apache/carbondata/store/TestUtil.java
@@ -18,21 +18,11 @@
package org.apache.carbondata.store;
import java.io.File;
-import java.io.FileFilter;
import java.io.IOException;
-import org.apache.carbondata.common.exceptions.sql.InvalidLoadOptionException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.path.CarbonTablePath;
-import org.apache.carbondata.sdk.file.CarbonWriter;
-import org.apache.carbondata.sdk.file.CarbonWriterBuilder;
-import org.apache.carbondata.sdk.file.Schema;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.util.StoreUtil;
-
-import org.junit.Assert;
public class TestUtil {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/HorizonClient.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/HorizonClient.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/HorizonClient.java
index eaa4583..e1b039e 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/HorizonClient.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/HorizonClient.java
@@ -28,7 +28,7 @@ import org.apache.carbondata.horizon.rest.model.view.CreateTableRequest;
import org.apache.carbondata.horizon.rest.model.view.DropTableRequest;
import org.apache.carbondata.horizon.rest.model.view.LoadRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectRequest;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
/**
* Client to send REST request to Horizon service
@@ -40,9 +40,9 @@ public interface HorizonClient extends Closeable {
/**
* Create a Table
* @param create descriptor for create table operation
- * @throws IOException if network or disk IO error occurs
+ * @throws CarbonException if network or disk IO error occurs
*/
- void createTable(CreateTableRequest create) throws IOException, StoreException;
+ void createTable(CreateTableRequest create) throws CarbonException;
/**
* Drop a Table, and remove all data in it
@@ -56,7 +56,7 @@ public interface HorizonClient extends Closeable {
* @param load descriptor for load operation
* @throws IOException if network or disk IO error occurs
*/
- void loadData(LoadRequest load) throws IOException, StoreException;
+ void loadData(LoadRequest load) throws IOException, CarbonException;
/**
* Scan a Table and return matched rows
@@ -64,7 +64,7 @@ public interface HorizonClient extends Closeable {
* @return matched rows
* @throws IOException if network or disk IO error occurs
*/
- List<CarbonRow> select(SelectRequest select) throws IOException, StoreException;
+ List<CarbonRow> select(SelectRequest select) throws IOException, CarbonException;
/**
* Executor a SQL statement
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/impl/SimpleHorizonClient.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/impl/SimpleHorizonClient.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/impl/SimpleHorizonClient.java
index 076df70..b24c8d2 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/impl/SimpleHorizonClient.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/client/impl/SimpleHorizonClient.java
@@ -31,7 +31,7 @@ import org.apache.carbondata.horizon.rest.model.view.DropTableRequest;
import org.apache.carbondata.horizon.rest.model.view.LoadRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectResponse;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
@@ -52,7 +52,7 @@ public class SimpleHorizonClient implements HorizonClient {
}
@Override
- public void createTable(CreateTableRequest create) throws IOException, StoreException {
+ public void createTable(CreateTableRequest create) throws CarbonException {
Objects.requireNonNull(create);
restTemplate.postForEntity(serviceUri + "/table/create", create, String.class);
}
@@ -64,18 +64,18 @@ public class SimpleHorizonClient implements HorizonClient {
}
@Override
- public void loadData(LoadRequest load) throws IOException, StoreException {
+ public void loadData(LoadRequest load) throws IOException, CarbonException {
Objects.requireNonNull(load);
restTemplate.postForEntity(serviceUri + "/table/load", load, String.class);
}
@Override
- public List<CarbonRow> select(SelectRequest select) throws IOException, StoreException {
+ public List<CarbonRow> select(SelectRequest select) throws IOException, CarbonException {
Objects.requireNonNull(select);
ResponseEntity<SelectResponse> response =
restTemplate.postForEntity(serviceUri + "/table/select", select, SelectResponse.class);
- Object[][] rows = Objects.requireNonNull(response.getBody()).getRows();
- List<CarbonRow> output = new ArrayList<>(rows.length);
+ List<Object[]> rows = Objects.requireNonNull(response.getBody()).getRows();
+ List<CarbonRow> output = new ArrayList<>(rows.size());
for (Object[] row : rows) {
output.add(new CarbonRow(row));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/Horizon.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/Horizon.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/Horizon.java
index a30b587..cffca07 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/Horizon.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/Horizon.java
@@ -17,7 +17,8 @@
package org.apache.carbondata.horizon.rest.controller;
-import org.apache.carbondata.store.api.conf.StoreConf;
+
+import org.apache.carbondata.sdk.store.conf.StoreConf;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/HorizonController.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/HorizonController.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/HorizonController.java
index a273f54..d3d1bd7 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/HorizonController.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/controller/HorizonController.java
@@ -18,7 +18,8 @@ package org.apache.carbondata.horizon.rest.controller;
import java.io.IOException;
import java.net.InetAddress;
-import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -36,14 +37,17 @@ import org.apache.carbondata.horizon.rest.model.view.DropTableRequest;
import org.apache.carbondata.horizon.rest.model.view.LoadRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectResponse;
-import org.apache.carbondata.store.api.CarbonStore;
-import org.apache.carbondata.store.api.CarbonStoreFactory;
-import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.descriptor.LoadDescriptor;
-import org.apache.carbondata.store.api.descriptor.SelectDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.sdk.store.conf.StoreConf;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+import org.apache.carbondata.store.devapi.InternalCarbonStore;
+import org.apache.carbondata.store.devapi.InternalCarbonStoreFactory;
+import org.apache.carbondata.store.devapi.ResultBatch;
+import org.apache.carbondata.store.devapi.ScanUnit;
+import org.apache.carbondata.store.devapi.Scanner;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
@@ -59,15 +63,15 @@ public class HorizonController {
private static LogService LOGGER =
LogServiceFactory.getLogService(HorizonController.class.getName());
- private CarbonStore store;
+ private InternalCarbonStore store;
- public HorizonController() throws StoreException {
+ public HorizonController() throws CarbonException {
String storeFile = System.getProperty("carbonstore.conf.file");
StoreConf storeConf = new StoreConf();
try {
storeConf.conf(StoreConf.STORE_LOCATION, CarbonProperties.getStorePath())
.conf(StoreConf.MASTER_HOST, InetAddress.getLocalHost().getHostAddress())
- .conf(StoreConf.MASTER_PORT, CarbonProperties.getSearchMasterPort())
+ .conf(StoreConf.STORE_PORT, CarbonProperties.getSearchMasterPort())
.conf(StoreConf.WORKER_HOST, InetAddress.getLocalHost().getHostAddress())
.conf(StoreConf.WORKER_PORT, CarbonProperties.getSearchWorkerPort())
.conf(StoreConf.WORKER_CORE_NUM, 2);
@@ -76,13 +80,10 @@ public class HorizonController {
storeConf.load(storeFile);
}
- } catch (UnknownHostException e) {
- throw new StoreException(e);
+ store = InternalCarbonStoreFactory.getStore(storeConf);
} catch (IOException e) {
- throw new StoreException(e);
+ throw new CarbonException(e);
}
-
- store = CarbonStoreFactory.getDistributedStore("GlobalStore", storeConf);
}
@RequestMapping(value = "echo")
@@ -92,7 +93,7 @@ public class HorizonController {
@RequestMapping(value = "/table/create", produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<String> createTable(
- @RequestBody CreateTableRequest request) throws StoreException, IOException {
+ @RequestBody CreateTableRequest request) throws CarbonException {
RequestValidator.validateTable(request);
TableDescriptor tableDescriptor = request.convertToDto();
store.createTable(tableDescriptor);
@@ -101,7 +102,7 @@ public class HorizonController {
@RequestMapping(value = "/table/drop", produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<String> dropTable(
- @RequestBody DropTableRequest request) throws StoreException, IOException {
+ @RequestBody DropTableRequest request) throws CarbonException {
RequestValidator.validateDrop(request);
store.dropTable(new TableIdentifier(request.getTableName(), request.getDatabaseName()));
return new ResponseEntity<>(String.valueOf(true), HttpStatus.OK);
@@ -109,7 +110,7 @@ public class HorizonController {
@RequestMapping(value = "/table/load", produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<String> load(@RequestBody LoadRequest request)
- throws StoreException, IOException {
+ throws CarbonException, IOException {
RequestValidator.validateLoad(request);
LoadDescriptor loadDescriptor = request.convertToDto();
store.loadData(loadDescriptor);
@@ -118,22 +119,29 @@ public class HorizonController {
@RequestMapping(value = "/table/select", produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<SelectResponse> select(@RequestBody SelectRequest request)
- throws StoreException, IOException {
+ throws CarbonException {
long start = System.currentTimeMillis();
RequestValidator.validateSelect(request);
TableIdentifier table = new TableIdentifier(request.getTableName(), request.getDatabaseName());
- CarbonTable carbonTable = store.getTable(table);
+ CarbonTable carbonTable = store.getCarbonTable(table);
Expression expression = Parser.parseFilter(request.getFilter(), carbonTable);
- SelectDescriptor selectDescriptor = new SelectDescriptor(
+ Scanner<CarbonRow> scanner = store.newScanner(table);
+ List<ScanUnit> scanUnits = scanner.prune(table, expression);
+ ScanDescriptor scanDescriptor = new ScanDescriptor(
table, request.getSelect(), expression, request.getLimit());
- List<CarbonRow> result = store.select(selectDescriptor);
- Iterator<CarbonRow> iterator = result.iterator();
- Object[][] output = new Object[result.size()][];
- int i = 0;
- while (iterator.hasNext()) {
- output[i] = (iterator.next().getData());
- i++;
+ ArrayList<Object[]> output = new ArrayList<>();
+ for (ScanUnit scanUnit : scanUnits) {
+ Iterator<? extends ResultBatch<CarbonRow>> iterator = scanner.scan(
+ scanUnit, scanDescriptor, new HashMap<String, String>());
+
+ while (iterator.hasNext()) {
+ ResultBatch<CarbonRow> rows = iterator.next();
+ while (rows.hasNext()) {
+ output.add(rows.next().getData());
+ }
+ }
}
+
long end = System.currentTimeMillis();
LOGGER.audit("[" + request.getRequestId() + "] HorizonController select " +
request.getDatabaseName() + "." + request.getTableName() +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
index fbba57b..15a30cb 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/validate/RequestValidator.java
@@ -21,63 +21,63 @@ import org.apache.carbondata.horizon.rest.model.view.CreateTableRequest;
import org.apache.carbondata.horizon.rest.model.view.DropTableRequest;
import org.apache.carbondata.horizon.rest.model.view.LoadRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectRequest;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
import org.apache.commons.lang.StringUtils;
public class RequestValidator {
- public static void validateSelect(SelectRequest request) throws StoreException {
+ public static void validateSelect(SelectRequest request) throws CarbonException {
if (request == null) {
- throw new StoreException("Select should not be null");
+ throw new CarbonException("Select should not be null");
}
if (StringUtils.isEmpty(request.getDatabaseName())) {
- throw new StoreException("database name is invalid");
+ throw new CarbonException("database name is invalid");
}
if (StringUtils.isEmpty(request.getTableName())) {
- throw new StoreException("table name is invalid");
+ throw new CarbonException("table name is invalid");
}
}
- public static void validateTable(CreateTableRequest request) throws StoreException {
+ public static void validateTable(CreateTableRequest request) throws CarbonException {
if (request == null) {
- throw new StoreException("TableDescriptor should not be null");
+ throw new CarbonException("TableDescriptor should not be null");
}
if (StringUtils.isEmpty(request.getDatabaseName())) {
- throw new StoreException("database name is invalid");
+ throw new CarbonException("database name is invalid");
}
if (StringUtils.isEmpty(request.getTableName())) {
- throw new StoreException("table name is invalid");
+ throw new CarbonException("table name is invalid");
}
if (request.getFields() == null || request.getFields().length == 0) {
- throw new StoreException("fields should not be empty");
+ throw new CarbonException("fields should not be empty");
}
}
- public static void validateLoad(LoadRequest request) throws StoreException {
+ public static void validateLoad(LoadRequest request) throws CarbonException {
if (request == null) {
- throw new StoreException("LoadDescriptor should not be null");
+ throw new CarbonException("LoadDescriptor should not be null");
}
if (StringUtils.isEmpty(request.getDatabaseName())) {
- throw new StoreException("database name is invalid");
+ throw new CarbonException("database name is invalid");
}
if (StringUtils.isEmpty(request.getTableName())) {
- throw new StoreException("table name is invalid");
+ throw new CarbonException("table name is invalid");
}
if (StringUtils.isEmpty(request.getInputPath())) {
- throw new StoreException("input path is invalid");
+ throw new CarbonException("input path is invalid");
}
}
- public static void validateDrop(DropTableRequest request) throws StoreException {
+ public static void validateDrop(DropTableRequest request) throws CarbonException {
if (request == null) {
- throw new StoreException("DropTableRequest should not be null");
+ throw new CarbonException("DropTableRequest should not be null");
}
if (StringUtils.isEmpty(request.getDatabaseName())) {
- throw new StoreException("database name is invalid");
+ throw new CarbonException("database name is invalid");
}
if (StringUtils.isEmpty(request.getTableName())) {
- throw new StoreException("table name is invalid");
+ throw new CarbonException("table name is invalid");
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java
index cf59f7f..623abee 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/CreateTableRequest.java
@@ -24,8 +24,8 @@ import java.util.Map;
import org.apache.carbondata.sdk.file.Field;
import org.apache.carbondata.sdk.file.Schema;
-import org.apache.carbondata.store.api.descriptor.TableDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
public class CreateTableRequest extends Request {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/FieldRequest.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/FieldRequest.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/FieldRequest.java
index b809d9e..200b3a2 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/FieldRequest.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/FieldRequest.java
@@ -108,7 +108,7 @@ public class FieldRequest {
field.setPrecision(precision);
field.setScale(scale);
field.setColumnComment(comment);
- field.setChildren(new LinkedList<StructField>());
+ field.setChildren(new LinkedList<Field>());
return field;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java
index c91f5f5..dfe21f6 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/LoadRequest.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.horizon.rest.model.view;
import java.util.HashMap;
import java.util.Map;
-import org.apache.carbondata.store.api.descriptor.LoadDescriptor;
-import org.apache.carbondata.store.api.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
public class LoadRequest extends Request {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java
index 6bf5c75..e4f8cf8 100644
--- a/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java
+++ b/store/horizon/src/main/java/org/apache/carbondata/horizon/rest/model/view/SelectResponse.java
@@ -17,23 +17,22 @@
package org.apache.carbondata.horizon.rest.model.view;
+import java.util.List;
+
public class SelectResponse extends Response {
- private Object[][] rows;
+ private List<Object[]> rows;
public SelectResponse() {
}
- public SelectResponse(SelectRequest request, String message, Object[][] rows) {
+ public SelectResponse(SelectRequest request, String message, List<Object[]> rows) {
super(request, message);
this.rows = rows;
}
- public Object[][] getRows() {
+ public List<Object[]> getRows() {
return rows;
}
- public void setRows(Object[][] rows) {
- this.rows = rows;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java
----------------------------------------------------------------------
diff --git a/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java b/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java
index 91a9dba..59c4bd1 100644
--- a/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java
+++ b/store/horizon/src/test/java/org/apache/carbondata/horizon/HorizonTest.java
@@ -32,9 +32,9 @@ import org.apache.carbondata.horizon.rest.model.view.LoadRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectRequest;
import org.apache.carbondata.horizon.rest.model.view.SelectResponse;
import org.apache.carbondata.store.api.conf.StoreConf;
-import org.apache.carbondata.store.api.exception.StoreException;
+import org.apache.carbondata.store.api.exception.CarbonException;
import org.apache.carbondata.store.impl.worker.Worker;
-import org.apache.carbondata.store.util.StoreUtil;
+import org.apache.carbondata.sdk.store.util.StoreUtil;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -109,13 +109,13 @@ public class HorizonTest {
SelectRequest select = createSelectRequest(5, null, "intField", "stringField");
SelectResponse result =
restTemplate.postForObject(serviceUri + "/table/select", select, SelectResponse.class);
- Assert.assertEquals(5, result.getRows().length);
+ Assert.assertEquals(5, result.getRows().size());
// select row with filter
SelectRequest filter = createSelectRequest(5, "intField = 11", "intField", "stringField");
SelectResponse filterResult =
restTemplate.postForObject(serviceUri + "/table/select", filter, SelectResponse.class);
- Assert.assertEquals(1, filterResult.getRows().length);
+ Assert.assertEquals(1, filterResult.getRows().size());
request = createDropTableRequest();
response = restTemplate.postForObject(serviceUri + "/table/drop", request, String.class);
@@ -173,7 +173,7 @@ public class HorizonTest {
}
@Test
- public void testHorizonClient() throws IOException, StoreException {
+ public void testHorizonClient() throws IOException, CarbonException {
HorizonClient client = new SimpleHorizonClient(serviceUri);
DropTableRequest drop = createDropTableRequest();
client.dropTable(drop);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/pom.xml
----------------------------------------------------------------------
diff --git a/store/sdk/pom.xml b/store/sdk/pom.xml
index aecf7e2..b16a290 100644
--- a/store/sdk/pom.xml
+++ b/store/sdk/pom.xml
@@ -51,8 +51,8 @@
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <source>1.7</source>
- <target>1.7</target>
+ <source>1.8</source>
+ <target>1.8</target>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
index fdd1f5a..771896b 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/AvroCarbonWriter.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.UUID;
+import java.util.stream.Collectors;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.logging.LogService;
@@ -30,6 +31,7 @@ import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDi
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.datatype.StructField;
+import org.apache.carbondata.core.metadata.datatype.StructType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.hadoop.api.CarbonTableOutputFormat;
import org.apache.carbondata.hadoop.internal.ObjectArrayWritable;
@@ -239,9 +241,9 @@ public class AvroCarbonWriter extends CarbonWriter {
return new Field(FieldName, DataTypes.DOUBLE);
case RECORD:
// recursively get the sub fields
- ArrayList<StructField> structSubFields = new ArrayList<>();
+ ArrayList<Field> structSubFields = new ArrayList<>();
for (Schema.Field avroSubField : childSchema.getFields()) {
- StructField structField = prepareSubFields(avroSubField.name(), avroSubField.schema());
+ Field structField = prepareSubFields(avroSubField.name(), avroSubField.schema());
if (structField != null) {
structSubFields.add(structField);
}
@@ -249,9 +251,9 @@ public class AvroCarbonWriter extends CarbonWriter {
return new Field(FieldName, "struct", structSubFields);
case ARRAY:
// recursively get the sub fields
- ArrayList<StructField> arraySubField = new ArrayList<>();
+ ArrayList<Field> arraySubField = new ArrayList<>();
// array will have only one sub field.
- StructField structField = prepareSubFields("val", childSchema.getElementType());
+ Field structField = prepareSubFields("val", childSchema.getElementType());
if (structField != null) {
arraySubField.add(structField);
return new Field(FieldName, "array", arraySubField);
@@ -266,51 +268,51 @@ public class AvroCarbonWriter extends CarbonWriter {
}
}
- private static StructField prepareSubFields(String FieldName, Schema childSchema) {
+ private static Field prepareSubFields(String FieldName, Schema childSchema) {
Schema.Type type = childSchema.getType();
LogicalType logicalType = childSchema.getLogicalType();
switch (type) {
case BOOLEAN:
- return new StructField(FieldName, DataTypes.BOOLEAN);
+ return new Field(FieldName, DataTypes.BOOLEAN);
case INT:
if (logicalType instanceof LogicalTypes.Date) {
- return new StructField(FieldName, DataTypes.DATE);
+ return new Field(FieldName, DataTypes.DATE);
} else {
LOGGER.warn("Unsupported logical type. Considering Data Type as INT for " + childSchema
.getName());
- return new StructField(FieldName, DataTypes.INT);
+ return new Field(FieldName, DataTypes.INT);
}
case LONG:
if (logicalType instanceof LogicalTypes.TimestampMillis
|| logicalType instanceof LogicalTypes.TimestampMicros) {
- return new StructField(FieldName, DataTypes.TIMESTAMP);
+ return new Field(FieldName, DataTypes.TIMESTAMP);
} else {
LOGGER.warn("Unsupported logical type. Considering Data Type as LONG for " + childSchema
.getName());
- return new StructField(FieldName, DataTypes.LONG);
+ return new Field(FieldName, DataTypes.LONG);
}
case DOUBLE:
- return new StructField(FieldName, DataTypes.DOUBLE);
+ return new Field(FieldName, DataTypes.DOUBLE);
case STRING:
- return new StructField(FieldName, DataTypes.STRING);
+ return new Field(FieldName, DataTypes.STRING);
case FLOAT:
- return new StructField(FieldName, DataTypes.DOUBLE);
+ return new Field(FieldName, DataTypes.DOUBLE);
case RECORD:
// recursively get the sub fields
- ArrayList<StructField> structSubFields = new ArrayList<>();
+ ArrayList<Field> structSubFields = new ArrayList<>();
for (Schema.Field avroSubField : childSchema.getFields()) {
- StructField structField = prepareSubFields(avroSubField.name(), avroSubField.schema());
+ Field structField = prepareSubFields(avroSubField.name(), avroSubField.schema());
if (structField != null) {
structSubFields.add(structField);
}
}
- return (new StructField(FieldName, DataTypes.createStructType(structSubFields)));
+ return (new Field(FieldName, createStructType(structSubFields)));
case ARRAY:
// recursively get the sub fields
// array will have only one sub field.
DataType subType = getMappingDataTypeForArrayRecord(childSchema.getElementType());
if (subType != null) {
- return (new StructField(FieldName, DataTypes.createArrayType(subType)));
+ return (new Field(FieldName, DataTypes.createArrayType(subType)));
} else {
return null;
}
@@ -322,6 +324,14 @@ public class AvroCarbonWriter extends CarbonWriter {
}
}
+ private static StructType createStructType(List<Field> fields) {
+ List<StructField> f = fields.stream().map(field ->
+ new StructField(field.getFieldName(), field.getDataType(),
+ createStructType(field.getChildren()).getFields())
+ ).collect(Collectors.toList());
+ return DataTypes.createStructType(f);
+ }
+
private static DataType getMappingDataTypeForArrayRecord(Schema childSchema) {
LogicalType logicalType = childSchema.getLogicalType();
switch (childSchema.getType()) {
@@ -360,14 +370,14 @@ public class AvroCarbonWriter extends CarbonWriter {
return DataTypes.DOUBLE;
case RECORD:
// recursively get the sub fields
- ArrayList<StructField> structSubFields = new ArrayList<>();
+ ArrayList<Field> structSubFields = new ArrayList<>();
for (Schema.Field avroSubField : childSchema.getFields()) {
- StructField structField = prepareSubFields(avroSubField.name(), avroSubField.schema());
+ Field structField = prepareSubFields(avroSubField.name(), avroSubField.schema());
if (structField != null) {
structSubFields.add(structField);
}
}
- return DataTypes.createStructType(structSubFields);
+ return createStructType(structSubFields);
case ARRAY:
// array will have only one sub field.
DataType subType = getMappingDataTypeForArrayRecord(childSchema.getElementType());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
index a9d725f..92ed0d8 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/CarbonWriterBuilder.java
@@ -513,8 +513,8 @@ public class CarbonWriterBuilder {
} else if (field.getDataType().getName().equalsIgnoreCase("STRUCT")) {
// Loop through the inner columns and for a StructData
List<StructField> structFieldsArray =
- new ArrayList<StructField>(field.getChildren().size());
- for (StructField childFld : field.getChildren()) {
+ new ArrayList<>(field.getChildren().size());
+ for (Field childFld : field.getChildren()) {
structFieldsArray
.add(new StructField(childFld.getFieldName(), childFld.getDataType()));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
index add10c1..0d70c3b 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Field.java
@@ -17,13 +17,13 @@
package org.apache.carbondata.sdk.file;
+import java.io.Serializable;
import java.util.LinkedList;
import java.util.List;
import org.apache.carbondata.common.annotations.InterfaceAudience;
import org.apache.carbondata.common.annotations.InterfaceStability;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.StructField;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -32,11 +32,11 @@ import org.apache.carbondata.core.util.DataTypeUtil;
*/
@InterfaceAudience.User
@InterfaceStability.Unstable
-public class Field {
+public class Field implements Serializable {
private String name;
private DataType type;
- private List<StructField> children;
+ private List<Field> children;
private String parent;
private String storeType = "columnnar";
private int schemaOrdinal = -1;
@@ -54,11 +54,11 @@ public class Field {
this(name, DataTypeUtil.valueOf(type));
}
- public Field(String name, String type, List<StructField> fields) {
+ public Field(String name, String type, List<Field> fields) {
this(name, DataTypeUtil.valueOf(type), fields);
}
- public Field(String name, DataType type, List<StructField> fields) {
+ public Field(String name, DataType type, List<Field> fields) {
this.name = name;
this.type = type;
this.children = fields;
@@ -91,11 +91,11 @@ public class Field {
return type;
}
- public List<StructField> getChildren() {
+ public List<Field> getChildren() {
return children;
}
- public void setChildren(List<StructField> children) {
+ public void setChildren(List<Field> children) {
this.children = children;
}
@@ -150,4 +150,5 @@ public class Field {
public void setColumnComment(String columnComment) {
this.columnComment = columnComment;
}
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
index c9622e1..075ae71 100644
--- a/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/file/Schema.java
@@ -18,6 +18,7 @@
package org.apache.carbondata.sdk.file;
import java.io.IOException;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -44,7 +45,7 @@ import org.apache.commons.lang.StringUtils;
*/
@InterfaceAudience.User
@InterfaceStability.Unstable
-public class Schema {
+public class Schema implements Serializable {
private Field[] fields;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/9f10122a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStore.java
----------------------------------------------------------------------
diff --git a/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStore.java b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStore.java
new file mode 100644
index 0000000..0472b75
--- /dev/null
+++ b/store/sdk/src/main/java/org/apache/carbondata/sdk/store/CarbonStore.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.sdk.store;
+
+import java.io.Closeable;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.carbondata.common.annotations.InterfaceAudience;
+import org.apache.carbondata.common.annotations.InterfaceStability;
+import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.datatype.StructType;
+import org.apache.carbondata.sdk.store.descriptor.LoadDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.ScanDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableDescriptor;
+import org.apache.carbondata.sdk.store.descriptor.TableIdentifier;
+import org.apache.carbondata.sdk.store.exception.CarbonException;
+
+/**
+ * Public Interface of CarbonStore
+ */
+@InterfaceAudience.User
+@InterfaceStability.Unstable
+public interface CarbonStore extends Closeable {
+
+ ////////////////////////////////////////////////////////////////////
+ ///// Metadata Operation /////
+ ////////////////////////////////////////////////////////////////////
+
+ /**
+ * Create a Table
+ * @param descriptor descriptor for create table operation
+ * @throws CarbonException if any error occurs
+ */
+ void createTable(TableDescriptor descriptor) throws CarbonException;
+
+ /**
+ * Drop a Table, and remove all data in it
+ * @param table table identifier
+ * @throws CarbonException if any error occurs
+ */
+ void dropTable(TableIdentifier table) throws CarbonException;
+
+ /**
+ * @return all table created
+ * @throws CarbonException if any error occurs
+ */
+ List<TableDescriptor> listTable() throws CarbonException;
+
+ /**
+ * Return table descriptor by specified identifier
+ * @param table table identifier
+ * @return table descriptor
+ * @throws CarbonException if any error occurs
+ */
+ TableDescriptor getDescriptor(TableIdentifier table) throws CarbonException;
+
+ /**
+ * Alter table operation
+ * @param table table identifier
+ * @param newTable new table descriptor to alter to
+ * @throws CarbonException if any error occurs
+ */
+ void alterTable(TableIdentifier table, TableDescriptor newTable) throws CarbonException;
+
+
+ ////////////////////////////////////////////////////////////////////
+ ///// Write Operation /////
+ ////////////////////////////////////////////////////////////////////
+
+ /**
+ * Trigger a Load into the table specified by load descriptor
+ * @param load descriptor for load operation
+ * @throws CarbonException if any error occurs
+ */
+ void loadData(LoadDescriptor load) throws CarbonException;
+
+ /**
+ * Return true if this table has primary key defined when create table using
+ * {@link #createTable(TableDescriptor)}
+ *
+ * For such table, upsert, delete and lookup is supported
+ *
+ * @return true if this table has primary key.
+ */
+ default boolean isPrimaryKeyDefined(TableIdentifier identifier) {
+ return false;
+ }
+
+ /**
+ * Insert a batch of rows if key is not exist, otherwise update the row
+ * @param row rows to be upsert
+ * @param schema schema of the input row (fields without the primary key)
+ * @throws CarbonException if any error occurs
+ */
+ void upsert(Iterator<KeyedRow> row, StructType schema) throws CarbonException;
+
+ /**
+ * Delete a batch of rows
+ * @param keys keys to be deleted
+ * @throws CarbonException if any error occurs
+ */
+ void delete(Iterator<PrimaryKey> keys) throws CarbonException;
+
+
+ ////////////////////////////////////////////////////////////////////
+ ///// Read Operation /////
+ ////////////////////////////////////////////////////////////////////
+
+ /**
+ * Scan the specified table and return matched rows
+ *
+ * @param select descriptor for scan operation
+ * @return matched rows
+ * @throws CarbonException if any error occurs
+ */
+ List<CarbonRow> scan(ScanDescriptor select) throws CarbonException;
+
+ /**
+ * Lookup and return a row with specified primary key
+ * @param key key to lookup
+ * @return matched row for the specified key
+ * @throws CarbonException if any error occurs
+ */
+ Row lookup(PrimaryKey key) throws CarbonException;
+
+ /**
+ * Lookup by filter expression and return a list of matched row
+ *
+ * @param tableIdentifier table identifier
+ * @param filterExpression filter expression, like "col3 = 1"
+ * @return matched row for the specified filter
+ * @throws CarbonException if any error occurs
+ */
+ List<Row> lookup(TableIdentifier tableIdentifier, String filterExpression) throws CarbonException;
+}