You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/11/04 18:35:45 UTC
[1/2] incubator-carbondata git commit: add interface
Repository: incubator-carbondata
Updated Branches:
refs/heads/master ac62b58d7 -> c910e6fd8
add interface
modify name
add comment
add format type
fix comment
fix comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/edb02ab5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/edb02ab5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/edb02ab5
Branch: refs/heads/master
Commit: edb02ab504dc775eefab992d16ae1f5338a614f4
Parents: ac62b58
Author: jackylk <ja...@huawei.com>
Authored: Thu Oct 27 18:39:39 2016 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Nov 5 00:04:21 2016 +0530
----------------------------------------------------------------------
.../carbondata/hadoop/CarbonInputSplit.java | 18 +-
.../hadoop/api/CarbonTableInputFormat.java | 170 +++++++++++++++
.../hadoop/api/CarbonTableOutputCommitter.java | 56 +++++
.../hadoop/api/CarbonTableOutputFormat.java | 41 ++++
.../hadoop/internal/CarbonFormatType.java | 24 +++
.../hadoop/internal/CarbonInputSplit.java | 33 +++
.../carbondata/hadoop/internal/index/Block.java | 44 ++++
.../carbondata/hadoop/internal/index/Index.java | 48 +++++
.../hadoop/internal/index/IndexLoader.java | 36 ++++
.../internal/index/impl/InMemoryBTreeIndex.java | 214 +++++++++++++++++++
.../index/impl/InMemoryBTreeIndexLoader.java | 31 +++
.../hadoop/internal/segment/Segment.java | 94 ++++++++
.../hadoop/internal/segment/SegmentManager.java | 56 +++++
.../internal/segment/SegmentManagerFactory.java | 30 +++
.../internal/segment/impl/IndexedSegment.java | 73 +++++++
.../segment/impl/zk/ZkSegmentManager.java | 56 +++++
16 files changed, 1023 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 01744f8..2cdefe9 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -22,7 +22,9 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
+import java.util.List;
+import org.apache.carbondata.hadoop.internal.index.Block;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
@@ -30,7 +32,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
/**
* Carbon input split to allow distributed read of CarbonInputFormat.
*/
-public class CarbonInputSplit extends FileSplit implements Serializable, Writable {
+public class CarbonInputSplit extends FileSplit implements Serializable, Writable, Block {
private static final long serialVersionUID = 3520344046772190207L;
private String segmentId;
@@ -84,4 +86,18 @@ public class CarbonInputSplit extends FileSplit implements Serializable, Writabl
return numberOfBlocklets;
}
+ @Override
+ public String getBlockPath() {
+ return getPath().getName();
+ }
+
+ @Override
+ public List<Long> getMatchedBlocklets() {
+ return null;
+ }
+
+ @Override
+ public boolean fullScan() {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
new file mode 100644
index 0000000..233a5b7
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.api;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.carbondata.hadoop.CarbonProjection;
+import org.apache.carbondata.hadoop.internal.CarbonInputSplit;
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManagerFactory;
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
+import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+
+/**
+ * Input format of CarbonData file.
+ * @param <T>
+ */
+public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
+
+ private static final String FILTER_PREDICATE =
+ "mapreduce.input.carboninputformat.filter.predicate";
+
+ private SegmentManager segmentManager;
+
+ public CarbonTableInputFormat() {
+ this.segmentManager = SegmentManagerFactory.getGlobalSegmentManager();
+ }
+
+ @Override
+ public RecordReader<Void, T> createRecordReader(InputSplit split,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ switch (((CarbonInputSplit)split).formatType()) {
+ case COLUMNR:
+ // TODO: create record reader for columnar format
+ break;
+ default:
+ throw new RuntimeException("Unsupported format type");
+ }
+ return null;
+ }
+
+ @Override
+ public List<InputSplit> getSplits(JobContext job) throws IOException {
+
+ // work as following steps:
+ // get all current valid segment
+ // for each segment, get all input split
+
+ List<InputSplit> output = new LinkedList<>();
+ Expression filter = getFilter(job.getConfiguration());
+ Segment[] segments = segmentManager.getAllValidSegments();
+ FilterResolverIntf filterResolver = CarbonInputFormatUtil.resolveFilter(filter, null);
+ for (Segment segment: segments) {
+ List<InputSplit> splits = segment.getSplits(job, filterResolver);
+ output.addAll(splits);
+ }
+ return output;
+ }
+
+ /**
+ * set the table path into configuration
+ * @param conf configuration of the job
+ * @param tablePath table path string
+ */
+ public void setTablePath(Configuration conf, String tablePath) {
+
+ }
+
+ /**
+ * return the table path in the configuration
+ * @param conf configuration of the job
+ * @return table path string
+ */
+ public String getTablePath(Configuration conf) {
+ return null;
+ }
+
+ /**
+ * set projection columns into configuration
+ * @param conf configuration of the job
+ * @param projection projection
+ */
+ public void setProjection(Configuration conf, CarbonProjection projection) {
+
+ }
+
+ /**
+ * return the projection in the configuration
+ * @param conf configuration of the job
+ * @return projection
+ */
+ public CarbonProjection getProjection(Configuration conf) {
+ return null;
+ }
+
+ /**
+ * set filter expression into the configuration
+ * @param conf configuration of the job
+ * @param filter filter expression
+ */
+ public void setFilter(Configuration conf, Expression filter) {
+ try {
+ String filterString = ObjectSerializationUtil.convertObjectToString(filter);
+ conf.set(FILTER_PREDICATE, filterString);
+ } catch (Exception e) {
+ throw new RuntimeException("Error while setting filter expression to Job", e);
+ }
+ }
+
+ /**
+ * return filter expression in the configuration
+ * @param conf configuration of the job
+ * @return filter expression
+ */
+ public Expression getFilter(Configuration conf) {
+ Object filter;
+ String filterExprString = conf.get(FILTER_PREDICATE);
+ if (filterExprString == null) {
+ return null;
+ }
+ try {
+ filter = ObjectSerializationUtil.convertStringToObject(filterExprString);
+ } catch (IOException e) {
+ throw new RuntimeException("Error while reading filter expression", e);
+ }
+ assert(filter instanceof Expression);
+ return (Expression) filter;
+ }
+
+ /**
+ * Optional API. It can be used by query optimizer to select index based on filter
+ * in the configuration of the job. After selecting index internally, index' name will be set
+ * in the configuration.
+ *
+ * The process of selection is simple, just use the default index. Subclass can provide a more
+ * advanced selection logic like cost based.
+ * @param conf job configuration
+ */
+ public void selectIndex(Configuration conf) {
+ // set the default index in configuration
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java
new file mode 100644
index 0000000..3b4a3a2
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.api;
+
+import java.io.IOException;
+
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.JobContext;
+
+public class CarbonTableOutputCommitter extends FileOutputCommitter {
+ private SegmentManager segmentManager;
+ private Segment newSegment;
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ // steps:
+ // call segment manager to open new segment for loading
+ // load data synchronously
+ // close the segment and make it available for reading
+
+ newSegment = segmentManager.openNewSegment();
+ super.setupJob(context);
+ }
+
+ @Override
+ public void abortJob(JobContext context, int runState) throws IOException {
+ segmentManager.closeSegment(newSegment);
+ super.abortJob(context, runState);
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ newSegment.setupForRead(context);
+ segmentManager.commitSegment(newSegment);
+ super.commitJob(context);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
new file mode 100644
index 0000000..2dcb1cb
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * Base class for all output format for CarbonData file.
+ * @param <T>
+ */
+public abstract class CarbonTableOutputFormat<T> extends FileOutputFormat<Void, T> {
+
+ @Override
+ public RecordWriter<Void, T> getRecordWriter(FileSystem ignored, JobConf job, String name,
+ Progressable progress) throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
new file mode 100644
index 0000000..58b2e67
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal;
+
+public enum CarbonFormatType {
+ COLUMNAR
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
new file mode 100644
index 0000000..2a982a2
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+
+/**
+ * Carbon input split can be different format, application should create the record reader based on format type.
+ */
+public abstract class CarbonInputSplit extends InputSplit {
+
+ /**
+ * @return the format type of this split.
+ */
+ public abstract CarbonFormatType formatType();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java
new file mode 100644
index 0000000..f62b568
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index;
+
+import java.util.List;
+
+/**
+ * Represent one HDFS Block (one CarbonData file).
+ */
+public interface Block {
+
+ /**
+ * @return the path of this block
+ */
+ String getBlockPath();
+
+ /**
+ * return all matched blocklets for scanning
+ * @return list of blocklet offset in the block
+ */
+ List<Long> getMatchedBlocklets();
+
+ /**
+ * @return true if need to do full scan of this block
+ */
+ boolean fullScan();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
new file mode 100644
index 0000000..8c4388a
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * An index is associate with one segment, it is used for filtering on files in the segment.
+ */
+public interface Index {
+
+ /**
+ * Index name
+ * @return index name
+ */
+ String getName();
+
+ /**
+ * Used to filter blocks based on filter
+ * @param job job
+ * @param filter filter
+ * @return filtered block
+ * @throws IOException
+ */
+ List<Block> filter(JobContext job, FilterResolverIntf filter) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java
new file mode 100644
index 0000000..c94d51c
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index;
+
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Used to load the index
+ */
+public interface IndexLoader {
+
+ /**
+ * load the index based on the index information set in the configuration
+ * @param segment the index from which segment to load
+ * @return the loaded index
+ */
+ Index load(Segment segment);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
new file mode 100644
index 0000000..e0840f7
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.carbon.datastore.SegmentTaskIndexStore;
+import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.carbon.datastore.block.BlockletInfos;
+import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException;
+import org.apache.carbondata.core.carbon.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.carbon.datastore.impl.btree.BlockBTreeLeafNode;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.internal.index.Block;
+import org.apache.carbondata.hadoop.internal.index.Index;
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.scan.filter.FilterUtil;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+class InMemoryBTreeIndex implements Index {
+
+ private static final Log LOG = LogFactory.getLog(InMemoryBTreeIndex.class);
+ private Segment segment;
+
+ InMemoryBTreeIndex(Segment segment) {
+ this.segment = segment;
+ }
+
+ @Override
+ public String getName() {
+ return null;
+ }
+
+ @Override
+ public List<Block> filter(JobContext job, FilterResolverIntf filter)
+ throws IOException {
+
+ List<Block> result = new LinkedList<>();
+
+ FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
+
+ AbsoluteTableIdentifier identifier = null;
+
+ //for this segment fetch blocks matching filter in BTree
+ List<DataRefNode> dataRefNodes = null;
+ try {
+ dataRefNodes = getDataBlocksOfSegment(job, filterExpressionProcessor, identifier, filter);
+ } catch (IndexBuilderException e) {
+ throw new IOException(e.getMessage());
+ }
+ for (DataRefNode dataRefNode : dataRefNodes) {
+ BlockBTreeLeafNode leafNode = (BlockBTreeLeafNode) dataRefNode;
+ TableBlockInfo tableBlockInfo = leafNode.getTableBlockInfo();
+ result.add(new CarbonInputSplit(segment.getId(), new Path(tableBlockInfo.getFilePath()),
+ tableBlockInfo.getBlockOffset(), tableBlockInfo.getBlockLength(),
+ tableBlockInfo.getLocations(), tableBlockInfo.getBlockletInfos().getNoOfBlockLets()));
+ }
+ return result;
+ }
+
+ private Map<String, AbstractIndex> getSegmentAbstractIndexs(JobContext job,
+ AbsoluteTableIdentifier identifier)
+ throws IOException, IndexBuilderException {
+ Map<String, AbstractIndex> segmentIndexMap =
+ SegmentTaskIndexStore.getInstance().getSegmentBTreeIfExists(identifier, segment.getId());
+
+ // if segment tree is not loaded, load the segment tree
+ if (segmentIndexMap == null) {
+ List<TableBlockInfo> tableBlockInfoList = getTableBlockInfo(job);
+ Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos = new HashMap<>();
+ segmentToTableBlocksInfos.put(segment.getId(), tableBlockInfoList);
+
+ // TODO: loadAndGetTaskIdToSegmentsMap can be optimized, use tableBlockInfoList as input
+ // get Btree blocks for given segment
+ segmentIndexMap = SegmentTaskIndexStore.getInstance()
+ .loadAndGetTaskIdToSegmentsMap(segmentToTableBlocksInfos, identifier);
+
+ }
+ return segmentIndexMap;
+ }
+
+ /**
+ * Below method will be used to get the table block info
+ *
+ * @param job job context
+ * @return list of table block
+ * @throws IOException
+ */
+ private List<TableBlockInfo> getTableBlockInfo(JobContext job) throws IOException {
+ List<TableBlockInfo> tableBlockInfoList = new ArrayList<>();
+
+ // identify table blocks from all file locations of given segment
+ for (InputSplit inputSplit : segment.getAllSplits(job)) {
+ CarbonInputSplit carbonInputSplit = (CarbonInputSplit) inputSplit;
+ BlockletInfos blockletInfos = new BlockletInfos(carbonInputSplit.getNumberOfBlocklets(), 0,
+ carbonInputSplit.getNumberOfBlocklets());
+ tableBlockInfoList.add(
+ new TableBlockInfo(carbonInputSplit.getPath().toString(), carbonInputSplit.getStart(),
+ segment.getId(), carbonInputSplit.getLocations(), carbonInputSplit.getLength(),
+ blockletInfos));
+ }
+ return tableBlockInfoList;
+ }
+
+ /**
+ * get data blocks of given segment
+ */
+ private List<DataRefNode> getDataBlocksOfSegment(JobContext job,
+ FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier identifier,
+ FilterResolverIntf resolver) throws IndexBuilderException, IOException {
+
+ QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
+ QueryStatistic statistic = new QueryStatistic();
+ Map<String, AbstractIndex> segmentIndexMap = getSegmentAbstractIndexs(job, identifier);
+
+ List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
+
+ // build result
+ for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
+
+ List<DataRefNode> filterredBlocks = null;
+ // if no filter is given get all blocks from Btree Index
+ if (null == resolver) {
+ filterredBlocks = getDataBlocksOfIndex(abstractIndex);
+ } else {
+ // apply filter and get matching blocks
+ try {
+ filterredBlocks =
+ filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(),
+ resolver, abstractIndex, identifier);
+ } catch (QueryExecutionException e) {
+ throw new IndexBuilderException(e.getMessage());
+ }
+ }
+ resultFilterredBlocks.addAll(filterredBlocks);
+ }
+ statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER,
+ System.currentTimeMillis());
+ recorder.recordStatistics(statistic);
+ recorder.logStatistics();
+ return resultFilterredBlocks;
+ }
+
+ /**
+ * get data blocks of given btree
+ */
+ private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
+ List<DataRefNode> blocks = new LinkedList<DataRefNode>();
+ SegmentProperties segmentProperties = abstractIndex.getSegmentProperties();
+
+ try {
+ IndexKey startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
+ IndexKey endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
+
+ // Add all blocks of btree into result
+ DataRefNodeFinder blockFinder =
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ DataRefNode startBlock =
+ blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
+ DataRefNode endBlock =
+ blockFinder.findLastDataBlock(abstractIndex.getDataRefNode(), endIndexKey);
+ while (startBlock != endBlock) {
+ blocks.add(startBlock);
+ startBlock = startBlock.getNextDataRefNode();
+ }
+ blocks.add(endBlock);
+
+ } catch (KeyGenException e) {
+ LOG.error("Could not generate start key", e);
+ }
+ return blocks;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java
new file mode 100644
index 0000000..78f89e1
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index.impl;
+
+import org.apache.carbondata.hadoop.internal.index.Index;
+import org.apache.carbondata.hadoop.internal.index.IndexLoader;
+import org.apache.hadoop.conf.Configuration;
+
+public class InMemoryBTreeIndexLoader implements IndexLoader {
+ @Override
+ public Index load(Configuration conf) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
new file mode 100644
index 0000000..eb7226f
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * Within a carbon table, each data load becomes one Segment, which stores all data files belong to this load in
+ * the segment folder.
+ */
+public abstract class Segment {
+
+ protected String id;
+
+ /**
+ * Path of the segment folder
+ */
+ private String path;
+
+ public Segment(String id, String path) {
+ this.id = id;
+ this.path = path;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ /**
+ * return all InputSplit of this segment, each file is a InputSplit
+ * @param job job context
+ * @return all InputSplit
+ * @throws IOException
+ */
+ public List<InputSplit> getAllSplits(JobContext job) throws IOException {
+ List<InputSplit> result = new ArrayList<>();
+ Path p = new Path(path);
+ FileSystem fs = p.getFileSystem(job.getConfiguration());
+
+ //TODO: filter out the hidden files
+ FileStatus[] files = fs.globStatus(p);
+ for (FileStatus file: files) {
+ // make split and add to result
+ }
+ return result;
+ }
+
+ /**
+ * get all files, implementation may use the input filter and index to prune files
+ * @param job job context
+ * @param filterResolver filter
+ * @return all files
+ */
+ public abstract List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver)
+ throws IOException;
+
+ /**
+ * This is called after Segment is loaded and before segment is committed,
+ * implementation should load indices if required
+ * @param job job context
+ */
+ public abstract void setupForRead(JobContext job) throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java
new file mode 100644
index 0000000..ca596e3
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment;
+
+import java.io.IOException;
+
+/**
+ * Segment manager is used to manage all segment within one table. It manages segment state internally in a
+ * transactional manner.
+ */
+public interface SegmentManager {
+
+ /**
+ * Used for getting all segments for scan
+ */
+ Segment[] getAllValidSegments();
+
+ /**
+ * Used for data load
+ * caller should open new segment by calling this, after data success, call commitSegment,
+ * call closeSegment if any failure
+ */
+ Segment openNewSegment() throws IOException;
+
+ /**
+ * Call this function when data load or compaction is completed successfully.
+ */
+ void commitSegment(Segment segment) throws IOException;
+
+ /**
+ * Call this function when data load or compaction failed.
+ */
+ void closeSegment(Segment segment) throws IOException;
+
+ /**
+ * Delete this segment
+ */
+ void deleteSegment(Segment segment) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java
new file mode 100644
index 0000000..7545603
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment;
+
+/**
+ * Used to get the global segment manager instance
+ */
+public class SegmentManagerFactory {
+ public static SegmentManager getGlobalSegmentManager() {
+ //TODO return the default implementation
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
new file mode 100644
index 0000000..158180f
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment.impl;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.carbondata.hadoop.internal.CarbonInputSplit;
+import org.apache.carbondata.hadoop.internal.index.Block;
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.index.Index;
+import org.apache.carbondata.hadoop.internal.index.IndexLoader;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * This segment is backed by index, thus getSplits can use the index to do file pruning.
+ */
+public class IndexedSegment extends Segment {
+
+ private IndexLoader loader;
+
+ public IndexedSegment(String name, String path, IndexLoader loader) {
+ super(name, path);
+ this.loader = loader;
+ }
+
+ @Override
+ public List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver)
+ throws IOException {
+ // do as following
+ // 1. create the index or get from cache by the filter name in the configuration
+ // 2. filter by index to get the filtered block
+ // 3. create input split from filtered block
+
+ List<InputSplit> output = new LinkedList<>();
+ Index index = loader.load(this);
+ List<Block> blocks = index.filter(job, filterResolver);
+ for (Block block: blocks) {
+ output.add(makeInputSplit(block));
+ }
+ return output;
+ }
+
+ @Override
+ public void setupForRead(JobContext job) throws IOException {
+
+ }
+
+ private InputSplit makeInputSplit(Block block) {
+ // TODO: get all required parameter from block
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java
new file mode 100644
index 0000000..337d92c
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment.impl.zk;
+
+import java.io.IOException;
+
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
+
+/**
+ * This class leverage Zookeeper and HDFS file to manage segments.
+ * Zookeeper is used for locking, and HDFS is for storing the state of segments.
+ */
+public class ZkSegmentManager implements SegmentManager {
+ @Override
+ public Segment[] getAllValidSegments() {
+ return new Segment[0];
+ }
+
+ @Override
+ public Segment openNewSegment() throws IOException {
+ return null;
+ }
+
+ @Override
+ public void commitSegment(Segment segment) throws IOException {
+
+ }
+
+ @Override
+ public void closeSegment(Segment segment) throws IOException {
+
+ }
+
+ @Override
+ public void deleteSegment(Segment segment) throws IOException {
+
+ }
+}
[2/2] incubator-carbondata git commit: [CARBONDATA-284] Abstracting
index and segment interface This closes #208
Posted by ra...@apache.org.
[CARBONDATA-284] Abstracting index and segment interface This closes #208
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c910e6fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c910e6fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c910e6fd
Branch: refs/heads/master
Commit: c910e6fd803d38646c7ef99731bdef1f077d1d08
Parents: ac62b58 edb02ab
Author: ravipesala <ra...@gmail.com>
Authored: Sat Nov 5 00:05:17 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Nov 5 00:05:17 2016 +0530
----------------------------------------------------------------------
.../carbondata/hadoop/CarbonInputSplit.java | 18 +-
.../hadoop/api/CarbonTableInputFormat.java | 170 +++++++++++++++
.../hadoop/api/CarbonTableOutputCommitter.java | 56 +++++
.../hadoop/api/CarbonTableOutputFormat.java | 41 ++++
.../hadoop/internal/CarbonFormatType.java | 24 +++
.../hadoop/internal/CarbonInputSplit.java | 33 +++
.../carbondata/hadoop/internal/index/Block.java | 44 ++++
.../carbondata/hadoop/internal/index/Index.java | 48 +++++
.../hadoop/internal/index/IndexLoader.java | 36 ++++
.../internal/index/impl/InMemoryBTreeIndex.java | 214 +++++++++++++++++++
.../index/impl/InMemoryBTreeIndexLoader.java | 31 +++
.../hadoop/internal/segment/Segment.java | 94 ++++++++
.../hadoop/internal/segment/SegmentManager.java | 56 +++++
.../internal/segment/SegmentManagerFactory.java | 30 +++
.../internal/segment/impl/IndexedSegment.java | 73 +++++++
.../segment/impl/zk/ZkSegmentManager.java | 56 +++++
16 files changed, 1023 insertions(+), 1 deletion(-)
----------------------------------------------------------------------