You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2016/11/04 18:35:45 UTC

[1/2] incubator-carbondata git commit: add interface

Repository: incubator-carbondata
Updated Branches:
  refs/heads/master ac62b58d7 -> c910e6fd8


add interface

modify name

add comment

add format type

fix comment

fix comment


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/edb02ab5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/edb02ab5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/edb02ab5

Branch: refs/heads/master
Commit: edb02ab504dc775eefab992d16ae1f5338a614f4
Parents: ac62b58
Author: jackylk <ja...@huawei.com>
Authored: Thu Oct 27 18:39:39 2016 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Nov 5 00:04:21 2016 +0530

----------------------------------------------------------------------
 .../carbondata/hadoop/CarbonInputSplit.java     |  18 +-
 .../hadoop/api/CarbonTableInputFormat.java      | 170 +++++++++++++++
 .../hadoop/api/CarbonTableOutputCommitter.java  |  56 +++++
 .../hadoop/api/CarbonTableOutputFormat.java     |  41 ++++
 .../hadoop/internal/CarbonFormatType.java       |  24 +++
 .../hadoop/internal/CarbonInputSplit.java       |  33 +++
 .../carbondata/hadoop/internal/index/Block.java |  44 ++++
 .../carbondata/hadoop/internal/index/Index.java |  48 +++++
 .../hadoop/internal/index/IndexLoader.java      |  36 ++++
 .../internal/index/impl/InMemoryBTreeIndex.java | 214 +++++++++++++++++++
 .../index/impl/InMemoryBTreeIndexLoader.java    |  31 +++
 .../hadoop/internal/segment/Segment.java        |  94 ++++++++
 .../hadoop/internal/segment/SegmentManager.java |  56 +++++
 .../internal/segment/SegmentManagerFactory.java |  30 +++
 .../internal/segment/impl/IndexedSegment.java   |  73 +++++++
 .../segment/impl/zk/ZkSegmentManager.java       |  56 +++++
 16 files changed, 1023 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 01744f8..2cdefe9 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -22,7 +22,9 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.io.Serializable;
+import java.util.List;
 
+import org.apache.carbondata.hadoop.internal.index.Block;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
@@ -30,7 +32,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 /**
  * Carbon input split to allow distributed read of CarbonInputFormat.
  */
-public class CarbonInputSplit extends FileSplit implements Serializable, Writable {
+public class CarbonInputSplit extends FileSplit implements Serializable, Writable, Block {
 
   private static final long serialVersionUID = 3520344046772190207L;
   private String segmentId;
@@ -84,4 +86,18 @@ public class CarbonInputSplit extends FileSplit implements Serializable, Writabl
     return numberOfBlocklets;
   }
 
+  @Override
+  public String getBlockPath() {
+    return getPath().getName();
+  }
+
+  @Override
+  public List<Long> getMatchedBlocklets() {
+    return null;
+  }
+
+  @Override
+  public boolean fullScan() {
+    return true;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
new file mode 100644
index 0000000..233a5b7
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.api;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.carbondata.hadoop.CarbonProjection;
+import org.apache.carbondata.hadoop.internal.CarbonInputSplit;
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManagerFactory;
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
+import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
+import org.apache.carbondata.scan.expression.Expression;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+
+/**
+ * Input format of CarbonData file.
+ * @param <T>
+ */
+public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
+
+  private static final String FILTER_PREDICATE =
+      "mapreduce.input.carboninputformat.filter.predicate";
+
+  private SegmentManager segmentManager;
+
+  public CarbonTableInputFormat() {
+    this.segmentManager = SegmentManagerFactory.getGlobalSegmentManager();
+  }
+
+  @Override
+  public RecordReader<Void, T> createRecordReader(InputSplit split,
+      TaskAttemptContext context) throws IOException, InterruptedException {
+    switch (((CarbonInputSplit)split).formatType()) {
+      case COLUMNR:
+        // TODO: create record reader for columnar format
+        break;
+      default:
+        throw new RuntimeException("Unsupported format type");
+    }
+    return null;
+  }
+
+  @Override
+  public List<InputSplit> getSplits(JobContext job) throws IOException {
+
+    // work as following steps:
+    // get all current valid segment
+    // for each segment, get all input split
+
+    List<InputSplit> output = new LinkedList<>();
+    Expression filter = getFilter(job.getConfiguration());
+    Segment[] segments = segmentManager.getAllValidSegments();
+    FilterResolverIntf filterResolver = CarbonInputFormatUtil.resolveFilter(filter, null);
+    for (Segment segment: segments) {
+      List<InputSplit> splits = segment.getSplits(job, filterResolver);
+      output.addAll(splits);
+    }
+    return output;
+  }
+
+  /**
+   * set the table path into configuration
+   * @param conf configuration of the job
+   * @param tablePath table path string
+   */
+  public void setTablePath(Configuration conf, String tablePath) {
+
+  }
+
+  /**
+   * return the table path in the configuration
+   * @param conf configuration of the job
+   * @return table path string
+   */
+  public String getTablePath(Configuration conf) {
+    return null;
+  }
+
+  /**
+   * set projection columns into configuration
+   * @param conf configuration of the job
+   * @param projection projection
+   */
+  public void setProjection(Configuration conf, CarbonProjection projection) {
+
+  }
+
+  /**
+   * return the projection in the configuration
+   * @param conf configuration of the job
+   * @return projection
+   */
+  public CarbonProjection getProjection(Configuration conf) {
+    return null;
+  }
+
+  /**
+   * set filter expression into the configuration
+   * @param conf configuration of the job
+   * @param filter filter expression
+   */
+  public void setFilter(Configuration conf, Expression filter) {
+    try {
+      String filterString = ObjectSerializationUtil.convertObjectToString(filter);
+      conf.set(FILTER_PREDICATE, filterString);
+    } catch (Exception e) {
+      throw new RuntimeException("Error while setting filter expression to Job", e);
+    }
+  }
+
+  /**
+   * return filter expression in the configuration
+   * @param conf configuration of the job
+   * @return filter expression
+   */
+  public Expression getFilter(Configuration conf) {
+    Object filter;
+    String filterExprString = conf.get(FILTER_PREDICATE);
+    if (filterExprString == null) {
+      return null;
+    }
+    try {
+      filter = ObjectSerializationUtil.convertStringToObject(filterExprString);
+    } catch (IOException e) {
+      throw new RuntimeException("Error while reading filter expression", e);
+    }
+    assert(filter instanceof Expression);
+    return (Expression) filter;
+  }
+
+  /**
+   * Optional API. It can be used by query optimizer to select index based on filter
+   * in the configuration of the job. After selecting index internally, index' name will be set
+   * in the configuration.
+   *
+   * The process of selection is simple, just use the default index. Subclass can provide a more
+   * advanced selection logic like cost based.
+   * @param conf job configuration
+   */
+  public void selectIndex(Configuration conf) {
+    // set the default index in configuration
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java
new file mode 100644
index 0000000..3b4a3a2
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputCommitter.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.api;
+
+import java.io.IOException;
+
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
+import org.apache.hadoop.mapred.FileOutputCommitter;
+import org.apache.hadoop.mapred.JobContext;
+
+public class CarbonTableOutputCommitter extends FileOutputCommitter {
+  private SegmentManager segmentManager;
+  private Segment newSegment;
+
+  @Override
+  public void setupJob(JobContext context) throws IOException {
+    // steps:
+    // call segment manager to open new segment for loading
+    // load data synchronously
+    // close the segment and make it available for reading
+
+    newSegment = segmentManager.openNewSegment();
+    super.setupJob(context);
+  }
+
+  @Override
+  public void abortJob(JobContext context, int runState) throws IOException {
+    segmentManager.closeSegment(newSegment);
+    super.abortJob(context, runState);
+  }
+
+  @Override
+  public void commitJob(JobContext context) throws IOException {
+    newSegment.setupForRead(context);
+    segmentManager.commitSegment(newSegment);
+    super.commitJob(context);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
new file mode 100644
index 0000000..2dcb1cb
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableOutputFormat.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordWriter;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * Base class for all output format for CarbonData file.
+ * @param <T>
+ */
+public abstract class CarbonTableOutputFormat<T> extends FileOutputFormat<Void, T> {
+
+  @Override
+  public RecordWriter<Void, T> getRecordWriter(FileSystem ignored, JobConf job, String name,
+      Progressable progress) throws IOException {
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
new file mode 100644
index 0000000..58b2e67
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonFormatType.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal;
+
+public enum CarbonFormatType {
+  COLUMNAR
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
new file mode 100644
index 0000000..2a982a2
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/CarbonInputSplit.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+
+/**
+ * Carbon input split can be different format, application should create the record reader based on format type.
+ */
+public abstract class CarbonInputSplit extends InputSplit {
+
+  /**
+   * @return the format type of this split.
+   */
+  public abstract CarbonFormatType formatType();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java
new file mode 100644
index 0000000..f62b568
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Block.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index;
+
+import java.util.List;
+
+/**
+ * Represent one HDFS Block (one CarbonData file).
+ */
+public interface Block {
+
+  /**
+   * @return the path of this block
+   */
+  String getBlockPath();
+
+  /**
+   * return all matched blocklets for scanning
+   * @return list of blocklet offset in the block
+   */
+  List<Long> getMatchedBlocklets();
+
+  /**
+   * @return true if need to do full scan of this block
+   */
+  boolean fullScan();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
new file mode 100644
index 0000000..8c4388a
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * An index is associate with one segment, it is used for filtering on files in the segment.
+ */
+public interface Index {
+
+  /**
+   * Index name
+   * @return index name
+   */
+  String getName();
+
+  /**
+   * Used to filter blocks based on filter
+   * @param job job
+   * @param filter filter
+   * @return filtered block
+   * @throws IOException
+   */
+  List<Block> filter(JobContext job, FilterResolverIntf filter) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java
new file mode 100644
index 0000000..c94d51c
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/IndexLoader.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index;
+
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Used to load the index
+ */
+public interface IndexLoader {
+
+  /**
+   * load the index based on the index information set in the configuration
+   * @param segment the index from which segment to load
+   * @return the loaded index
+   */
+  Index load(Segment segment);
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
new file mode 100644
index 0000000..e0840f7
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.carbon.datastore.SegmentTaskIndexStore;
+import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.carbon.datastore.block.BlockletInfos;
+import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException;
+import org.apache.carbondata.core.carbon.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.carbon.datastore.impl.btree.BlockBTreeLeafNode;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants;
+import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsRecorder;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.internal.index.Block;
+import org.apache.carbondata.hadoop.internal.index.Index;
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.scan.filter.FilterUtil;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+class InMemoryBTreeIndex implements Index {
+
+  private static final Log LOG = LogFactory.getLog(InMemoryBTreeIndex.class);
+  private Segment segment;
+
+  InMemoryBTreeIndex(Segment segment) {
+    this.segment = segment;
+  }
+
+  @Override
+  public String getName() {
+    return null;
+  }
+
+  @Override
+  public List<Block> filter(JobContext job, FilterResolverIntf filter)
+      throws IOException {
+
+    List<Block> result = new LinkedList<>();
+
+    FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
+
+    AbsoluteTableIdentifier identifier = null;
+
+    //for this segment fetch blocks matching filter in BTree
+    List<DataRefNode> dataRefNodes = null;
+    try {
+      dataRefNodes = getDataBlocksOfSegment(job, filterExpressionProcessor, identifier, filter);
+    } catch (IndexBuilderException e) {
+      throw new IOException(e.getMessage());
+    }
+    for (DataRefNode dataRefNode : dataRefNodes) {
+      BlockBTreeLeafNode leafNode = (BlockBTreeLeafNode) dataRefNode;
+      TableBlockInfo tableBlockInfo = leafNode.getTableBlockInfo();
+      result.add(new CarbonInputSplit(segment.getId(), new Path(tableBlockInfo.getFilePath()),
+          tableBlockInfo.getBlockOffset(), tableBlockInfo.getBlockLength(),
+          tableBlockInfo.getLocations(), tableBlockInfo.getBlockletInfos().getNoOfBlockLets()));
+    }
+    return result;
+  }
+
+  private Map<String, AbstractIndex> getSegmentAbstractIndexs(JobContext job,
+      AbsoluteTableIdentifier identifier)
+      throws IOException, IndexBuilderException {
+    Map<String, AbstractIndex> segmentIndexMap =
+        SegmentTaskIndexStore.getInstance().getSegmentBTreeIfExists(identifier, segment.getId());
+
+    // if segment tree is not loaded, load the segment tree
+    if (segmentIndexMap == null) {
+      List<TableBlockInfo> tableBlockInfoList = getTableBlockInfo(job);
+      Map<String, List<TableBlockInfo>> segmentToTableBlocksInfos = new HashMap<>();
+      segmentToTableBlocksInfos.put(segment.getId(), tableBlockInfoList);
+
+      // TODO: loadAndGetTaskIdToSegmentsMap can be optimized, use tableBlockInfoList as input
+      // get Btree blocks for given segment
+      segmentIndexMap = SegmentTaskIndexStore.getInstance()
+          .loadAndGetTaskIdToSegmentsMap(segmentToTableBlocksInfos, identifier);
+
+    }
+    return segmentIndexMap;
+  }
+
+  /**
+   * Below method will be used to get the table block info
+   *
+   * @param job                     job context
+   * @return list of table block
+   * @throws IOException
+   */
+  private List<TableBlockInfo> getTableBlockInfo(JobContext job) throws IOException {
+    List<TableBlockInfo> tableBlockInfoList = new ArrayList<>();
+
+    // identify table blocks from all file locations of given segment
+    for (InputSplit inputSplit : segment.getAllSplits(job)) {
+      CarbonInputSplit carbonInputSplit = (CarbonInputSplit) inputSplit;
+      BlockletInfos blockletInfos = new BlockletInfos(carbonInputSplit.getNumberOfBlocklets(), 0,
+          carbonInputSplit.getNumberOfBlocklets());
+      tableBlockInfoList.add(
+          new TableBlockInfo(carbonInputSplit.getPath().toString(), carbonInputSplit.getStart(),
+              segment.getId(), carbonInputSplit.getLocations(), carbonInputSplit.getLength(),
+              blockletInfos));
+    }
+    return tableBlockInfoList;
+  }
+
+  /**
+   * get data blocks of given segment
+   */
+  private List<DataRefNode> getDataBlocksOfSegment(JobContext job,
+      FilterExpressionProcessor filterExpressionProcessor, AbsoluteTableIdentifier identifier,
+      FilterResolverIntf resolver) throws IndexBuilderException, IOException {
+
+    QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
+    QueryStatistic statistic = new QueryStatistic();
+    Map<String, AbstractIndex> segmentIndexMap = getSegmentAbstractIndexs(job, identifier);
+
+    List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
+
+    // build result
+    for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
+
+      List<DataRefNode> filterredBlocks = null;
+      // if no filter is given get all blocks from Btree Index
+      if (null == resolver) {
+        filterredBlocks = getDataBlocksOfIndex(abstractIndex);
+      } else {
+        // apply filter and get matching blocks
+        try {
+          filterredBlocks =
+              filterExpressionProcessor.getFilterredBlocks(abstractIndex.getDataRefNode(),
+                  resolver, abstractIndex, identifier);
+        } catch (QueryExecutionException e) {
+          throw new IndexBuilderException(e.getMessage());
+        }
+      }
+      resultFilterredBlocks.addAll(filterredBlocks);
+    }
+    statistic.addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER,
+        System.currentTimeMillis());
+    recorder.recordStatistics(statistic);
+    recorder.logStatistics();
+    return resultFilterredBlocks;
+  }
+
+  /**
+   * get data blocks of given btree
+   */
+  private List<DataRefNode> getDataBlocksOfIndex(AbstractIndex abstractIndex) {
+    List<DataRefNode> blocks = new LinkedList<DataRefNode>();
+    SegmentProperties segmentProperties = abstractIndex.getSegmentProperties();
+
+    try {
+      IndexKey startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
+      IndexKey endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
+
+      // Add all blocks of btree into result
+      DataRefNodeFinder blockFinder =
+          new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+      DataRefNode startBlock =
+          blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
+      DataRefNode endBlock =
+          blockFinder.findLastDataBlock(abstractIndex.getDataRefNode(), endIndexKey);
+      while (startBlock != endBlock) {
+        blocks.add(startBlock);
+        startBlock = startBlock.getNextDataRefNode();
+      }
+      blocks.add(endBlock);
+
+    } catch (KeyGenException e) {
+      LOG.error("Could not generate start key", e);
+    }
+    return blocks;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java
new file mode 100644
index 0000000..78f89e1
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndexLoader.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.index.impl;
+
+import org.apache.carbondata.hadoop.internal.index.Index;
+import org.apache.carbondata.hadoop.internal.index.IndexLoader;
+import org.apache.hadoop.conf.Configuration;
+
+public class InMemoryBTreeIndexLoader implements IndexLoader {
+  @Override
+  public Index load(Configuration conf) {
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
new file mode 100644
index 0000000..eb7226f
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * Within a carbon table, each data load becomes one Segment, which stores all data files belong to this load in
+ * the segment folder.
+ */
+public abstract class Segment {
+
+  protected String id;
+
+  /**
+   * Path of the segment folder
+   */
+  private String path;
+
+  public Segment(String id, String path) {
+    this.id = id;
+    this.path = path;
+  }
+
+  public String getId() {
+    return id;
+  }
+
+  public String getPath() {
+    return path;
+  }
+
+  /**
+   * return all InputSplit of this segment, each file is a InputSplit
+   * @param job job context
+   * @return all InputSplit
+   * @throws IOException
+   */
+  public List<InputSplit> getAllSplits(JobContext job) throws IOException {
+    List<InputSplit> result = new ArrayList<>();
+    Path p = new Path(path);
+    FileSystem fs = p.getFileSystem(job.getConfiguration());
+
+    //TODO: filter out the hidden files
+    FileStatus[] files = fs.globStatus(p);
+    for (FileStatus file: files) {
+      // make split and add to result
+    }
+    return result;
+  }
+
+  /**
+   * get all files, implementation may use the input filter and index to prune files
+   * @param job job context
+   * @param filterResolver filter
+   * @return all files
+   */
+  public abstract List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver)
+      throws IOException;
+
+  /**
+   * This is called after Segment is loaded and before segment is committed,
+   * implementation should load indices if required
+   * @param job job context
+   */
+  public abstract void setupForRead(JobContext job) throws IOException;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java
new file mode 100644
index 0000000..ca596e3
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManager.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment;
+
+import java.io.IOException;
+
+/**
+ * Segment manager is used to manage all segment within one table. It manages segment state internally in a
+ * transactional manner.
+ */
+public interface SegmentManager {
+
+  /**
+   * Used for getting all segments for scan
+   */
+  Segment[] getAllValidSegments();
+
+  /**
+   * Used for data load
+   * caller should open new segment by calling this, after data success, call commitSegment,
+   * call closeSegment if any failure
+   */
+  Segment openNewSegment() throws IOException;
+
+  /**
+   * Call this function when data load or compaction is completed successfully.
+   */
+  void commitSegment(Segment segment) throws IOException;
+
+  /**
+   * Call this function when data load or compaction failed.
+   */
+  void closeSegment(Segment segment) throws IOException;
+
+  /**
+   * Delete this segment
+   */
+  void deleteSegment(Segment segment) throws IOException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java
new file mode 100644
index 0000000..7545603
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/SegmentManagerFactory.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment;
+
+/**
+ * Used to get the global segment manager instance
+ */
+public class SegmentManagerFactory {
+  public static SegmentManager getGlobalSegmentManager() {
+    //TODO return the default implementation
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
new file mode 100644
index 0000000..158180f
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment.impl;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.carbondata.hadoop.internal.CarbonInputSplit;
+import org.apache.carbondata.hadoop.internal.index.Block;
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.index.Index;
+import org.apache.carbondata.hadoop.internal.index.IndexLoader;
+import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * This segment is backed by index, thus getSplits can use the index to do file pruning.
+ */
+public class IndexedSegment extends Segment {
+
+  private IndexLoader loader;
+
+  public IndexedSegment(String name, String path, IndexLoader loader) {
+    super(name, path);
+    this.loader = loader;
+  }
+
+  @Override
+  public List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver)
+      throws IOException {
+    // do as following
+    // 1. create the index or get from cache by the filter name in the configuration
+    // 2. filter by index to get the filtered block
+    // 3. create input split from filtered block
+
+    List<InputSplit> output = new LinkedList<>();
+    Index index = loader.load(this);
+    List<Block> blocks = index.filter(job, filterResolver);
+    for (Block block: blocks) {
+      output.add(makeInputSplit(block));
+    }
+    return output;
+  }
+
+  @Override
+  public void setupForRead(JobContext job) throws IOException {
+
+  }
+
+  private InputSplit makeInputSplit(Block block) {
+    // TODO: get all required parameter from block
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/edb02ab5/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java
new file mode 100644
index 0000000..337d92c
--- /dev/null
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/zk/ZkSegmentManager.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.hadoop.internal.segment.impl.zk;
+
+import java.io.IOException;
+
+import org.apache.carbondata.hadoop.internal.segment.Segment;
+import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
+
+/**
+ * This class leverage Zookeeper and HDFS file to manage segments.
+ * Zookeeper is used for locking, and HDFS is for storing the state of segments.
+ */
+public class ZkSegmentManager implements SegmentManager {
+  @Override
+  public Segment[] getAllValidSegments() {
+    return new Segment[0];
+  }
+
+  @Override
+  public Segment openNewSegment() throws IOException {
+    return null;
+  }
+
+  @Override
+  public void commitSegment(Segment segment) throws IOException {
+
+  }
+
+  @Override
+  public void closeSegment(Segment segment) throws IOException {
+
+  }
+
+  @Override
+  public void deleteSegment(Segment segment) throws IOException {
+
+  }
+}



[2/2] incubator-carbondata git commit: [CARBONDATA-284] Abstracting index and segment interface This closes #208

Posted by ra...@apache.org.
[CARBONDATA-284] Abstracting index and segment interface This closes #208


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c910e6fd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c910e6fd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c910e6fd

Branch: refs/heads/master
Commit: c910e6fd803d38646c7ef99731bdef1f077d1d08
Parents: ac62b58 edb02ab
Author: ravipesala <ra...@gmail.com>
Authored: Sat Nov 5 00:05:17 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Nov 5 00:05:17 2016 +0530

----------------------------------------------------------------------
 .../carbondata/hadoop/CarbonInputSplit.java     |  18 +-
 .../hadoop/api/CarbonTableInputFormat.java      | 170 +++++++++++++++
 .../hadoop/api/CarbonTableOutputCommitter.java  |  56 +++++
 .../hadoop/api/CarbonTableOutputFormat.java     |  41 ++++
 .../hadoop/internal/CarbonFormatType.java       |  24 +++
 .../hadoop/internal/CarbonInputSplit.java       |  33 +++
 .../carbondata/hadoop/internal/index/Block.java |  44 ++++
 .../carbondata/hadoop/internal/index/Index.java |  48 +++++
 .../hadoop/internal/index/IndexLoader.java      |  36 ++++
 .../internal/index/impl/InMemoryBTreeIndex.java | 214 +++++++++++++++++++
 .../index/impl/InMemoryBTreeIndexLoader.java    |  31 +++
 .../hadoop/internal/segment/Segment.java        |  94 ++++++++
 .../hadoop/internal/segment/SegmentManager.java |  56 +++++
 .../internal/segment/SegmentManagerFactory.java |  30 +++
 .../internal/segment/impl/IndexedSegment.java   |  73 +++++++
 .../segment/impl/zk/ZkSegmentManager.java       |  56 +++++
 16 files changed, 1023 insertions(+), 1 deletion(-)
----------------------------------------------------------------------