You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/15 17:29:21 UTC
[1/2] carbondata git commit: Added data map interfaces
Repository: carbondata
Updated Branches:
refs/heads/master c3cde5af1 -> 7bb822031
Added data map interfaces
Fixed comments
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b4343469
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b4343469
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b4343469
Branch: refs/heads/master
Commit: b434346922632899b2bf77d5e4d5469caf63d40a
Parents: c3cde5a
Author: ravipesala <ra...@gmail.com>
Authored: Sun Jun 4 18:32:48 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jun 16 01:18:40 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/events/ChangeEvent.java | 35 +++++++
.../carbondata/core/events/EventListener.java | 25 +++++
.../carbondata/core/indexstore/Blocklet.java | 42 ++++++++
.../carbondata/core/indexstore/DataMap.java | 54 ++++++++++
.../core/indexstore/DataMapDistributable.java | 56 ++++++++++
.../core/indexstore/DataMapStoreManager.java | 105 +++++++++++++++++++
.../carbondata/core/indexstore/DataMapType.java | 34 ++++++
.../core/indexstore/DataMapWriter.java | 50 +++++++++
.../core/indexstore/TableDataMap.java | 94 +++++++++++++++++
9 files changed, 495 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java b/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java
new file mode 100644
index 0000000..de02c64
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/events/ChangeEvent.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.events;
+
+/**
+ * Change event for any updates in store.
+ */
+public interface ChangeEvent<T> {
+
+ EventType getEventType();
+
+ T getEventData();
+
+ void setEventData(T data);
+
+ enum EventType {
+ INSERT,DELETE,UPDATE,REFRESH
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/events/EventListener.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/events/EventListener.java b/core/src/main/java/org/apache/carbondata/core/events/EventListener.java
new file mode 100644
index 0000000..adf45b2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/events/EventListener.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.events;
+
+/**
+ * Event listener
+ */
+public interface EventListener {
+
+ void fireEvent(ChangeEvent event);
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
new file mode 100644
index 0000000..597c46c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.Serializable;
+
+/**
+ * Blocklet
+ */
+public class Blocklet implements Serializable {
+
+ private String path;
+
+ private String blockletId;
+
+ public Blocklet(String path, String blockletId) {
+ this.path = path;
+ this.blockletId = blockletId;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public String getBlockletId() {
+ return blockletId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
new file mode 100644
index 0000000..2651f15
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.util.List;
+
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * Interface for adding and retrieving index data.
+ */
+public interface DataMap {
+
+ /**
+ * Give the writer to write the data.
+ *
+ * @return
+ */
+ DataMapWriter getWriter();
+
+ /**
+ * It is called to load the data map to memory or to initialize it.
+ */
+ void init(String path);
+
+ /**
+ * Prune the datamap with filter expression. It returns the list of
+ * blocklets where these filters can exist.
+ *
+ * @param filterExp
+ * @return
+ */
+ List<Blocklet> prune(FilterResolverIntf filterExp);
+
+ /**
+ * Clear complete index table and release memory.
+ */
+ void clear();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java
new file mode 100644
index 0000000..4c379f3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapDistributable.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import org.apache.carbondata.core.datastore.block.Distributable;
+
+/**
+ * Distributable class for datamap.
+ */
+public abstract class DataMapDistributable implements Distributable {
+
+ private String tablePath;
+
+ private String segmentId;
+
+ private String dataMapName;
+
+ public String getTablePath() {
+ return tablePath;
+ }
+
+ public void setTablePath(String tablePath) {
+ this.tablePath = tablePath;
+ }
+
+ public String getSegmentId() {
+ return segmentId;
+ }
+
+ public void setSegmentId(String segmentId) {
+ this.segmentId = segmentId;
+ }
+
+ public String getDataMapName() {
+ return dataMapName;
+ }
+
+ public void setDataMapName(String dataMapName) {
+ this.dataMapName = dataMapName;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
new file mode 100644
index 0000000..06638ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+/**
+ * It maintains all the index tables in it.
+ */
+public class DataMapStoreManager {
+
+ private static DataMapStoreManager instance = new DataMapStoreManager();
+
+ private Map<DataMapType, Map<String, TableDataMap>> dataMapMappping = new HashMap<>();
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(DataMapStoreManager.class.getName());
+
+ private DataMapStoreManager() {
+
+ }
+
+ /**
+ * Get the datamap for reading data.
+ *
+ * @param dataMapName
+ * @param mapType
+ * @return
+ */
+ public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
+ DataMapType mapType) {
+ Map<String, TableDataMap> map = dataMapMappping.get(mapType);
+ TableDataMap dataMap = null;
+ if (map == null) {
+ throw new RuntimeException("Datamap does not exist");
+ } else {
+ dataMap = map.get(dataMapName);
+ if (dataMap == null) {
+ throw new RuntimeException("Datamap does not exist");
+ }
+ }
+ // Initialize datamap
+ dataMap.init(identifier, dataMapName);
+ return dataMap;
+ }
+
+ /**
+ * Create new datamap instance using datamap type and path
+ *
+ * @param mapType
+ * @return
+ */
+ public TableDataMap createTableDataMap(AbsoluteTableIdentifier identifier, DataMapType mapType,
+ String dataMapName) {
+ Map<String, TableDataMap> map = dataMapMappping.get(mapType);
+ if (map == null) {
+ map = new HashMap<>();
+ dataMapMappping.put(mapType, map);
+ }
+ TableDataMap dataMap = map.get(dataMapName);
+ if (dataMap != null) {
+ throw new RuntimeException("Already datamap exists in that path with type " + mapType);
+ }
+
+ try {
+ //TODO create datamap using @mapType.getClassName())
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+ dataMap.init(identifier, dataMapName);
+ map.put(dataMapName, dataMap);
+ return dataMap;
+ }
+
+ public void clearDataMap(String dataMapName, DataMapType mapType) {
+ Map<String, TableDataMap> map = dataMapMappping.get(mapType);
+ if (map != null && map.get(dataMapName) != null) {
+ map.remove(dataMapName).clear();
+ }
+ }
+
+ public static DataMapStoreManager getInstance() {
+ return instance;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
new file mode 100644
index 0000000..b6a0f5b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+/**
+ * Datamap type
+ */
+public enum DataMapType {
+ BLOCKLET("org.apache.carbondata.datamap.BlockletDataMap");
+
+ private String className;
+
+ DataMapType(String className) {
+ this.className = className;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java
new file mode 100644
index 0000000..bd8be09
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.DataOutput;
+
+/**
+ * Data Map writer
+ */
+public interface DataMapWriter<T> {
+
+ /**
+ * Initialize the data map writer with output stream
+ *
+ * @param outStream
+ */
+ void init(DataOutput outStream);
+
+ /**
+ * Add the index row to the in-memory store.
+ */
+ void writeData(T data);
+
+ /**
+ * Get the added row count
+ *
+ * @return
+ */
+ int getRowCount();
+
+ /**
+ * Finish writing of data map table, otherwise it will not be allowed to read.
+ */
+ void finish();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/b4343469/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
new file mode 100644
index 0000000..e1532c8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.util.List;
+
+import org.apache.carbondata.core.events.EventListener;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * DataMap at the table level, user can add any number of datamaps for one table. Depends
+ * on the filter condition it can prune the blocklets.
+ */
+public interface TableDataMap extends EventListener {
+
+ /**
+ * It is called to initialize and load the required table datamap metadata.
+ */
+ void init(AbsoluteTableIdentifier identifier, String dataMapName);
+
+ /**
+ * Gives the writer to write the metadata information of this datamap at table level.
+ *
+ * @return
+ */
+ DataMapWriter getWriter();
+
+ /**
+ * Create the datamap using the segmentid and name.
+ *
+ * @param identifier
+ * @param segmentId
+ * @return
+ */
+ DataMap createDataMap(AbsoluteTableIdentifier identifier, String segmentId);
+
+ /**
+ * Pass the valid segments and prune the datamap using filter expression
+ *
+ * @param segmentIds
+ * @param filterExp
+ * @return
+ */
+ List<Blocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp);
+
+ /**
+ * This is used for making the datamap distributable.
+ * It takes the valid segments and returns all the datamaps as distributable objects so that
+ * it can be distributed across machines.
+ *
+ * @return
+ */
+ List<DataMapDistributable> toDistributable(List<String> segmentIds);
+
+ /**
+ * This method is used from any machine after it is distributed. It takes the distributable object
+ * to prune the filters.
+ *
+ * @param distributable
+ * @param filterExp
+ * @return
+ */
+ List<Blocklet> prune(DataMapDistributable distributable, FilterResolverIntf filterExp);
+
+ /**
+ * This method checks whether the columns and the type of filters supported
+ * for this datamap or not
+ *
+ * @param filterExp
+ * @return
+ */
+ boolean isFiltersSupported(FilterResolverIntf filterExp);
+
+ /**
+ * Clears table level datamap
+ */
+ void clear();
+
+}
[2/2] carbondata git commit: [CARBONDATA-1088] Added interfaces for
Data Map frame work. This closes #958
Posted by ja...@apache.org.
[CARBONDATA-1088] Added interfaces for Data Map frame work. This closes #958
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7bb82203
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7bb82203
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7bb82203
Branch: refs/heads/master
Commit: 7bb8220318207f42d33527948b95196b0cadd5b6
Parents: c3cde5a b434346
Author: jackylk <ja...@huawei.com>
Authored: Fri Jun 16 01:29:12 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jun 16 01:29:12 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/events/ChangeEvent.java | 35 +++++++
.../carbondata/core/events/EventListener.java | 25 +++++
.../carbondata/core/indexstore/Blocklet.java | 42 ++++++++
.../carbondata/core/indexstore/DataMap.java | 54 ++++++++++
.../core/indexstore/DataMapDistributable.java | 56 ++++++++++
.../core/indexstore/DataMapStoreManager.java | 105 +++++++++++++++++++
.../carbondata/core/indexstore/DataMapType.java | 34 ++++++
.../core/indexstore/DataMapWriter.java | 50 +++++++++
.../core/indexstore/TableDataMap.java | 94 +++++++++++++++++
9 files changed, 495 insertions(+)
----------------------------------------------------------------------