You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@carbondata.apache.org by ravipesala <gi...@git.apache.org> on 2018/12/23 19:38:44 UTC
[GitHub] carbondata pull request #3019: [WIP] Carbon Presto hive metastore
GitHub user ravipesala opened a pull request:
https://github.com/apache/carbondata/pull/3019
[WIP] Carbon Presto hive metastore
Be sure to do all of the following checklist to help us incorporate
your contribution quickly and easily:
- [ ] Any interfaces changed?
- [ ] Any backward compatibility impacted?
- [ ] Document update required?
- [ ] Testing done
Please provide details on
- Whether new unit test cases have been added or why no new tests are required?
- How it is tested? Please attach test report.
- Is it a performance related change? Please attach the performance test report.
- Any additional information to help reviewers in testing this change.
- [ ] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA.
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/ravipesala/incubator-carbondata presto-hive-metastore
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/carbondata/pull/3019.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #3019
----
commit 50f7a330e797bcdd71598fbf3145a71fa5df7215
Author: ravipesala <ra...@...>
Date: 2018-12-19T16:30:57Z
Create carbon table as hive metastore table
commit 8621a302ff2f26470b19d31b24a21ff03e4a2831
Author: ravipesala <ra...@...>
Date: 2018-12-19T15:49:41Z
Added Hive Metastore to Carbon Presto
----
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by kumarvishal09 <gi...@git.apache.org>.
Github user kumarvishal09 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244090651
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java ---
@@ -17,69 +17,179 @@
package org.apache.carbondata.presto;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.*;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import static java.util.Objects.requireNonNull;
-import com.facebook.presto.spi.ConnectorHandleResolver;
+import org.apache.carbondata.presto.impl.CarbonTableConfig;
+
+import com.facebook.presto.hive.HiveConnector;
+import com.facebook.presto.hive.HiveConnectorFactory;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveProcedureModule;
+import com.facebook.presto.hive.HiveSchemaProperties;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveStorageFormat;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.NodeVersion;
+import com.facebook.presto.hive.RebindSafeMBeanServer;
+import com.facebook.presto.hive.authentication.HiveAuthenticationModule;
+import com.facebook.presto.hive.metastore.HiveMetastoreModule;
+import com.facebook.presto.hive.s3.HiveS3Module;
+import com.facebook.presto.hive.security.HiveSecurityModule;
+import com.facebook.presto.hive.security.PartitionsAwareAccessControl;
+import com.facebook.presto.spi.NodeManager;
+import com.facebook.presto.spi.PageIndexerFactory;
+import com.facebook.presto.spi.PageSorter;
import com.facebook.presto.spi.classloader.ThreadContextClassLoader;
-import com.facebook.presto.spi.connector.*;
-import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorMetadata;
+import com.facebook.presto.spi.connector.Connector;
+import com.facebook.presto.spi.connector.ConnectorAccessControl;
+import com.facebook.presto.spi.connector.ConnectorContext;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.connector.ConnectorSplitManager;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorSplitManager;
-import com.google.common.base.Throwables;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeNodePartitioningProvider;
+import com.facebook.presto.spi.procedure.Procedure;
+import com.facebook.presto.spi.type.TypeManager;
+import com.google.common.collect.ImmutableSet;
import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
import io.airlift.bootstrap.Bootstrap;
import io.airlift.bootstrap.LifeCycleManager;
+import io.airlift.event.client.EventModule;
import io.airlift.json.JsonModule;
+import io.airlift.units.DataSize;
+import org.weakref.jmx.guice.MBeanModule;
+import sun.reflect.ConstructorAccessor;
+import static com.google.common.base.Throwables.throwIfUnchecked;
+import static io.airlift.configuration.ConfigBinder.configBinder;
/**
* Build Carbondata Connector
* It will be called by CarbondataPlugin
*/
-public class CarbondataConnectorFactory implements ConnectorFactory {
+public class CarbondataConnectorFactory extends HiveConnectorFactory {
private final String name;
--- End diff --
name is not used, please remove
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244278709
--- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala ---
@@ -157,7 +157,7 @@ case class CarbonCreateTableCommand(
| tableName "$tableName",
| dbName "$dbName",
| tablePath "$tablePath",
- | path "$tablePath",
+ | path "${FileFactory.addSchemeIfNotExists(tablePath)}",
--- End diff --
ok
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244254822
--- Diff: integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala ---
@@ -129,6 +130,21 @@ class PrestoServer {
}
}
+ def execute(query: String) = {
+
+ Try {
+ val conn: Connection = createJdbcConnection(dbName)
+ logger.info(s"***** executing the query ***** \n $query")
--- End diff --
rename logger to LOGGER
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/2063/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by qiuchenjian <gi...@git.apache.org>.
Github user qiuchenjian commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r243822523
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java ---
@@ -113,7 +132,7 @@ private PrestoCarbonVectorizedRecordReader createReader(ConnectorSplit split,
PrestoCarbonVectorizedRecordReader reader =
new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel,
(AbstractDetailQueryResultIterator) iterator, readSupport);
- reader.setTaskId(carbondataSplit.getIndex());
+ reader.setTaskId(Long.parseLong(carbonSplit.getSchema().getProperty("index")));
--- End diff --
I think it‘s better to catch the NumberFormatException
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244276046
--- Diff: integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala ---
@@ -80,7 +80,7 @@ object CarbonDataStoreCreator {
UUID.randomUUID().toString))
// val factFilePath: String = new File(dataFilePath).getCanonicalPath
val storeDir: File = new File(absoluteTableIdentifier.getTablePath)
- CarbonUtil.deleteFoldersAndFiles(storeDir)
+// CarbonUtil.deleteFoldersAndFiles(storeDir)
--- End diff --
ok
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244172111
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java ---
@@ -17,69 +17,177 @@
package org.apache.carbondata.presto;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.*;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import static java.util.Objects.requireNonNull;
-import com.facebook.presto.spi.ConnectorHandleResolver;
+import org.apache.carbondata.presto.impl.CarbonTableConfig;
+
+import com.facebook.presto.hive.HiveConnector;
+import com.facebook.presto.hive.HiveConnectorFactory;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveProcedureModule;
+import com.facebook.presto.hive.HiveSchemaProperties;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveStorageFormat;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.NodeVersion;
+import com.facebook.presto.hive.RebindSafeMBeanServer;
+import com.facebook.presto.hive.authentication.HiveAuthenticationModule;
+import com.facebook.presto.hive.metastore.HiveMetastoreModule;
+import com.facebook.presto.hive.s3.HiveS3Module;
+import com.facebook.presto.hive.security.HiveSecurityModule;
+import com.facebook.presto.hive.security.PartitionsAwareAccessControl;
+import com.facebook.presto.spi.NodeManager;
+import com.facebook.presto.spi.PageIndexerFactory;
+import com.facebook.presto.spi.PageSorter;
import com.facebook.presto.spi.classloader.ThreadContextClassLoader;
-import com.facebook.presto.spi.connector.*;
-import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorMetadata;
+import com.facebook.presto.spi.connector.Connector;
+import com.facebook.presto.spi.connector.ConnectorAccessControl;
+import com.facebook.presto.spi.connector.ConnectorContext;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.connector.ConnectorSplitManager;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorSplitManager;
-import com.google.common.base.Throwables;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeNodePartitioningProvider;
+import com.facebook.presto.spi.procedure.Procedure;
+import com.facebook.presto.spi.type.TypeManager;
+import com.google.common.collect.ImmutableSet;
import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
import io.airlift.bootstrap.Bootstrap;
import io.airlift.bootstrap.LifeCycleManager;
+import io.airlift.event.client.EventModule;
import io.airlift.json.JsonModule;
+import io.airlift.units.DataSize;
+import org.weakref.jmx.guice.MBeanModule;
+import sun.reflect.ConstructorAccessor;
+import static com.google.common.base.Throwables.throwIfUnchecked;
+import static io.airlift.configuration.ConfigBinder.configBinder;
/**
* Build Carbondata Connector
* It will be called by CarbondataPlugin
*/
-public class CarbondataConnectorFactory implements ConnectorFactory {
+public class CarbondataConnectorFactory extends HiveConnectorFactory {
- private final String name;
private final ClassLoader classLoader;
public CarbondataConnectorFactory(String connectorName, ClassLoader classLoader) {
- this.name = connectorName;
+ super(connectorName, classLoader, null);
this.classLoader = requireNonNull(classLoader, "classLoader is null");
}
- @Override public String getName() {
- return name;
- }
-
- @Override public ConnectorHandleResolver getHandleResolver() {
- return new CarbondataHandleResolver();
- }
-
- @Override public Connector create(String connectorId, Map<String, String> config,
+ @Override public Connector create(String catalogName, Map<String, String> config,
ConnectorContext context) {
requireNonNull(config, "config is null");
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) {
- Bootstrap app = new Bootstrap(new JsonModule(),
- new CarbondataModule(connectorId, context.getTypeManager()));
+ Bootstrap app = new Bootstrap(new EventModule(), new MBeanModule(), new JsonModule(),
+ new CarbondataModule(catalogName), new HiveS3Module(catalogName),
+ new HiveMetastoreModule(catalogName, Optional.ofNullable(null)), new HiveSecurityModule(),
+ new HiveAuthenticationModule(), new HiveProcedureModule(), binder -> {
+ javax.management.MBeanServer platformMBeanServer =
+ ManagementFactory.getPlatformMBeanServer();
+ binder.bind(javax.management.MBeanServer.class)
+ .toInstance(new RebindSafeMBeanServer(platformMBeanServer));
+ binder.bind(NodeVersion.class)
+ .toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion()));
+ binder.bind(NodeManager.class).toInstance(context.getNodeManager());
+ binder.bind(TypeManager.class).toInstance(context.getTypeManager());
+ binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory());
+ binder.bind(PageSorter.class).toInstance(context.getPageSorter());
+ configBinder(binder).bindConfig(CarbonTableConfig.class);
+ });
Injector injector =
app.strictConfig().doNotInitializeLogging().setRequiredConfigurationProperties(config)
.initialize();
+ setCarbonEnum();
+
LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
- ConnectorMetadata metadata = injector.getInstance(CarbondataMetadata.class);
+ HiveMetadataFactory metadataFactory = injector.getInstance(HiveMetadataFactory.class);
+ HiveTransactionManager transactionManager =
+ injector.getInstance(HiveTransactionManager.class);
ConnectorSplitManager splitManager = injector.getInstance(ConnectorSplitManager.class);
ConnectorPageSourceProvider connectorPageSource =
injector.getInstance(ConnectorPageSourceProvider.class);
+ ConnectorPageSinkProvider pageSinkProvider =
+ injector.getInstance(ConnectorPageSinkProvider.class);
+ ConnectorNodePartitioningProvider connectorDistributionProvider =
+ injector.getInstance(ConnectorNodePartitioningProvider.class);
+ HiveSessionProperties hiveSessionProperties =
+ injector.getInstance(HiveSessionProperties.class);
+ HiveTableProperties hiveTableProperties = injector.getInstance(HiveTableProperties.class);
+ ConnectorAccessControl accessControl =
+ new PartitionsAwareAccessControl(injector.getInstance(ConnectorAccessControl.class));
+ Set<Procedure> procedures = injector.getInstance(Key.get(new TypeLiteral<Set<Procedure>>() {
+ }));
- return new CarbondataConnector(lifeCycleManager,
- new ClassLoaderSafeConnectorMetadata(metadata, classLoader),
- new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader), classLoader,
- new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader));
+ return new HiveConnector(lifeCycleManager, metadataFactory, transactionManager,
+ new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader),
+ new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader),
+ new ClassLoaderSafeConnectorPageSinkProvider(pageSinkProvider, classLoader),
+ new ClassLoaderSafeNodePartitioningProvider(connectorDistributionProvider, classLoader),
+ ImmutableSet.of(), procedures, hiveSessionProperties.getSessionProperties(),
+ HiveSchemaProperties.SCHEMA_PROPERTIES, hiveTableProperties.getTableProperties(),
+ accessControl, classLoader);
} catch (Exception e) {
- throw Throwables.propagate(e);
+ throwIfUnchecked(e);
+ throw new RuntimeException(e);
}
}
+
+ /**
+ * Set the Carbon format enum to HiveStorageFormat, its a hack but for time being it is best
+ * choice to avoid lot of code change.
+ *
+ * @throws Exception
+ */
+ private void setCarbonEnum() throws Exception {
+ for (HiveStorageFormat format : HiveStorageFormat.values()) {
+ if (format.name().equals("CARBON")) {
+ return;
+ }
+ }
+ Constructor<?>[] declaredConstructors = HiveStorageFormat.class.getDeclaredConstructors();
+ declaredConstructors[0].setAccessible(true);
+ Field constructorAccessorField = Constructor.class.getDeclaredField("constructorAccessor");
+ constructorAccessorField.setAccessible(true);
+ ConstructorAccessor ca =
+ (ConstructorAccessor) constructorAccessorField.get(declaredConstructors[0]);
+ if (ca == null) {
+ Method acquireConstructorAccessorMethod =
+ Constructor.class.getDeclaredMethod("acquireConstructorAccessor");
+ acquireConstructorAccessorMethod.setAccessible(true);
+ ca = (ConstructorAccessor) acquireConstructorAccessorMethod.invoke(declaredConstructors[0]);
+ }
+ Object instance = ca.newInstance(new Object[] { "CARBON", HiveStorageFormat.values().length, "",
+ "org.apache.carbondata.hadoop.api.CarbonFileInputFormat",
--- End diff --
please add assertion for serdes and outputformat class also
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1931/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10174/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244097200
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java ---
@@ -17,62 +17,150 @@
package org.apache.carbondata.presto;
-import javax.inject.Inject;
+import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
-import org.apache.carbondata.presto.impl.CarbonTableConfig;
import org.apache.carbondata.presto.impl.CarbonTableReader;
+import com.facebook.presto.hive.CoercionPolicy;
+import com.facebook.presto.hive.DirectoryLister;
+import com.facebook.presto.hive.FileFormatDataSourceStats;
+import com.facebook.presto.hive.GenericHiveRecordCursorProvider;
+import com.facebook.presto.hive.HadoopDirectoryLister;
+import com.facebook.presto.hive.HdfsConfiguration;
+import com.facebook.presto.hive.HdfsConfigurationUpdater;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveClientModule;
+import com.facebook.presto.hive.HiveCoercionPolicy;
+import com.facebook.presto.hive.HiveConnectorId;
+import com.facebook.presto.hive.HiveEventClient;
+import com.facebook.presto.hive.HiveFileWriterFactory;
+import com.facebook.presto.hive.HiveHdfsConfiguration;
+import com.facebook.presto.hive.HiveLocationService;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveNodePartitioningProvider;
+import com.facebook.presto.hive.HivePageSinkProvider;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePartitionManager;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveSplitManager;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.HiveTypeTranslator;
+import com.facebook.presto.hive.HiveWriterStats;
+import com.facebook.presto.hive.LocationService;
+import com.facebook.presto.hive.NamenodeStats;
+import com.facebook.presto.hive.OrcFileWriterConfig;
+import com.facebook.presto.hive.OrcFileWriterFactory;
+import com.facebook.presto.hive.PartitionUpdate;
+import com.facebook.presto.hive.RcFileFileWriterFactory;
+import com.facebook.presto.hive.TableParameterCodec;
+import com.facebook.presto.hive.TransactionalMetadata;
+import com.facebook.presto.hive.TypeTranslator;
+import com.facebook.presto.hive.orc.DwrfPageSourceFactory;
+import com.facebook.presto.hive.orc.OrcPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetRecordCursorProvider;
+import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.ConnectorSplitManager;
-import com.facebook.presto.spi.type.Type;
-import com.facebook.presto.spi.type.TypeManager;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
import com.google.inject.Binder;
-import com.google.inject.Module;
import com.google.inject.Scopes;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
+import io.airlift.event.client.EventClient;
-import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
-import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.inject.multibindings.Multibinder.newSetBinder;
import static io.airlift.configuration.ConfigBinder.configBinder;
+import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
-public class CarbondataModule implements Module {
+import static org.weakref.jmx.ObjectNames.generatedNameOf;
+import static org.weakref.jmx.guice.ExportBinder.newExporter;
+
+public class CarbondataModule extends HiveClientModule {
private final String connectorId;
- private final TypeManager typeManager;
- public CarbondataModule(String connectorId, TypeManager typeManager) {
+ public CarbondataModule(String connectorId) {
+ super(connectorId);
this.connectorId = requireNonNull(connectorId, "connector id is null");
- this.typeManager = requireNonNull(typeManager, "typeManager is null");
}
@Override public void configure(Binder binder) {
- binder.bind(TypeManager.class).toInstance(typeManager);
+ binder.bind(HiveConnectorId.class).toInstance(new HiveConnectorId(connectorId));
+ binder.bind(TypeTranslator.class).toInstance(new HiveTypeTranslator());
+ binder.bind(CoercionPolicy.class).to(HiveCoercionPolicy.class).in(Scopes.SINGLETON);
- binder.bind(CarbondataConnectorId.class).toInstance(new CarbondataConnectorId(connectorId));
- binder.bind(CarbondataMetadata.class).in(Scopes.SINGLETON);
- binder.bind(CarbonTableReader.class).in(Scopes.SINGLETON);
+ binder.bind(HdfsConfigurationUpdater.class).in(Scopes.SINGLETON);
+ binder.bind(HdfsConfiguration.class).to(HiveHdfsConfiguration.class).in(Scopes.SINGLETON);
+ binder.bind(HdfsEnvironment.class).in(Scopes.SINGLETON);
+ binder.bind(DirectoryLister.class).to(HadoopDirectoryLister.class).in(Scopes.SINGLETON);
+ configBinder(binder).bindConfig(HiveClientConfig.class);
+
+ binder.bind(HiveSessionProperties.class).in(Scopes.SINGLETON);
+ binder.bind(HiveTableProperties.class).in(Scopes.SINGLETON);
+
+ binder.bind(NamenodeStats.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(NamenodeStats.class)
+ .as(generatedNameOf(NamenodeStats.class, connectorId));
+
+ Multibinder<HiveRecordCursorProvider> recordCursorProviderBinder =
+ newSetBinder(binder, HiveRecordCursorProvider.class);
+ recordCursorProviderBinder.addBinding().to(ParquetRecordCursorProvider.class)
+ .in(Scopes.SINGLETON);
+ recordCursorProviderBinder.addBinding().to(GenericHiveRecordCursorProvider.class)
+ .in(Scopes.SINGLETON);
+
+ binder.bind(HiveWriterStats.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(HiveWriterStats.class)
+ .as(generatedNameOf(HiveWriterStats.class, connectorId));
+
+ newSetBinder(binder, EventClient.class).addBinding().to(HiveEventClient.class)
+ .in(Scopes.SINGLETON);
+ binder.bind(HivePartitionManager.class).in(Scopes.SINGLETON);
+ binder.bind(LocationService.class).to(HiveLocationService.class).in(Scopes.SINGLETON);
+ binder.bind(TableParameterCodec.class).in(Scopes.SINGLETON);
+ binder.bind(HiveMetadataFactory.class).in(Scopes.SINGLETON);
+ binder.bind(new TypeLiteral<Supplier<TransactionalMetadata>>() {
+ }).to(HiveMetadataFactory.class).in(Scopes.SINGLETON);
+ binder.bind(HiveTransactionManager.class).in(Scopes.SINGLETON);
binder.bind(ConnectorSplitManager.class).to(CarbondataSplitManager.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(ConnectorSplitManager.class)
+ .as(generatedNameOf(HiveSplitManager.class, connectorId));
binder.bind(ConnectorPageSourceProvider.class).to(CarbondataPageSourceProvider.class)
.in(Scopes.SINGLETON);
- binder.bind(CarbondataHandleResolver.class).in(Scopes.SINGLETON);
- configBinder(binder).bindConfig(CarbonTableConfig.class);
- }
+ binder.bind(ConnectorPageSinkProvider.class).to(HivePageSinkProvider.class)
+ .in(Scopes.SINGLETON);
+ binder.bind(ConnectorNodePartitioningProvider.class).to(HiveNodePartitioningProvider.class)
+ .in(Scopes.SINGLETON);
+
+ jsonCodecBinder(binder).bindJsonCodec(PartitionUpdate.class);
- public static final class TypeDeserializer extends FromStringDeserializer<Type> {
- private final TypeManager typeManager;
+ binder.bind(FileFormatDataSourceStats.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(FileFormatDataSourceStats.class)
+ .as(generatedNameOf(FileFormatDataSourceStats.class, connectorId));
- @Inject public TypeDeserializer(TypeManager typeManager) {
- super(Type.class);
- this.typeManager = requireNonNull(typeManager, "typeManager is null");
- }
+ Multibinder<HivePageSourceFactory> pageSourceFactoryBinder =
+ newSetBinder(binder, HivePageSourceFactory.class);
+ pageSourceFactoryBinder.addBinding().to(OrcPageSourceFactory.class).in(Scopes.SINGLETON);
+ pageSourceFactoryBinder.addBinding().to(DwrfPageSourceFactory.class).in(Scopes.SINGLETON);
+ pageSourceFactoryBinder.addBinding().to(ParquetPageSourceFactory.class).in(Scopes.SINGLETON);
--- End diff --
It is needed as it can read all formats from carbon connector itself
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244278116
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java ---
@@ -17,62 +17,150 @@
package org.apache.carbondata.presto;
-import javax.inject.Inject;
+import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
-import org.apache.carbondata.presto.impl.CarbonTableConfig;
import org.apache.carbondata.presto.impl.CarbonTableReader;
+import com.facebook.presto.hive.CoercionPolicy;
+import com.facebook.presto.hive.DirectoryLister;
+import com.facebook.presto.hive.FileFormatDataSourceStats;
+import com.facebook.presto.hive.GenericHiveRecordCursorProvider;
+import com.facebook.presto.hive.HadoopDirectoryLister;
+import com.facebook.presto.hive.HdfsConfiguration;
+import com.facebook.presto.hive.HdfsConfigurationUpdater;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveClientModule;
+import com.facebook.presto.hive.HiveCoercionPolicy;
+import com.facebook.presto.hive.HiveConnectorId;
+import com.facebook.presto.hive.HiveEventClient;
+import com.facebook.presto.hive.HiveFileWriterFactory;
+import com.facebook.presto.hive.HiveHdfsConfiguration;
+import com.facebook.presto.hive.HiveLocationService;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveNodePartitioningProvider;
+import com.facebook.presto.hive.HivePageSinkProvider;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePartitionManager;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveSplitManager;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.HiveTypeTranslator;
+import com.facebook.presto.hive.HiveWriterStats;
+import com.facebook.presto.hive.LocationService;
+import com.facebook.presto.hive.NamenodeStats;
+import com.facebook.presto.hive.OrcFileWriterConfig;
+import com.facebook.presto.hive.OrcFileWriterFactory;
+import com.facebook.presto.hive.PartitionUpdate;
+import com.facebook.presto.hive.RcFileFileWriterFactory;
+import com.facebook.presto.hive.TableParameterCodec;
+import com.facebook.presto.hive.TransactionalMetadata;
+import com.facebook.presto.hive.TypeTranslator;
+import com.facebook.presto.hive.orc.DwrfPageSourceFactory;
+import com.facebook.presto.hive.orc.OrcPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetRecordCursorProvider;
+import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.ConnectorSplitManager;
-import com.facebook.presto.spi.type.Type;
-import com.facebook.presto.spi.type.TypeManager;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
import com.google.inject.Binder;
-import com.google.inject.Module;
import com.google.inject.Scopes;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
+import io.airlift.event.client.EventClient;
-import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
-import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.inject.multibindings.Multibinder.newSetBinder;
import static io.airlift.configuration.ConfigBinder.configBinder;
+import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
-public class CarbondataModule implements Module {
+import static org.weakref.jmx.ObjectNames.generatedNameOf;
+import static org.weakref.jmx.guice.ExportBinder.newExporter;
+
+public class CarbondataModule extends HiveClientModule {
--- End diff --
ok
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/3019
@chenliang613 if we are planning to contribute to Presto then it would be under presto-hive connector, it is just like how ORC, parquet and other formats supported under this connector. It would be much simpler and lesser code than the current implementation. Now we are overriding HiveConnector to add our carbon format to it, if we change presto-hive connector to add carbon format then it would be adding of read/write factories to it would be enough.
CarbonData cannot be another individual connector on its own as it lacks features like metadata , authentication etc which are readily available in presto-hive connector. That is the reason we are overriding HiveConnector.
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244269584
--- Diff: integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonCreateTableCommand.scala ---
@@ -157,7 +157,7 @@ case class CarbonCreateTableCommand(
| tableName "$tableName",
| dbName "$dbName",
| tablePath "$tablePath",
- | path "$tablePath",
+ | path "${FileFactory.addSchemeIfNotExists(tablePath)}",
--- End diff --
Can you update the comment from line 147 to 150. I feel it need to be rephrased
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244269435
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java ---
@@ -17,62 +17,150 @@
package org.apache.carbondata.presto;
-import javax.inject.Inject;
+import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
-import org.apache.carbondata.presto.impl.CarbonTableConfig;
import org.apache.carbondata.presto.impl.CarbonTableReader;
+import com.facebook.presto.hive.CoercionPolicy;
+import com.facebook.presto.hive.DirectoryLister;
+import com.facebook.presto.hive.FileFormatDataSourceStats;
+import com.facebook.presto.hive.GenericHiveRecordCursorProvider;
+import com.facebook.presto.hive.HadoopDirectoryLister;
+import com.facebook.presto.hive.HdfsConfiguration;
+import com.facebook.presto.hive.HdfsConfigurationUpdater;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveClientModule;
+import com.facebook.presto.hive.HiveCoercionPolicy;
+import com.facebook.presto.hive.HiveConnectorId;
+import com.facebook.presto.hive.HiveEventClient;
+import com.facebook.presto.hive.HiveFileWriterFactory;
+import com.facebook.presto.hive.HiveHdfsConfiguration;
+import com.facebook.presto.hive.HiveLocationService;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveNodePartitioningProvider;
+import com.facebook.presto.hive.HivePageSinkProvider;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePartitionManager;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveSplitManager;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.HiveTypeTranslator;
+import com.facebook.presto.hive.HiveWriterStats;
+import com.facebook.presto.hive.LocationService;
+import com.facebook.presto.hive.NamenodeStats;
+import com.facebook.presto.hive.OrcFileWriterConfig;
+import com.facebook.presto.hive.OrcFileWriterFactory;
+import com.facebook.presto.hive.PartitionUpdate;
+import com.facebook.presto.hive.RcFileFileWriterFactory;
+import com.facebook.presto.hive.TableParameterCodec;
+import com.facebook.presto.hive.TransactionalMetadata;
+import com.facebook.presto.hive.TypeTranslator;
+import com.facebook.presto.hive.orc.DwrfPageSourceFactory;
+import com.facebook.presto.hive.orc.OrcPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetRecordCursorProvider;
+import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.ConnectorSplitManager;
-import com.facebook.presto.spi.type.Type;
-import com.facebook.presto.spi.type.TypeManager;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
import com.google.inject.Binder;
-import com.google.inject.Module;
import com.google.inject.Scopes;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
+import io.airlift.event.client.EventClient;
-import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
-import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.inject.multibindings.Multibinder.newSetBinder;
import static io.airlift.configuration.ConfigBinder.configBinder;
+import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
-public class CarbondataModule implements Module {
+import static org.weakref.jmx.ObjectNames.generatedNameOf;
+import static org.weakref.jmx.guice.ExportBinder.newExporter;
+
+public class CarbondataModule extends HiveClientModule {
--- End diff --
Please add comment for this class
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244269478
--- Diff: core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java ---
@@ -369,6 +369,24 @@ public static boolean createNewLockFile(String filePath, FileType fileType) thro
LOCAL, HDFS, ALLUXIO, VIEWFS, S3
}
+ public static String addSchemeIfNotExists(String filePath) {
--- End diff --
add comment
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244254924
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java ---
@@ -78,32 +72,33 @@
private static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__";
/**
- * @param carbondataColumnHandle
+ * @param columnHandle
* @return
*/
- private static DataType spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) {
- Type colType = carbondataColumnHandle.getColumnType();
- if (colType == BooleanType.BOOLEAN) {
+ private static DataType spi2CarbondataTypeMapper(HiveColumnHandle columnHandle) {
+ HiveType colType = columnHandle.getHiveType();
+ if (colType.equals(HiveType.HIVE_BOOLEAN)) {
return DataTypes.BOOLEAN;
- } else if (colType == SmallintType.SMALLINT) {
+ } else if (colType.equals(HiveType.HIVE_SHORT)) {
return DataTypes.SHORT;
- } else if (colType == IntegerType.INTEGER) {
+ } else if (colType.equals(HiveType.HIVE_INT)) {
return DataTypes.INT;
- } else if (colType == BigintType.BIGINT) {
+ } else if (colType.equals(HiveType.HIVE_LONG)) {
return DataTypes.LONG;
- } else if (colType == DoubleType.DOUBLE) {
+ } else if (colType.equals(HiveType.HIVE_DOUBLE)) {
return DataTypes.DOUBLE;
- } else if (colType == VarcharType.VARCHAR) {
+ } else if (colType.equals(HiveType.HIVE_STRING)) {
return DataTypes.STRING;
- } else if (colType == DateType.DATE) {
+ } else if (colType.equals(HiveType.HIVE_DATE)) {
return DataTypes.DATE;
- } else if (colType == TimestampType.TIMESTAMP) {
+ } else if (colType.equals(HiveType.HIVE_TIMESTAMP)) {
return DataTypes.TIMESTAMP;
- } else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale()))) {
- return DataTypes.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale());
- } else {
+ }
+ else if (colType.getTypeInfo() instanceof DecimalTypeInfo) {
+ DecimalTypeInfo typeInfo = (DecimalTypeInfo) colType.getTypeInfo();
+ return DataTypes.createDecimalType(typeInfo.getPrecision(),typeInfo.getScale());
+ }
+ else {
--- End diff --
move up
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1924/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1955/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244254915
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java ---
@@ -78,32 +72,33 @@
private static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__";
/**
- * @param carbondataColumnHandle
+ * @param columnHandle
* @return
*/
- private static DataType spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) {
- Type colType = carbondataColumnHandle.getColumnType();
- if (colType == BooleanType.BOOLEAN) {
+ private static DataType spi2CarbondataTypeMapper(HiveColumnHandle columnHandle) {
+ HiveType colType = columnHandle.getHiveType();
+ if (colType.equals(HiveType.HIVE_BOOLEAN)) {
return DataTypes.BOOLEAN;
- } else if (colType == SmallintType.SMALLINT) {
+ } else if (colType.equals(HiveType.HIVE_SHORT)) {
return DataTypes.SHORT;
- } else if (colType == IntegerType.INTEGER) {
+ } else if (colType.equals(HiveType.HIVE_INT)) {
return DataTypes.INT;
- } else if (colType == BigintType.BIGINT) {
+ } else if (colType.equals(HiveType.HIVE_LONG)) {
return DataTypes.LONG;
- } else if (colType == DoubleType.DOUBLE) {
+ } else if (colType.equals(HiveType.HIVE_DOUBLE)) {
return DataTypes.DOUBLE;
- } else if (colType == VarcharType.VARCHAR) {
+ } else if (colType.equals(HiveType.HIVE_STRING)) {
return DataTypes.STRING;
- } else if (colType == DateType.DATE) {
+ } else if (colType.equals(HiveType.HIVE_DATE)) {
return DataTypes.DATE;
- } else if (colType == TimestampType.TIMESTAMP) {
+ } else if (colType.equals(HiveType.HIVE_TIMESTAMP)) {
return DataTypes.TIMESTAMP;
- } else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale()))) {
- return DataTypes.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale());
- } else {
+ }
+ else if (colType.getTypeInfo() instanceof DecimalTypeInfo) {
--- End diff --
move up
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on the issue:
https://github.com/apache/carbondata/pull/3019
LGTM
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244276202
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java ---
@@ -78,32 +72,33 @@
private static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__";
/**
- * @param carbondataColumnHandle
+ * @param columnHandle
* @return
*/
- private static DataType spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) {
- Type colType = carbondataColumnHandle.getColumnType();
- if (colType == BooleanType.BOOLEAN) {
+ private static DataType spi2CarbondataTypeMapper(HiveColumnHandle columnHandle) {
+ HiveType colType = columnHandle.getHiveType();
+ if (colType.equals(HiveType.HIVE_BOOLEAN)) {
return DataTypes.BOOLEAN;
- } else if (colType == SmallintType.SMALLINT) {
+ } else if (colType.equals(HiveType.HIVE_SHORT)) {
return DataTypes.SHORT;
- } else if (colType == IntegerType.INTEGER) {
+ } else if (colType.equals(HiveType.HIVE_INT)) {
return DataTypes.INT;
- } else if (colType == BigintType.BIGINT) {
+ } else if (colType.equals(HiveType.HIVE_LONG)) {
return DataTypes.LONG;
- } else if (colType == DoubleType.DOUBLE) {
+ } else if (colType.equals(HiveType.HIVE_DOUBLE)) {
return DataTypes.DOUBLE;
- } else if (colType == VarcharType.VARCHAR) {
+ } else if (colType.equals(HiveType.HIVE_STRING)) {
return DataTypes.STRING;
- } else if (colType == DateType.DATE) {
+ } else if (colType.equals(HiveType.HIVE_DATE)) {
return DataTypes.DATE;
- } else if (colType == TimestampType.TIMESTAMP) {
+ } else if (colType.equals(HiveType.HIVE_TIMESTAMP)) {
return DataTypes.TIMESTAMP;
- } else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale()))) {
- return DataTypes.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale());
- } else {
+ }
+ else if (colType.getTypeInfo() instanceof DecimalTypeInfo) {
--- End diff --
ok
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/2070/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by qiuchenjian <gi...@git.apache.org>.
Github user qiuchenjian commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r243821704
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java ---
@@ -43,63 +44,81 @@
import static org.apache.carbondata.presto.Types.checkType;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveColumnHandle;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePageSourceProvider;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSplit;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorPageSource;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.ConnectorSplit;
-import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
+import com.facebook.presto.spi.type.TypeManager;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskAttemptContextImpl;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
-
/**
* Provider Class for Carbondata Page Source class.
*/
-public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider {
+public class CarbondataPageSourceProvider extends HivePageSourceProvider {
- private String connectorId;
private CarbonTableReader carbonTableReader;
private String queryId ;
-
- @Inject public CarbondataPageSourceProvider(CarbondataConnectorId connectorId,
+ private HdfsEnvironment hdfsEnvironment;
+
+ @Inject public CarbondataPageSourceProvider(
+ HiveClientConfig hiveClientConfig,
+ HdfsEnvironment hdfsEnvironment,
+ Set<HiveRecordCursorProvider> cursorProviders,
+ Set<HivePageSourceFactory> pageSourceFactories,
+ TypeManager typeManager,
CarbonTableReader carbonTableReader) {
- this.connectorId = requireNonNull(connectorId, "connectorId is null").toString();
+ super(hiveClientConfig, hdfsEnvironment, cursorProviders, pageSourceFactories, typeManager);
this.carbonTableReader = requireNonNull(carbonTableReader, "carbonTableReader is null");
+ this.hdfsEnvironment = hdfsEnvironment;
}
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle,
ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns) {
- this.queryId = ((CarbondataSplit)split).getQueryId();
+ HiveSplit carbonSplit =
+ checkType(split, HiveSplit.class, "split is not class HiveSplit");
+ if (carbonSplit.getSchema().getProperty("queryId") == null) {
+ return super.createPageSource(transactionHandle, session, split, columns);
+ }
+ this.queryId = carbonSplit.getSchema().getProperty("queryId");
+ Configuration configuration = this.hdfsEnvironment.getConfiguration(
+ new HdfsEnvironment.HdfsContext(session, carbonSplit.getDatabase(), carbonSplit.getTable()),
+ new Path(carbonSplit.getSchema().getProperty("tablePath")));
+ configuration = carbonTableReader.updateS3Properties(configuration);
CarbonDictionaryDecodeReadSupport readSupport = new CarbonDictionaryDecodeReadSupport();
PrestoCarbonVectorizedRecordReader carbonRecordReader =
- createReader(split, columns, readSupport);
+ createReader(carbonSplit, columns, readSupport, configuration);
return new CarbondataPageSource(carbonRecordReader, columns);
}
/**
- * @param split
+ * @param carbonSplit
* @param columns
* @param readSupport
--- End diff --
missing param of conf
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244097115
--- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala ---
@@ -72,69 +74,107 @@ object CarbonSessionExample {
val path = s"$rootPath/examples/spark2/src/main/resources/data.csv"
// scalastyle:off
- spark.sql(
- s"""
- | LOAD DATA LOCAL INPATH '$path'
- | INTO TABLE source
- | OPTIONS('HEADER'='true', 'COMPLEX_DELIMITER_LEVEL_1'='#')
- """.stripMargin)
- // scalastyle:on
-
- spark.sql(
- s"""
- | SELECT charField, stringField, intField
- | FROM source
- | WHERE stringfield = 'spark' AND decimalField > 40
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE length(stringField) = 5
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """.stripMargin).show()
-
- spark.sql("SELECT count(stringField) FROM source").show()
-
- spark.sql(
- s"""
- | SELECT sum(intField), stringField
- | FROM source
- | GROUP BY stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT t1.*, t2.*
- | FROM source t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | WITH t1 AS (
- | SELECT * FROM source
- | UNION ALL
- | SELECT * FROM source
- | )
- | SELECT t1.*, t2.*
- | FROM t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source
- | WHERE stringField = 'spark' and floatField > 2.8
- """.stripMargin).show()
+// spark.sql(
--- End diff --
done
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10177/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10184/
---
[GitHub] carbondata issue #3019: [WIP] Carbon Presto hive metastore
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1915/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10305/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244097233
--- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala ---
@@ -72,69 +74,107 @@ object CarbonSessionExample {
val path = s"$rootPath/examples/spark2/src/main/resources/data.csv"
// scalastyle:off
- spark.sql(
- s"""
- | LOAD DATA LOCAL INPATH '$path'
- | INTO TABLE source
- | OPTIONS('HEADER'='true', 'COMPLEX_DELIMITER_LEVEL_1'='#')
- """.stripMargin)
- // scalastyle:on
-
- spark.sql(
- s"""
- | SELECT charField, stringField, intField
- | FROM source
- | WHERE stringfield = 'spark' AND decimalField > 40
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE length(stringField) = 5
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """.stripMargin).show()
-
- spark.sql("SELECT count(stringField) FROM source").show()
-
- spark.sql(
- s"""
- | SELECT sum(intField), stringField
- | FROM source
- | GROUP BY stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT t1.*, t2.*
- | FROM source t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | WITH t1 AS (
- | SELECT * FROM source
- | UNION ALL
- | SELECT * FROM source
- | )
- | SELECT t1.*, t2.*
- | FROM t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source
- | WHERE stringField = 'spark' and floatField > 2.8
- """.stripMargin).show()
+// spark.sql(
+// s"""
+// | LOAD DATA LOCAL INPATH '$path'
+// | INTO TABLE source
+// | OPTIONS('HEADER'='true', 'COMPLEX_DELIMITER_LEVEL_1'='#')
+// """.stripMargin)
+// // scalastyle:on
+//
+// spark.sql(
+// s"""
+// | CREATE TABLE source_cs(
+// | shortField SHORT,
+// | intField INT,
+// | bigintField LONG,
+// | doubleField DOUBLE,
+// | stringField STRING,
+// | timestampField TIMESTAMP,
+// | decimalField DECIMAL(18,2),
+// | dateField DATE,
+// | charField CHAR(5),
+// | floatField FLOAT
+// | )
+// | using carbon
+// | location 'file://${ExampleUtils.storeLocation}'
+// """.stripMargin)
+//
+// spark.sql("insert into source_cs select * from source")
+//
+// spark.sql(
+// s"""
+// | CREATE TABLE source_par(
+// | shortField SHORT,
+// | intField INT,
+// | bigintField LONG,
+// | doubleField DOUBLE,
+// | stringField STRING,
+// | timestampField TIMESTAMP,
+// | decimalField DECIMAL(18,2),
+// | dateField DATE,
+// | charField CHAR(5),
+// | floatField FLOAT
+// | )
+// | using parquet
+// """.stripMargin)
+//
+// spark.sql("insert into source_par select * from source")
+// spark.sql(
+// s"""
+// | SELECT charField, stringField, intField
+// | FROM source
+// | WHERE stringfield = 'spark' AND decimalField > 40
+// """.stripMargin).show()
+//
+// spark.sql(
+// s"""
+// | SELECT *
+// | FROM source WHERE length(stringField) = 5
+// """.stripMargin).show()
+//
+// spark.sql(
+// s"""
+// | SELECT *
+// | FROM source WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+// """.stripMargin).show()
+//
+// spark.sql("SELECT count(stringField) FROM source").show()
+//
+// spark.sql(
+// s"""
+// | SELECT sum(intField), stringField
+// | FROM source
+// | GROUP BY stringField
+// """.stripMargin).show()
+//
+// spark.sql(
+// s"""
--- End diff --
Reverted
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10324/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244276214
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java ---
@@ -78,32 +72,33 @@
private static final String HIVE_DEFAULT_DYNAMIC_PARTITION = "__HIVE_DEFAULT_PARTITION__";
/**
- * @param carbondataColumnHandle
+ * @param columnHandle
* @return
*/
- private static DataType spi2CarbondataTypeMapper(CarbondataColumnHandle carbondataColumnHandle) {
- Type colType = carbondataColumnHandle.getColumnType();
- if (colType == BooleanType.BOOLEAN) {
+ private static DataType spi2CarbondataTypeMapper(HiveColumnHandle columnHandle) {
+ HiveType colType = columnHandle.getHiveType();
+ if (colType.equals(HiveType.HIVE_BOOLEAN)) {
return DataTypes.BOOLEAN;
- } else if (colType == SmallintType.SMALLINT) {
+ } else if (colType.equals(HiveType.HIVE_SHORT)) {
return DataTypes.SHORT;
- } else if (colType == IntegerType.INTEGER) {
+ } else if (colType.equals(HiveType.HIVE_INT)) {
return DataTypes.INT;
- } else if (colType == BigintType.BIGINT) {
+ } else if (colType.equals(HiveType.HIVE_LONG)) {
return DataTypes.LONG;
- } else if (colType == DoubleType.DOUBLE) {
+ } else if (colType.equals(HiveType.HIVE_DOUBLE)) {
return DataTypes.DOUBLE;
- } else if (colType == VarcharType.VARCHAR) {
+ } else if (colType.equals(HiveType.HIVE_STRING)) {
return DataTypes.STRING;
- } else if (colType == DateType.DATE) {
+ } else if (colType.equals(HiveType.HIVE_DATE)) {
return DataTypes.DATE;
- } else if (colType == TimestampType.TIMESTAMP) {
+ } else if (colType.equals(HiveType.HIVE_TIMESTAMP)) {
return DataTypes.TIMESTAMP;
- } else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale()))) {
- return DataTypes.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale());
- } else {
+ }
+ else if (colType.getTypeInfo() instanceof DecimalTypeInfo) {
+ DecimalTypeInfo typeInfo = (DecimalTypeInfo) colType.getTypeInfo();
+ return DataTypes.createDecimalType(typeInfo.getPrecision(),typeInfo.getScale());
+ }
+ else {
--- End diff --
ok
---
[GitHub] carbondata issue #3019: [WIP] Carbon Presto hive metastore
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1920/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244278266
--- Diff: core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java ---
@@ -369,6 +369,24 @@ public static boolean createNewLockFile(String filePath, FileType fileType) thro
LOCAL, HDFS, ALLUXIO, VIEWFS, S3
}
+ public static String addSchemeIfNotExists(String filePath) {
--- End diff --
ok
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ajantha-bhat <gi...@git.apache.org>.
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r243849816
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataModule.java ---
@@ -17,62 +17,150 @@
package org.apache.carbondata.presto;
-import javax.inject.Inject;
+import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
-import org.apache.carbondata.presto.impl.CarbonTableConfig;
import org.apache.carbondata.presto.impl.CarbonTableReader;
+import com.facebook.presto.hive.CoercionPolicy;
+import com.facebook.presto.hive.DirectoryLister;
+import com.facebook.presto.hive.FileFormatDataSourceStats;
+import com.facebook.presto.hive.GenericHiveRecordCursorProvider;
+import com.facebook.presto.hive.HadoopDirectoryLister;
+import com.facebook.presto.hive.HdfsConfiguration;
+import com.facebook.presto.hive.HdfsConfigurationUpdater;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveClientModule;
+import com.facebook.presto.hive.HiveCoercionPolicy;
+import com.facebook.presto.hive.HiveConnectorId;
+import com.facebook.presto.hive.HiveEventClient;
+import com.facebook.presto.hive.HiveFileWriterFactory;
+import com.facebook.presto.hive.HiveHdfsConfiguration;
+import com.facebook.presto.hive.HiveLocationService;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveNodePartitioningProvider;
+import com.facebook.presto.hive.HivePageSinkProvider;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePartitionManager;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveSplitManager;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.HiveTypeTranslator;
+import com.facebook.presto.hive.HiveWriterStats;
+import com.facebook.presto.hive.LocationService;
+import com.facebook.presto.hive.NamenodeStats;
+import com.facebook.presto.hive.OrcFileWriterConfig;
+import com.facebook.presto.hive.OrcFileWriterFactory;
+import com.facebook.presto.hive.PartitionUpdate;
+import com.facebook.presto.hive.RcFileFileWriterFactory;
+import com.facebook.presto.hive.TableParameterCodec;
+import com.facebook.presto.hive.TransactionalMetadata;
+import com.facebook.presto.hive.TypeTranslator;
+import com.facebook.presto.hive.orc.DwrfPageSourceFactory;
+import com.facebook.presto.hive.orc.OrcPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetPageSourceFactory;
+import com.facebook.presto.hive.parquet.ParquetRecordCursorProvider;
+import com.facebook.presto.hive.rcfile.RcFilePageSourceFactory;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.ConnectorSplitManager;
-import com.facebook.presto.spi.type.Type;
-import com.facebook.presto.spi.type.TypeManager;
-import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
import com.google.inject.Binder;
-import com.google.inject.Module;
import com.google.inject.Scopes;
+import com.google.inject.TypeLiteral;
+import com.google.inject.multibindings.Multibinder;
+import io.airlift.event.client.EventClient;
-import static com.facebook.presto.spi.type.TypeSignature.parseTypeSignature;
-import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.inject.multibindings.Multibinder.newSetBinder;
import static io.airlift.configuration.ConfigBinder.configBinder;
+import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
-public class CarbondataModule implements Module {
+import static org.weakref.jmx.ObjectNames.generatedNameOf;
+import static org.weakref.jmx.guice.ExportBinder.newExporter;
+
+public class CarbondataModule extends HiveClientModule {
private final String connectorId;
- private final TypeManager typeManager;
- public CarbondataModule(String connectorId, TypeManager typeManager) {
+ public CarbondataModule(String connectorId) {
+ super(connectorId);
this.connectorId = requireNonNull(connectorId, "connector id is null");
- this.typeManager = requireNonNull(typeManager, "typeManager is null");
}
@Override public void configure(Binder binder) {
- binder.bind(TypeManager.class).toInstance(typeManager);
+ binder.bind(HiveConnectorId.class).toInstance(new HiveConnectorId(connectorId));
+ binder.bind(TypeTranslator.class).toInstance(new HiveTypeTranslator());
+ binder.bind(CoercionPolicy.class).to(HiveCoercionPolicy.class).in(Scopes.SINGLETON);
- binder.bind(CarbondataConnectorId.class).toInstance(new CarbondataConnectorId(connectorId));
- binder.bind(CarbondataMetadata.class).in(Scopes.SINGLETON);
- binder.bind(CarbonTableReader.class).in(Scopes.SINGLETON);
+ binder.bind(HdfsConfigurationUpdater.class).in(Scopes.SINGLETON);
+ binder.bind(HdfsConfiguration.class).to(HiveHdfsConfiguration.class).in(Scopes.SINGLETON);
+ binder.bind(HdfsEnvironment.class).in(Scopes.SINGLETON);
+ binder.bind(DirectoryLister.class).to(HadoopDirectoryLister.class).in(Scopes.SINGLETON);
+ configBinder(binder).bindConfig(HiveClientConfig.class);
+
+ binder.bind(HiveSessionProperties.class).in(Scopes.SINGLETON);
+ binder.bind(HiveTableProperties.class).in(Scopes.SINGLETON);
+
+ binder.bind(NamenodeStats.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(NamenodeStats.class)
+ .as(generatedNameOf(NamenodeStats.class, connectorId));
+
+ Multibinder<HiveRecordCursorProvider> recordCursorProviderBinder =
+ newSetBinder(binder, HiveRecordCursorProvider.class);
+ recordCursorProviderBinder.addBinding().to(ParquetRecordCursorProvider.class)
+ .in(Scopes.SINGLETON);
+ recordCursorProviderBinder.addBinding().to(GenericHiveRecordCursorProvider.class)
+ .in(Scopes.SINGLETON);
+
+ binder.bind(HiveWriterStats.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(HiveWriterStats.class)
+ .as(generatedNameOf(HiveWriterStats.class, connectorId));
+
+ newSetBinder(binder, EventClient.class).addBinding().to(HiveEventClient.class)
+ .in(Scopes.SINGLETON);
+ binder.bind(HivePartitionManager.class).in(Scopes.SINGLETON);
+ binder.bind(LocationService.class).to(HiveLocationService.class).in(Scopes.SINGLETON);
+ binder.bind(TableParameterCodec.class).in(Scopes.SINGLETON);
+ binder.bind(HiveMetadataFactory.class).in(Scopes.SINGLETON);
+ binder.bind(new TypeLiteral<Supplier<TransactionalMetadata>>() {
+ }).to(HiveMetadataFactory.class).in(Scopes.SINGLETON);
+ binder.bind(HiveTransactionManager.class).in(Scopes.SINGLETON);
binder.bind(ConnectorSplitManager.class).to(CarbondataSplitManager.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(ConnectorSplitManager.class)
+ .as(generatedNameOf(HiveSplitManager.class, connectorId));
binder.bind(ConnectorPageSourceProvider.class).to(CarbondataPageSourceProvider.class)
.in(Scopes.SINGLETON);
- binder.bind(CarbondataHandleResolver.class).in(Scopes.SINGLETON);
- configBinder(binder).bindConfig(CarbonTableConfig.class);
- }
+ binder.bind(ConnectorPageSinkProvider.class).to(HivePageSinkProvider.class)
+ .in(Scopes.SINGLETON);
+ binder.bind(ConnectorNodePartitioningProvider.class).to(HiveNodePartitioningProvider.class)
+ .in(Scopes.SINGLETON);
+
+ jsonCodecBinder(binder).bindJsonCodec(PartitionUpdate.class);
- public static final class TypeDeserializer extends FromStringDeserializer<Type> {
- private final TypeManager typeManager;
+ binder.bind(FileFormatDataSourceStats.class).in(Scopes.SINGLETON);
+ newExporter(binder).export(FileFormatDataSourceStats.class)
+ .as(generatedNameOf(FileFormatDataSourceStats.class, connectorId));
- @Inject public TypeDeserializer(TypeManager typeManager) {
- super(Type.class);
- this.typeManager = requireNonNull(typeManager, "typeManager is null");
- }
+ Multibinder<HivePageSourceFactory> pageSourceFactoryBinder =
+ newSetBinder(binder, HivePageSourceFactory.class);
+ pageSourceFactoryBinder.addBinding().to(OrcPageSourceFactory.class).in(Scopes.SINGLETON);
+ pageSourceFactoryBinder.addBinding().to(DwrfPageSourceFactory.class).in(Scopes.SINGLETON);
+ pageSourceFactoryBinder.addBinding().to(ParquetPageSourceFactory.class).in(Scopes.SINGLETON);
--- End diff --
why orc and parquet PageSource are used here ? can add comment about this ?
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2275/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2268/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244312730
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java ---
@@ -43,63 +43,78 @@
import static org.apache.carbondata.presto.Types.checkType;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveColumnHandle;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePageSourceProvider;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSplit;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorPageSource;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.ConnectorSplit;
-import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
+import com.facebook.presto.spi.type.TypeManager;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskAttemptContextImpl;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
-
/**
* Provider Class for Carbondata Page Source class.
*/
-public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider {
+public class CarbondataPageSourceProvider extends HivePageSourceProvider {
- private String connectorId;
private CarbonTableReader carbonTableReader;
private String queryId ;
-
- @Inject public CarbondataPageSourceProvider(CarbondataConnectorId connectorId,
+ private HdfsEnvironment hdfsEnvironment;
+
+ @Inject public CarbondataPageSourceProvider(
+ HiveClientConfig hiveClientConfig,
+ HdfsEnvironment hdfsEnvironment,
+ Set<HiveRecordCursorProvider> cursorProviders,
+ Set<HivePageSourceFactory> pageSourceFactories,
+ TypeManager typeManager,
CarbonTableReader carbonTableReader) {
- this.connectorId = requireNonNull(connectorId, "connectorId is null").toString();
+ super(hiveClientConfig, hdfsEnvironment, cursorProviders, pageSourceFactories, typeManager);
this.carbonTableReader = requireNonNull(carbonTableReader, "carbonTableReader is null");
+ this.hdfsEnvironment = hdfsEnvironment;
}
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle,
ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns) {
- this.queryId = ((CarbondataSplit)split).getQueryId();
+ HiveSplit carbonSplit =
+ checkType(split, HiveSplit.class, "split is not class HiveSplit");
+ if (carbonSplit.getSchema().getProperty("queryId") == null) {
+ return super.createPageSource(transactionHandle, session, split, columns);
+ }
+ this.queryId = carbonSplit.getSchema().getProperty("queryId");
--- End diff --
ok
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by jackylk <gi...@git.apache.org>.
Github user jackylk commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244254647
--- Diff: integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala ---
@@ -80,7 +80,7 @@ object CarbonDataStoreCreator {
UUID.randomUUID().toString))
// val factFilePath: String = new File(dataFilePath).getCanonicalPath
val storeDir: File = new File(absoluteTableIdentifier.getTablePath)
- CarbonUtil.deleteFoldersAndFiles(storeDir)
+// CarbonUtil.deleteFoldersAndFiles(storeDir)
--- End diff --
delete it if not required, same for line 81
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2259/
---
[GitHub] carbondata issue #3019: [WIP] Carbon Presto hive metastore
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2126/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2141/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2134/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by chenliang613 <gi...@git.apache.org>.
Github user chenliang613 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244003542
--- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala ---
@@ -72,69 +74,107 @@ object CarbonSessionExample {
val path = s"$rootPath/examples/spark2/src/main/resources/data.csv"
// scalastyle:off
- spark.sql(
- s"""
- | LOAD DATA LOCAL INPATH '$path'
- | INTO TABLE source
- | OPTIONS('HEADER'='true', 'COMPLEX_DELIMITER_LEVEL_1'='#')
- """.stripMargin)
- // scalastyle:on
-
- spark.sql(
- s"""
- | SELECT charField, stringField, intField
- | FROM source
- | WHERE stringfield = 'spark' AND decimalField > 40
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE length(stringField) = 5
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """.stripMargin).show()
-
- spark.sql("SELECT count(stringField) FROM source").show()
-
- spark.sql(
- s"""
- | SELECT sum(intField), stringField
- | FROM source
- | GROUP BY stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT t1.*, t2.*
- | FROM source t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | WITH t1 AS (
- | SELECT * FROM source
- | UNION ALL
- | SELECT * FROM source
- | )
- | SELECT t1.*, t2.*
- | FROM t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source
- | WHERE stringField = 'spark' and floatField > 2.8
- """.stripMargin).show()
+// spark.sql(
+// s"""
+// | LOAD DATA LOCAL INPATH '$path'
+// | INTO TABLE source
+// | OPTIONS('HEADER'='true', 'COMPLEX_DELIMITER_LEVEL_1'='#')
+// """.stripMargin)
+// // scalastyle:on
+//
+// spark.sql(
+// s"""
+// | CREATE TABLE source_cs(
+// | shortField SHORT,
+// | intField INT,
+// | bigintField LONG,
+// | doubleField DOUBLE,
+// | stringField STRING,
+// | timestampField TIMESTAMP,
+// | decimalField DECIMAL(18,2),
+// | dateField DATE,
+// | charField CHAR(5),
+// | floatField FLOAT
+// | )
+// | using carbon
+// | location 'file://${ExampleUtils.storeLocation}'
+// """.stripMargin)
+//
+// spark.sql("insert into source_cs select * from source")
+//
+// spark.sql(
+// s"""
+// | CREATE TABLE source_par(
+// | shortField SHORT,
+// | intField INT,
+// | bigintField LONG,
+// | doubleField DOUBLE,
+// | stringField STRING,
+// | timestampField TIMESTAMP,
+// | decimalField DECIMAL(18,2),
+// | dateField DATE,
+// | charField CHAR(5),
+// | floatField FLOAT
+// | )
+// | using parquet
+// """.stripMargin)
+//
+// spark.sql("insert into source_par select * from source")
+// spark.sql(
+// s"""
+// | SELECT charField, stringField, intField
+// | FROM source
+// | WHERE stringfield = 'spark' AND decimalField > 40
+// """.stripMargin).show()
+//
+// spark.sql(
+// s"""
+// | SELECT *
+// | FROM source WHERE length(stringField) = 5
+// """.stripMargin).show()
+//
+// spark.sql(
+// s"""
+// | SELECT *
+// | FROM source WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+// """.stripMargin).show()
+//
+// spark.sql("SELECT count(stringField) FROM source").show()
+//
+// spark.sql(
+// s"""
+// | SELECT sum(intField), stringField
+// | FROM source
+// | GROUP BY stringField
+// """.stripMargin).show()
+//
+// spark.sql(
+// s"""
--- End diff --
why disable all these code?
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10208/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244097091
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java ---
@@ -113,7 +132,7 @@ private PrestoCarbonVectorizedRecordReader createReader(ConnectorSplit split,
PrestoCarbonVectorizedRecordReader reader =
new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel,
(AbstractDetailQueryResultIterator) iterator, readSupport);
- reader.setTaskId(carbondataSplit.getIndex());
+ reader.setTaskId(Long.parseLong(carbonSplit.getSchema().getProperty("index")));
--- End diff --
ok, generalized
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/carbondata/pull/3019
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ajantha-bhat <gi...@git.apache.org>.
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244153304
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java ---
@@ -17,69 +17,177 @@
package org.apache.carbondata.presto;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.*;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import static java.util.Objects.requireNonNull;
-import com.facebook.presto.spi.ConnectorHandleResolver;
+import org.apache.carbondata.presto.impl.CarbonTableConfig;
+
+import com.facebook.presto.hive.HiveConnector;
+import com.facebook.presto.hive.HiveConnectorFactory;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveProcedureModule;
+import com.facebook.presto.hive.HiveSchemaProperties;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveStorageFormat;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.NodeVersion;
+import com.facebook.presto.hive.RebindSafeMBeanServer;
+import com.facebook.presto.hive.authentication.HiveAuthenticationModule;
+import com.facebook.presto.hive.metastore.HiveMetastoreModule;
+import com.facebook.presto.hive.s3.HiveS3Module;
+import com.facebook.presto.hive.security.HiveSecurityModule;
+import com.facebook.presto.hive.security.PartitionsAwareAccessControl;
+import com.facebook.presto.spi.NodeManager;
+import com.facebook.presto.spi.PageIndexerFactory;
+import com.facebook.presto.spi.PageSorter;
import com.facebook.presto.spi.classloader.ThreadContextClassLoader;
-import com.facebook.presto.spi.connector.*;
-import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorMetadata;
+import com.facebook.presto.spi.connector.Connector;
+import com.facebook.presto.spi.connector.ConnectorAccessControl;
+import com.facebook.presto.spi.connector.ConnectorContext;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.connector.ConnectorSplitManager;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorSplitManager;
-import com.google.common.base.Throwables;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeNodePartitioningProvider;
+import com.facebook.presto.spi.procedure.Procedure;
+import com.facebook.presto.spi.type.TypeManager;
+import com.google.common.collect.ImmutableSet;
import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
import io.airlift.bootstrap.Bootstrap;
import io.airlift.bootstrap.LifeCycleManager;
+import io.airlift.event.client.EventModule;
import io.airlift.json.JsonModule;
+import io.airlift.units.DataSize;
+import org.weakref.jmx.guice.MBeanModule;
+import sun.reflect.ConstructorAccessor;
+import static com.google.common.base.Throwables.throwIfUnchecked;
+import static io.airlift.configuration.ConfigBinder.configBinder;
/**
* Build Carbondata Connector
* It will be called by CarbondataPlugin
*/
-public class CarbondataConnectorFactory implements ConnectorFactory {
+public class CarbondataConnectorFactory extends HiveConnectorFactory {
- private final String name;
private final ClassLoader classLoader;
public CarbondataConnectorFactory(String connectorName, ClassLoader classLoader) {
- this.name = connectorName;
+ super(connectorName, classLoader, null);
this.classLoader = requireNonNull(classLoader, "classLoader is null");
}
- @Override public String getName() {
- return name;
- }
-
- @Override public ConnectorHandleResolver getHandleResolver() {
- return new CarbondataHandleResolver();
- }
-
- @Override public Connector create(String connectorId, Map<String, String> config,
+ @Override public Connector create(String catalogName, Map<String, String> config,
ConnectorContext context) {
requireNonNull(config, "config is null");
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(classLoader)) {
- Bootstrap app = new Bootstrap(new JsonModule(),
- new CarbondataModule(connectorId, context.getTypeManager()));
+ Bootstrap app = new Bootstrap(new EventModule(), new MBeanModule(), new JsonModule(),
+ new CarbondataModule(catalogName), new HiveS3Module(catalogName),
+ new HiveMetastoreModule(catalogName, Optional.ofNullable(null)), new HiveSecurityModule(),
+ new HiveAuthenticationModule(), new HiveProcedureModule(), binder -> {
+ javax.management.MBeanServer platformMBeanServer =
+ ManagementFactory.getPlatformMBeanServer();
+ binder.bind(javax.management.MBeanServer.class)
+ .toInstance(new RebindSafeMBeanServer(platformMBeanServer));
+ binder.bind(NodeVersion.class)
+ .toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion()));
+ binder.bind(NodeManager.class).toInstance(context.getNodeManager());
+ binder.bind(TypeManager.class).toInstance(context.getTypeManager());
+ binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory());
+ binder.bind(PageSorter.class).toInstance(context.getPageSorter());
+ configBinder(binder).bindConfig(CarbonTableConfig.class);
+ });
Injector injector =
app.strictConfig().doNotInitializeLogging().setRequiredConfigurationProperties(config)
.initialize();
+ setCarbonEnum();
+
LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
- ConnectorMetadata metadata = injector.getInstance(CarbondataMetadata.class);
+ HiveMetadataFactory metadataFactory = injector.getInstance(HiveMetadataFactory.class);
+ HiveTransactionManager transactionManager =
+ injector.getInstance(HiveTransactionManager.class);
ConnectorSplitManager splitManager = injector.getInstance(ConnectorSplitManager.class);
ConnectorPageSourceProvider connectorPageSource =
injector.getInstance(ConnectorPageSourceProvider.class);
+ ConnectorPageSinkProvider pageSinkProvider =
+ injector.getInstance(ConnectorPageSinkProvider.class);
+ ConnectorNodePartitioningProvider connectorDistributionProvider =
+ injector.getInstance(ConnectorNodePartitioningProvider.class);
+ HiveSessionProperties hiveSessionProperties =
+ injector.getInstance(HiveSessionProperties.class);
+ HiveTableProperties hiveTableProperties = injector.getInstance(HiveTableProperties.class);
+ ConnectorAccessControl accessControl =
+ new PartitionsAwareAccessControl(injector.getInstance(ConnectorAccessControl.class));
+ Set<Procedure> procedures = injector.getInstance(Key.get(new TypeLiteral<Set<Procedure>>() {
+ }));
- return new CarbondataConnector(lifeCycleManager,
- new ClassLoaderSafeConnectorMetadata(metadata, classLoader),
- new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader), classLoader,
- new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader));
+ return new HiveConnector(lifeCycleManager, metadataFactory, transactionManager,
+ new ClassLoaderSafeConnectorSplitManager(splitManager, classLoader),
+ new ClassLoaderSafeConnectorPageSourceProvider(connectorPageSource, classLoader),
+ new ClassLoaderSafeConnectorPageSinkProvider(pageSinkProvider, classLoader),
+ new ClassLoaderSafeNodePartitioningProvider(connectorDistributionProvider, classLoader),
+ ImmutableSet.of(), procedures, hiveSessionProperties.getSessionProperties(),
+ HiveSchemaProperties.SCHEMA_PROPERTIES, hiveTableProperties.getTableProperties(),
+ accessControl, classLoader);
} catch (Exception e) {
- throw Throwables.propagate(e);
+ throwIfUnchecked(e);
+ throw new RuntimeException(e);
}
}
+
+ /**
+ * Set the Carbon format enum to HiveStorageFormat, its a hack but for time being it is best
+ * choice to avoid lot of code change.
+ *
+ * @throws Exception
+ */
+ private void setCarbonEnum() throws Exception {
+ for (HiveStorageFormat format : HiveStorageFormat.values()) {
+ if (format.name().equals("CARBON")) {
+ return;
+ }
+ }
+ Constructor<?>[] declaredConstructors = HiveStorageFormat.class.getDeclaredConstructors();
+ declaredConstructors[0].setAccessible(true);
+ Field constructorAccessorField = Constructor.class.getDeclaredField("constructorAccessor");
+ constructorAccessorField.setAccessible(true);
+ ConstructorAccessor ca =
+ (ConstructorAccessor) constructorAccessorField.get(declaredConstructors[0]);
+ if (ca == null) {
+ Method acquireConstructorAccessorMethod =
+ Constructor.class.getDeclaredMethod("acquireConstructorAccessor");
+ acquireConstructorAccessorMethod.setAccessible(true);
+ ca = (ConstructorAccessor) acquireConstructorAccessorMethod.invoke(declaredConstructors[0]);
+ }
+ Object instance = ca.newInstance(new Object[] { "CARBON", HiveStorageFormat.values().length, "",
+ "org.apache.carbondata.hadoop.api.CarbonFileInputFormat",
--- End diff --
should this be TableInputFormat ?
because in CarbondataPageSourceProvider.java we have used TableInputFormat
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1932/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244276118
--- Diff: integration/presto/src/test/scala/org/apache/carbondata/presto/server/PrestoServer.scala ---
@@ -129,6 +130,21 @@ class PrestoServer {
}
}
+ def execute(query: String) = {
+
+ Try {
+ val conn: Connection = createJdbcConnection(dbName)
+ logger.info(s"***** executing the query ***** \n $query")
--- End diff --
ok
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/2053/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2230/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10316/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244097070
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java ---
@@ -43,63 +44,81 @@
import static org.apache.carbondata.presto.Types.checkType;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveColumnHandle;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePageSourceProvider;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSplit;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorPageSource;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.ConnectorSplit;
-import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
+import com.facebook.presto.spi.type.TypeManager;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskAttemptContextImpl;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
-
/**
* Provider Class for Carbondata Page Source class.
*/
-public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider {
+public class CarbondataPageSourceProvider extends HivePageSourceProvider {
- private String connectorId;
private CarbonTableReader carbonTableReader;
private String queryId ;
-
- @Inject public CarbondataPageSourceProvider(CarbondataConnectorId connectorId,
+ private HdfsEnvironment hdfsEnvironment;
+
+ @Inject public CarbondataPageSourceProvider(
+ HiveClientConfig hiveClientConfig,
+ HdfsEnvironment hdfsEnvironment,
+ Set<HiveRecordCursorProvider> cursorProviders,
+ Set<HivePageSourceFactory> pageSourceFactories,
+ TypeManager typeManager,
CarbonTableReader carbonTableReader) {
- this.connectorId = requireNonNull(connectorId, "connectorId is null").toString();
+ super(hiveClientConfig, hdfsEnvironment, cursorProviders, pageSourceFactories, typeManager);
this.carbonTableReader = requireNonNull(carbonTableReader, "carbonTableReader is null");
+ this.hdfsEnvironment = hdfsEnvironment;
}
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle,
ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns) {
- this.queryId = ((CarbondataSplit)split).getQueryId();
+ HiveSplit carbonSplit =
+ checkType(split, HiveSplit.class, "split is not class HiveSplit");
+ if (carbonSplit.getSchema().getProperty("queryId") == null) {
+ return super.createPageSource(transactionHandle, session, split, columns);
+ }
+ this.queryId = carbonSplit.getSchema().getProperty("queryId");
+ Configuration configuration = this.hdfsEnvironment.getConfiguration(
+ new HdfsEnvironment.HdfsContext(session, carbonSplit.getDatabase(), carbonSplit.getTable()),
+ new Path(carbonSplit.getSchema().getProperty("tablePath")));
+ configuration = carbonTableReader.updateS3Properties(configuration);
CarbonDictionaryDecodeReadSupport readSupport = new CarbonDictionaryDecodeReadSupport();
PrestoCarbonVectorizedRecordReader carbonRecordReader =
- createReader(split, columns, readSupport);
+ createReader(carbonSplit, columns, readSupport, configuration);
return new CarbondataPageSource(carbonRecordReader, columns);
}
/**
- * @param split
+ * @param carbonSplit
* @param columns
* @param readSupport
--- End diff --
ok
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by kumarvishal09 <gi...@git.apache.org>.
Github user kumarvishal09 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244104691
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataPageSourceProvider.java ---
@@ -43,63 +43,78 @@
import static org.apache.carbondata.presto.Types.checkType;
+import com.facebook.presto.hive.HdfsEnvironment;
+import com.facebook.presto.hive.HiveClientConfig;
+import com.facebook.presto.hive.HiveColumnHandle;
+import com.facebook.presto.hive.HivePageSourceFactory;
+import com.facebook.presto.hive.HivePageSourceProvider;
+import com.facebook.presto.hive.HiveRecordCursorProvider;
+import com.facebook.presto.hive.HiveSplit;
import com.facebook.presto.spi.ColumnHandle;
import com.facebook.presto.spi.ConnectorPageSource;
import com.facebook.presto.spi.ConnectorSession;
import com.facebook.presto.spi.ConnectorSplit;
-import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.SchemaTableName;
import com.facebook.presto.spi.connector.ConnectorTransactionHandle;
+import com.facebook.presto.spi.type.TypeManager;
import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TaskAttemptContextImpl;
import org.apache.hadoop.mapred.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskType;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
-
/**
* Provider Class for Carbondata Page Source class.
*/
-public class CarbondataPageSourceProvider implements ConnectorPageSourceProvider {
+public class CarbondataPageSourceProvider extends HivePageSourceProvider {
- private String connectorId;
private CarbonTableReader carbonTableReader;
private String queryId ;
-
- @Inject public CarbondataPageSourceProvider(CarbondataConnectorId connectorId,
+ private HdfsEnvironment hdfsEnvironment;
+
+ @Inject public CarbondataPageSourceProvider(
+ HiveClientConfig hiveClientConfig,
+ HdfsEnvironment hdfsEnvironment,
+ Set<HiveRecordCursorProvider> cursorProviders,
+ Set<HivePageSourceFactory> pageSourceFactories,
+ TypeManager typeManager,
CarbonTableReader carbonTableReader) {
- this.connectorId = requireNonNull(connectorId, "connectorId is null").toString();
+ super(hiveClientConfig, hdfsEnvironment, cursorProviders, pageSourceFactories, typeManager);
this.carbonTableReader = requireNonNull(carbonTableReader, "carbonTableReader is null");
+ this.hdfsEnvironment = hdfsEnvironment;
}
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle,
ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns) {
- this.queryId = ((CarbondataSplit)split).getQueryId();
+ HiveSplit carbonSplit =
+ checkType(split, HiveSplit.class, "split is not class HiveSplit");
+ if (carbonSplit.getSchema().getProperty("queryId") == null) {
+ return super.createPageSource(transactionHandle, session, split, columns);
+ }
+ this.queryId = carbonSplit.getSchema().getProperty("queryId");
--- End diff --
Move this line above If condition and in if condition check if queryId is null
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ajantha-bhat <gi...@git.apache.org>.
Github user ajantha-bhat commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r243821888
--- Diff: examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala ---
@@ -72,69 +74,107 @@ object CarbonSessionExample {
val path = s"$rootPath/examples/spark2/src/main/resources/data.csv"
// scalastyle:off
- spark.sql(
- s"""
- | LOAD DATA LOCAL INPATH '$path'
- | INTO TABLE source
- | OPTIONS('HEADER'='true', 'COMPLEX_DELIMITER_LEVEL_1'='#')
- """.stripMargin)
- // scalastyle:on
-
- spark.sql(
- s"""
- | SELECT charField, stringField, intField
- | FROM source
- | WHERE stringfield = 'spark' AND decimalField > 40
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE length(stringField) = 5
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """.stripMargin).show()
-
- spark.sql("SELECT count(stringField) FROM source").show()
-
- spark.sql(
- s"""
- | SELECT sum(intField), stringField
- | FROM source
- | GROUP BY stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT t1.*, t2.*
- | FROM source t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | WITH t1 AS (
- | SELECT * FROM source
- | UNION ALL
- | SELECT * FROM source
- | )
- | SELECT t1.*, t2.*
- | FROM t1, source t2
- | WHERE t1.stringField = t2.stringField
- """.stripMargin).show()
-
- spark.sql(
- s"""
- | SELECT *
- | FROM source
- | WHERE stringField = 'spark' and floatField > 2.8
- """.stripMargin).show()
+// spark.sql(
--- End diff --
please revert the changes in this file
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by chenliang613 <gi...@git.apache.org>.
Github user chenliang613 commented on the issue:
https://github.com/apache/carbondata/pull/3019
remove CarbondataConnector.java in this pr by using hive connector. in future, if consider contributing carbondata integraton to presto community, how to handle ?
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10185/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder2.1/1921/
---
[GitHub] carbondata issue #3019: [CARBONDATA-3194] Integrating Carbon with Presto usi...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Success with Spark 2.2.1, Please check CI http://95.216.28.178:8080/job/ApacheCarbonPRBuilder1/2131/
---
[GitHub] carbondata issue #3019: [WIP] Carbon Presto hive metastore
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/3019
Build Failed with Spark 2.3.2, Please check CI http://136.243.101.176:8080/job/carbondataprbuilder2.3/10169/
---
[GitHub] carbondata pull request #3019: [CARBONDATA-3194] Integrating Carbon with Pre...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/3019#discussion_r244097300
--- Diff: integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataConnectorFactory.java ---
@@ -17,69 +17,179 @@
package org.apache.carbondata.presto;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.*;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
import static java.util.Objects.requireNonNull;
-import com.facebook.presto.spi.ConnectorHandleResolver;
+import org.apache.carbondata.presto.impl.CarbonTableConfig;
+
+import com.facebook.presto.hive.HiveConnector;
+import com.facebook.presto.hive.HiveConnectorFactory;
+import com.facebook.presto.hive.HiveMetadataFactory;
+import com.facebook.presto.hive.HiveProcedureModule;
+import com.facebook.presto.hive.HiveSchemaProperties;
+import com.facebook.presto.hive.HiveSessionProperties;
+import com.facebook.presto.hive.HiveStorageFormat;
+import com.facebook.presto.hive.HiveTableProperties;
+import com.facebook.presto.hive.HiveTransactionManager;
+import com.facebook.presto.hive.NodeVersion;
+import com.facebook.presto.hive.RebindSafeMBeanServer;
+import com.facebook.presto.hive.authentication.HiveAuthenticationModule;
+import com.facebook.presto.hive.metastore.HiveMetastoreModule;
+import com.facebook.presto.hive.s3.HiveS3Module;
+import com.facebook.presto.hive.security.HiveSecurityModule;
+import com.facebook.presto.hive.security.PartitionsAwareAccessControl;
+import com.facebook.presto.spi.NodeManager;
+import com.facebook.presto.spi.PageIndexerFactory;
+import com.facebook.presto.spi.PageSorter;
import com.facebook.presto.spi.classloader.ThreadContextClassLoader;
-import com.facebook.presto.spi.connector.*;
-import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorMetadata;
+import com.facebook.presto.spi.connector.Connector;
+import com.facebook.presto.spi.connector.ConnectorAccessControl;
+import com.facebook.presto.spi.connector.ConnectorContext;
+import com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSinkProvider;
+import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
+import com.facebook.presto.spi.connector.ConnectorSplitManager;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSinkProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorPageSourceProvider;
import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeConnectorSplitManager;
-import com.google.common.base.Throwables;
+import com.facebook.presto.spi.connector.classloader.ClassLoaderSafeNodePartitioningProvider;
+import com.facebook.presto.spi.procedure.Procedure;
+import com.facebook.presto.spi.type.TypeManager;
+import com.google.common.collect.ImmutableSet;
import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
import io.airlift.bootstrap.Bootstrap;
import io.airlift.bootstrap.LifeCycleManager;
+import io.airlift.event.client.EventModule;
import io.airlift.json.JsonModule;
+import io.airlift.units.DataSize;
+import org.weakref.jmx.guice.MBeanModule;
+import sun.reflect.ConstructorAccessor;
+import static com.google.common.base.Throwables.throwIfUnchecked;
+import static io.airlift.configuration.ConfigBinder.configBinder;
/**
* Build Carbondata Connector
* It will be called by CarbondataPlugin
*/
-public class CarbondataConnectorFactory implements ConnectorFactory {
+public class CarbondataConnectorFactory extends HiveConnectorFactory {
private final String name;
--- End diff --
removed
---