You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@carbondata.apache.org by aaron <94...@qq.com> on 2018/09/03 00:27:36 UTC
Re: error occur when I load data to s3
*update the aws-java-sdk and hadoop-aws to below version, then authorization
works.
com.amazonaws:aws-java-sdk:1.10.75.1,org.apache.hadoop:hadoop-aws:2.7.3*
*But we still can not load data, the exception is same.
carbon.sql("LOAD DATA INPATH
'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE test_s3_table")*
18/09/02 21:49:47 ERROR CarbonLoaderUtil: main Unable to unlock Table lock
for tabledefault.test_s3_table during table status updation
18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main
java.lang.ArrayIndexOutOfBoundsException
at java.lang.System.arraycopy(Native Method)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
at org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
at
org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
at
org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
at org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
at
$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)
at
$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:39)
at
$line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:41)
at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:43)
at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:45)
at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:47)
at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:49)
at $line25.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:51)
at $line25.$read$$iw$$iw$$iw$$iw.<init>(<console>:53)
at $line25.$read$$iw$$iw$$iw.<init>(<console>:55)
at $line25.$read$$iw$$iw.<init>(<console>:57)
at $line25.$read$$iw.<init>(<console>:59)
at $line25.$read.<init>(<console>:61)
at $line25.$read$.<init>(<console>:65)
at $line25.$read$.<clinit>(<console>)
at $line25.$eval$.$print$lzycompute(<console>:7)
at $line25.$eval$.$print(<console>:6)
at $line25.$eval.$print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
at
scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
at
scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
at
scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
at
scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at
scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at
scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
at org.apache.spark.repl.Main$.doMain(Main.scala:74)
at org.apache.spark.repl.Main$.main(Main.scala:54)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
18/09/02 21:49:47 AUDIT CarbonLoadDataCommand:
[aaron.local][aaron][Thread-1]Dataload failure for default.test_s3_table.
Please check the logs
18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main Got exception
java.lang.ArrayIndexOutOfBoundsException when processing data. But this
command does not support undo yet, skipping the undo part.
java.lang.ArrayIndexOutOfBoundsException
at java.lang.System.arraycopy(Native Method)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
at
org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
at
org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
at
org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
at
org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
... 52 elided
aaron wrote
> Hi dear community, could anybody please kindly tell me what happened?
>
> *Env*:
>
> 1.spark 2.2.1 + carbon1.4.1
> 2.spark.jars.packages
> com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.2
> 3.spark.driver.extraClassPath
> file:///usr/local/Cellar/apache-spark/2.2.1/lib/*
> spark.executor.extraClassPath
> file:///usr/local/Cellar/apache-spark/2.2.1/lib/*
> lib folder include below jars
> -rw-r--r--@ 1 aaron staff 52M Aug 29 20:50
> apache-carbondata-1.4.1-bin-spark2.2.1-hadoop2.7.2.jar
> -rw-r--r-- 1 aaron staff 764K Aug 29 21:33 httpclient-4.5.4.jar
> -rw-r--r-- 1 aaron staff 314K Aug 29 21:40 httpcore-4.4.jar
>
>
> *Code*:
>
> import org.apache.spark.sql.SparkSession
> import org.apache.spark.sql.CarbonSession._
> import org.apache.spark.sql.catalyst.util._
> import org.apache.carbondata.core.util.CarbonProperties
> import org.apache.carbondata.core.constants.CarbonCommonConstants
> CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE,
> "HDFSLOCK")
> val carbon =
> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
> "xxx").config("spark.hadoop.fs.s3a.secret.key",
> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
>
> carbon.sql("CREATE TABLE IF NOT EXISTS test_s3_table(id string, name
> string,
> city string, age Int) STORED BY 'carbondata' LOCATION
> 's3a://key:password@aaron-s3-poc/'")
> carbon.sql("LOAD DATA INPATH
> 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
> test_s3_table")
>
> *s3 files,*
>
> aws s3 ls s3://aaron-s3-poc/ --human --recursive
> 2018-08-29 22:13:32 0 Bytes LockFiles/tablestatus.lock
> 2018-08-29 21:41:36 616 Bytes Metadata/schema
>
>
> *Issue 1,* when I create table, carbondata raise Exception
> "com.amazonaws.AmazonClientException: Unable to load AWS credentials from
> any provider in the chain" even if
> a. I set related properties in spark-default.conf like
> spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
> spark.hadoop.fs.s3a.awsAccessKeyId=xxx
> spark.hadoop.fs.s3a.awsSecretAccessKey=xxx
> spark.hadoop.fs.s3a.access.key=xxx
> spark.hadoop.fs.s3a.secret.key=xxx
> b.config in code
> val carbon =
> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
> "xxx").config("spark.hadoop.fs.s3a.secret.key",
> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
> c. spark-submit conf
> Finally I succeed when I put credentials in LOCATION
> 's3a://key:password@aaron-s3-poc/'", But it's very strange. Who could tell
> me why?
>
>
> *Issue 2,* Load data failed
>
> scala> carbon.sql("LOAD DATA INPATH
> 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
> test_s3_table")
> 18/08/29 22:13:35 ERROR CarbonLoaderUtil: main Unable to unlock Table lock
> for tabledefault.test_s3_table during table status updation
> 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
> at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
> at
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.
> <init>
> (Dataset.scala:183)
> at
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
> at
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
> at
> $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :36)
> at
> $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :41)
> at
> $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :43)
> at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :45)
> at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :47)
> at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :49)
> at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :51)
> at $line21.$read$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :53)
> at $line21.$read$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :55)
> at $line21.$read$$iw$$iw$$iw.
> <init>
> (
> <console>
> :57)
> at $line21.$read$$iw$$iw.
> <init>
> (
> <console>
> :59)
> at $line21.$read$$iw.
> <init>
> (
> <console>
> :61)
> at $line21.$read.
> <init>
> (
> <console>
> :63)
> at $line21.$read$.
> <init>
> (
> <console>
> :67)
> at $line21.$read$.
> <clinit>
> (
> <console>
> )
> at $line21.$eval$.$print$lzycompute(
> <console>
> :7)
> at $line21.$eval$.$print(
> <console>
> :6)
> at $line21.$eval.$print(
> <console>
> )
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
> at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
> at
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
> at
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
> at
> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
> at
> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
> at
> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
> at
> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
> at org.apache.spark.repl.Main$.doMain(Main.scala:74)
> at org.apache.spark.repl.Main$.main(Main.scala:54)
> at org.apache.spark.repl.Main.main(Main.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
> at
> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 18/08/29 22:13:35 AUDIT CarbonLoadDataCommand:
> [aaron.local][aaron][Thread-1]Dataload failure for default.test_s3_table.
> Please check the logs
> 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main Got exception
> java.lang.ArrayIndexOutOfBoundsException when processing data. But this
> command does not support undo yet, skipping the undo part.
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
> at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
> at
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.
> <init>
> (Dataset.scala:183)
> at
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
> at
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
> ... 52 elided
>
>
> Thanks
> Aaron
>
>
>
> --
> Sent from:
> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by aaron <94...@qq.com>.
Thanks, I will have a try.
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by aaron <94...@qq.com>.
Hi kunalkapoor, Thanks very much for your guidance, you are totally right!
It works now.
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by Kunal Kapoor <ku...@gmail.com>.
Hi Aaron,
I tried running similar commands from my environment, Load data command was
successful.
From analysing the logs the exception seems to be coming while lock file
creation.
Can you try the same scenario by configuring the `carbon.lock.path`
property in carbon.properties to any HDFS location:
*example:*
carbon.lock.path=hdfs://hacluster/mylockFiles
Thanks
Kunal Kapoor
On Tue, Sep 4, 2018 at 12:17 PM aaron <94...@qq.com> wrote:
> Hi kunalkapoor, I'd like give you more debug log as below.
>
>
> application/x-www-form-urlencoded; charset=utf-8
> Tue, 04 Sep 2018 06:45:10 GMT
> /aa-sdk-test2/carbon-data/example/LockFiles/concurrentload.lock"
> 18/09/04 14:45:10 DEBUG request: Sending Request: GET
> https://aa-sdk-test2.s3.us-east-1.amazonaws.com
> /carbon-data/example/LockFiles/concurrentload.lock Headers: (Authorization:
> AWS AKIAIAQX5F5B2MLQPRGQ:Ap8rHsiPQPYUdcBb2Ojb/MA9q+I=, User-Agent:
> aws-sdk-java/1.7.4 Mac_OS_X/10.13.6
> Java_HotSpot(TM)_64-Bit_Server_VM/25.144-b01/1.8.0_144, Range: bytes=0--1,
> Date: Tue, 04 Sep 2018 06:45:10 GMT, Content-Type:
> application/x-www-form-urlencoded; charset=utf-8, )
> 18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection request:
> [route: {s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443][total
> kept
> alive: 1; route allocated: 1 of 15; total allocated: 1 of 15]
> 18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection leased:
> [id: 1][route:
> {s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443][total kept
> alive:
> 0; route allocated: 1 of 15; total allocated: 1 of 15]
> 18/09/04 14:45:10 DEBUG SdkHttpClient: Stale connection check
> 18/09/04 14:45:10 DEBUG RequestAddCookies: CookieSpec selected: default
> 18/09/04 14:45:10 DEBUG RequestAuthCache: Auth cache not set in the context
> 18/09/04 14:45:10 DEBUG RequestProxyAuthentication: Proxy auth state:
> UNCHALLENGED
> 18/09/04 14:45:10 DEBUG SdkHttpClient: Attempt 1 to execute request
> 18/09/04 14:45:10 DEBUG DefaultClientConnection: Sending request: GET
> /carbon-data/example/LockFiles/concurrentload.lock HTTP/1.1
> 18/09/04 14:45:10 DEBUG wire: >> "GET
> /carbon-data/example/LockFiles/concurrentload.lock HTTP/1.1[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "Host:
> aa-sdk-test2.s3.us-east-1.amazonaws.com[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "Authorization: AWS
> AKIAIAQX5F5B2MLQPRGQ:Ap8rHsiPQPYUdcBb2Ojb/MA9q+I=[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "User-Agent: aws-sdk-java/1.7.4
> Mac_OS_X/10.13.6
> Java_HotSpot(TM)_64-Bit_Server_VM/25.144-b01/1.8.0_144[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "Range: bytes=0--1[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "Date: Tue, 04 Sep 2018 06:45:10
> GMT[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "Content-Type:
> application/x-www-form-urlencoded; charset=utf-8[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "Connection: Keep-Alive[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: >> "[\r][\n]"
> 18/09/04 14:45:10 DEBUG headers: >> GET
> /carbon-data/example/LockFiles/concurrentload.lock HTTP/1.1
> 18/09/04 14:45:10 DEBUG headers: >> Host:
> aa-sdk-test2.s3.us-east-1.amazonaws.com
> 18/09/04 14:45:10 DEBUG headers: >> Authorization: AWS
> AKIAIAQX5F5B2MLQPRGQ:Ap8rHsiPQPYUdcBb2Ojb/MA9q+I=
> 18/09/04 14:45:10 DEBUG headers: >> User-Agent: aws-sdk-java/1.7.4
> Mac_OS_X/10.13.6 Java_HotSpot(TM)_64-Bit_Server_VM/25.144-b01/1.8.0_144
> 18/09/04 14:45:10 DEBUG headers: >> Range: bytes=0--1
> 18/09/04 14:45:10 DEBUG headers: >> Date: Tue, 04 Sep 2018 06:45:10 GMT
> 18/09/04 14:45:10 DEBUG headers: >> Content-Type:
> application/x-www-form-urlencoded; charset=utf-8
> 18/09/04 14:45:10 DEBUG headers: >> Connection: Keep-Alive
> 18/09/04 14:45:10 DEBUG wire: << "HTTP/1.1 200 OK[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "x-amz-id-2:
>
> ooaOvIUsvupOOYOCVRY7y4TUanV9xJbcAqfd+w31xAkGRptm1blE5E5yMobmKsmRyGj9crhGCao=[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "x-amz-request-id:
> A1AD0240EBDD2234[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "Date: Tue, 04 Sep 2018 06:45:11
> GMT[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "Last-Modified: Tue, 04 Sep 2018 06:45:05
> GMT[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "ETag:
> "d41d8cd98f00b204e9800998ecf8427e"[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "Accept-Ranges: bytes[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "Content-Type:
> application/octet-stream[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "Content-Length: 0[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "Server: AmazonS3[\r][\n]"
> 18/09/04 14:45:10 DEBUG wire: << "[\r][\n]"
> 18/09/04 14:45:10 DEBUG DefaultClientConnection: Receiving response:
> HTTP/1.1 200 OK
> 18/09/04 14:45:10 DEBUG headers: << HTTP/1.1 200 OK
> 18/09/04 14:45:10 DEBUG headers: << x-amz-id-2:
>
> ooaOvIUsvupOOYOCVRY7y4TUanV9xJbcAqfd+w31xAkGRptm1blE5E5yMobmKsmRyGj9crhGCao=
> 18/09/04 14:45:10 DEBUG headers: << x-amz-request-id: A1AD0240EBDD2234
> 18/09/04 14:45:10 DEBUG headers: << Date: Tue, 04 Sep 2018 06:45:11 GMT
> 18/09/04 14:45:10 DEBUG headers: << Last-Modified: Tue, 04 Sep 2018
> 06:45:05
> GMT
> 18/09/04 14:45:10 DEBUG headers: << ETag:
> "d41d8cd98f00b204e9800998ecf8427e"
> 18/09/04 14:45:10 DEBUG headers: << Accept-Ranges: bytes
> 18/09/04 14:45:10 DEBUG headers: << Content-Type: application/octet-stream
> 18/09/04 14:45:10 DEBUG headers: << Content-Length: 0
> 18/09/04 14:45:10 DEBUG headers: << Server: AmazonS3
> 18/09/04 14:45:10 DEBUG SdkHttpClient: Connection can be kept alive
> indefinitely
> 18/09/04 14:45:10 DEBUG request: Received successful response: 200, AWS
> Request ID: A1AD0240EBDD2234
> 18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection [id:
> 1][route: {s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443] can be
> kept alive indefinitely
> 18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection
> released:
> [id: 1][route:
> {s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443][total kept
> alive:
> 1; route allocated: 1 of 15; total allocated: 1 of 15]
> 18/09/04 14:45:10 DEBUG S3AFileSystem: OutputStream for key
> 'carbon-data/example/LockFiles/concurrentload.lock' writing to tempfile:
> /tmp/hadoop-aaron/s3a/output-8508205130207286174.tmp
> 18/09/04 14:45:10 ERROR CarbonLoadDataCommand: main
> java.lang.ArrayIndexOutOfBoundsException
> at
> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
> at
> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.acquireConcurrentLoadLock(CarbonLoadDataCommand.scala:399)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:259)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
> at
> org.apache.carbondata.examples.S3Example$.main(S3Example.scala:91)
> at org.apache.carbondata.examples.S3Example.main(S3Example.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
>
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
> at
> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> at
> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 18/09/04 14:45:10 AUDIT CarbonLoadDataCommand:
> [aaron.lan.appannie.com][aaron][Thread-1]Dataload failure for
> default.carbon_table. Please check the logs
> 18/09/04 14:45:10 DEBUG Client: The ping interval is 60000 ms.
> 18/09/04 14:45:10 DEBUG Client: Connecting to localhost/127.0.0.1:9000
> 18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
> localhost/127.0.0.1:9000 from aaron: starting, having connections 1
> 18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
> localhost/127.0.0.1:9000 from aaron sending #3
> 18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
> localhost/127.0.0.1:9000 from aaron got value #3
> 18/09/04 14:45:10 DEBUG ProtobufRpcEngine: Call: getFileInfo took 6ms
> 18/09/04 14:45:10 DEBUG AbstractDFSCarbonFile: main Exception occurred:File
> does not exist:
> hdfs://localhost:9000/usr/carbon-meta/partition/default/carbon_table
> 18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
> localhost/127.0.0.1:9000 from aaron sending #4
> 18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
> localhost/127.0.0.1:9000 from aaron got value #4
> 18/09/04 14:45:10 DEBUG ProtobufRpcEngine: Call: getFileInfo took 3ms
> 18/09/04 14:45:10 ERROR CarbonLoadDataCommand: main Got exception
> java.lang.ArrayIndexOutOfBoundsException when processing data. But this
> command does not support undo yet, skipping the undo part.
> Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException
> at
> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
> at
> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.acquireConcurrentLoadLock(CarbonLoadDataCommand.scala:399)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:259)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
> at
> org.apache.carbondata.examples.S3Example$.main(S3Example.scala:91)
> at org.apache.carbondata.examples.S3Example.main(S3Example.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
>
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
> at
> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> at
> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 18/09/04 14:45:10 INFO SparkContext: Invoking stop() from shutdown hook
> 18/09/04 14:45:10 INFO SparkUI: Stopped Spark web UI at
> http://localhost:4040
> 18/09/04 14:45:10 INFO MapOutputTrackerMasterEndpoint:
> MapOutputTrackerMasterEndpoint stopped!
> 18/09/04 14:45:10 INFO MemoryStore: MemoryStore cleared
> 18/09/04 14:45:10 INFO BlockManager: BlockManager stopped
> 18/09/04 14:45:10 INFO BlockManagerMaster: BlockManagerMaster stopped
> 18/09/04 14:45:10 INFO
> OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:
> OutputCommitCoordinator stopped!
> 18/09/04 14:45:10 INFO SparkContext: Successfully stopped SparkContext
> 18/09/04 14:45:10 INFO ShutdownHookManager: Shutdown hook called
> 18/09/04 14:45:10 INFO ShutdownHookManager: Deleting directory
>
> /private/var/folders/dd/n9pmb1nj0dncx5rd_s2rm9_40000gn/T/spark-f1e5dab8-a7db-4107-a3bf-c7253ba7ac06
> 18/09/04 14:45:10 DEBUG IdleConnectionReaper: Reaper thread:
> java.lang.InterruptedException: sleep interrupted
> at java.lang.Thread.sleep(Native Method)
> at
> com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:112)
> 18/09/04 14:45:10 DEBUG IdleConnectionReaper: Shutting down reaper thread.
> 18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection manager
> is shutting down
> 18/09/04 14:45:10 DEBUG DefaultClientConnection: Connection
> 0.0.0.0:59398<->54.231.82.12:443 closed
> 18/09/04 14:45:10 DEBUG DefaultClientConnection: Connection
> 0.0.0.0:59398<->54.231.82.12:443 closed
> 18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection manager
> shut down
> 18/09/04 14:45:10 DEBUG Client: stopping client from cache:
> org.apache.hadoop.ipc.Client@18ab86a2
> 18/09/04 14:45:10 DEBUG Client: removing client from cache:
> org.apache.hadoop.ipc.Client@18ab86a2
> 18/09/04 14:45:10 DEBUG Client: stopping actual client because no more
> references remain: org.apache.hadoop.ipc.Client@18ab86a2
> 18/09/04 14:45:10 DEBUG Client: Stopping client
> 18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
> localhost/127.0.0.1:9000 from aaron: closed
> 18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
> localhost/127.0.0.1:9000 from aaron: stopped, remaining connections 0
>
>
>
> --
> Sent from:
> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
>
Re: error occur when I load data to s3
Posted by aaron <94...@qq.com>.
Hi kunalkapoor, I'd like give you more debug log as below.
application/x-www-form-urlencoded; charset=utf-8
Tue, 04 Sep 2018 06:45:10 GMT
/aa-sdk-test2/carbon-data/example/LockFiles/concurrentload.lock"
18/09/04 14:45:10 DEBUG request: Sending Request: GET
https://aa-sdk-test2.s3.us-east-1.amazonaws.com
/carbon-data/example/LockFiles/concurrentload.lock Headers: (Authorization:
AWS AKIAIAQX5F5B2MLQPRGQ:Ap8rHsiPQPYUdcBb2Ojb/MA9q+I=, User-Agent:
aws-sdk-java/1.7.4 Mac_OS_X/10.13.6
Java_HotSpot(TM)_64-Bit_Server_VM/25.144-b01/1.8.0_144, Range: bytes=0--1,
Date: Tue, 04 Sep 2018 06:45:10 GMT, Content-Type:
application/x-www-form-urlencoded; charset=utf-8, )
18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection request:
[route: {s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443][total kept
alive: 1; route allocated: 1 of 15; total allocated: 1 of 15]
18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection leased:
[id: 1][route:
{s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443][total kept alive:
0; route allocated: 1 of 15; total allocated: 1 of 15]
18/09/04 14:45:10 DEBUG SdkHttpClient: Stale connection check
18/09/04 14:45:10 DEBUG RequestAddCookies: CookieSpec selected: default
18/09/04 14:45:10 DEBUG RequestAuthCache: Auth cache not set in the context
18/09/04 14:45:10 DEBUG RequestProxyAuthentication: Proxy auth state:
UNCHALLENGED
18/09/04 14:45:10 DEBUG SdkHttpClient: Attempt 1 to execute request
18/09/04 14:45:10 DEBUG DefaultClientConnection: Sending request: GET
/carbon-data/example/LockFiles/concurrentload.lock HTTP/1.1
18/09/04 14:45:10 DEBUG wire: >> "GET
/carbon-data/example/LockFiles/concurrentload.lock HTTP/1.1[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "Host:
aa-sdk-test2.s3.us-east-1.amazonaws.com[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "Authorization: AWS
AKIAIAQX5F5B2MLQPRGQ:Ap8rHsiPQPYUdcBb2Ojb/MA9q+I=[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "User-Agent: aws-sdk-java/1.7.4
Mac_OS_X/10.13.6
Java_HotSpot(TM)_64-Bit_Server_VM/25.144-b01/1.8.0_144[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "Range: bytes=0--1[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "Date: Tue, 04 Sep 2018 06:45:10
GMT[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "Content-Type:
application/x-www-form-urlencoded; charset=utf-8[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "Connection: Keep-Alive[\r][\n]"
18/09/04 14:45:10 DEBUG wire: >> "[\r][\n]"
18/09/04 14:45:10 DEBUG headers: >> GET
/carbon-data/example/LockFiles/concurrentload.lock HTTP/1.1
18/09/04 14:45:10 DEBUG headers: >> Host:
aa-sdk-test2.s3.us-east-1.amazonaws.com
18/09/04 14:45:10 DEBUG headers: >> Authorization: AWS
AKIAIAQX5F5B2MLQPRGQ:Ap8rHsiPQPYUdcBb2Ojb/MA9q+I=
18/09/04 14:45:10 DEBUG headers: >> User-Agent: aws-sdk-java/1.7.4
Mac_OS_X/10.13.6 Java_HotSpot(TM)_64-Bit_Server_VM/25.144-b01/1.8.0_144
18/09/04 14:45:10 DEBUG headers: >> Range: bytes=0--1
18/09/04 14:45:10 DEBUG headers: >> Date: Tue, 04 Sep 2018 06:45:10 GMT
18/09/04 14:45:10 DEBUG headers: >> Content-Type:
application/x-www-form-urlencoded; charset=utf-8
18/09/04 14:45:10 DEBUG headers: >> Connection: Keep-Alive
18/09/04 14:45:10 DEBUG wire: << "HTTP/1.1 200 OK[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "x-amz-id-2:
ooaOvIUsvupOOYOCVRY7y4TUanV9xJbcAqfd+w31xAkGRptm1blE5E5yMobmKsmRyGj9crhGCao=[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "x-amz-request-id:
A1AD0240EBDD2234[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "Date: Tue, 04 Sep 2018 06:45:11
GMT[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "Last-Modified: Tue, 04 Sep 2018 06:45:05
GMT[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "ETag:
"d41d8cd98f00b204e9800998ecf8427e"[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "Accept-Ranges: bytes[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "Content-Type:
application/octet-stream[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "Content-Length: 0[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "Server: AmazonS3[\r][\n]"
18/09/04 14:45:10 DEBUG wire: << "[\r][\n]"
18/09/04 14:45:10 DEBUG DefaultClientConnection: Receiving response:
HTTP/1.1 200 OK
18/09/04 14:45:10 DEBUG headers: << HTTP/1.1 200 OK
18/09/04 14:45:10 DEBUG headers: << x-amz-id-2:
ooaOvIUsvupOOYOCVRY7y4TUanV9xJbcAqfd+w31xAkGRptm1blE5E5yMobmKsmRyGj9crhGCao=
18/09/04 14:45:10 DEBUG headers: << x-amz-request-id: A1AD0240EBDD2234
18/09/04 14:45:10 DEBUG headers: << Date: Tue, 04 Sep 2018 06:45:11 GMT
18/09/04 14:45:10 DEBUG headers: << Last-Modified: Tue, 04 Sep 2018 06:45:05
GMT
18/09/04 14:45:10 DEBUG headers: << ETag: "d41d8cd98f00b204e9800998ecf8427e"
18/09/04 14:45:10 DEBUG headers: << Accept-Ranges: bytes
18/09/04 14:45:10 DEBUG headers: << Content-Type: application/octet-stream
18/09/04 14:45:10 DEBUG headers: << Content-Length: 0
18/09/04 14:45:10 DEBUG headers: << Server: AmazonS3
18/09/04 14:45:10 DEBUG SdkHttpClient: Connection can be kept alive
indefinitely
18/09/04 14:45:10 DEBUG request: Received successful response: 200, AWS
Request ID: A1AD0240EBDD2234
18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection [id:
1][route: {s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443] can be
kept alive indefinitely
18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection released:
[id: 1][route:
{s}->https://aa-sdk-test2.s3.us-east-1.amazonaws.com:443][total kept alive:
1; route allocated: 1 of 15; total allocated: 1 of 15]
18/09/04 14:45:10 DEBUG S3AFileSystem: OutputStream for key
'carbon-data/example/LockFiles/concurrentload.lock' writing to tempfile:
/tmp/hadoop-aaron/s3a/output-8508205130207286174.tmp
18/09/04 14:45:10 ERROR CarbonLoadDataCommand: main
java.lang.ArrayIndexOutOfBoundsException
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
at org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
at
org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.acquireConcurrentLoadLock(CarbonLoadDataCommand.scala:399)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:259)
at
org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
at org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
at org.apache.carbondata.examples.S3Example$.main(S3Example.scala:91)
at org.apache.carbondata.examples.S3Example.main(S3Example.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
18/09/04 14:45:10 AUDIT CarbonLoadDataCommand:
[aaron.lan.appannie.com][aaron][Thread-1]Dataload failure for
default.carbon_table. Please check the logs
18/09/04 14:45:10 DEBUG Client: The ping interval is 60000 ms.
18/09/04 14:45:10 DEBUG Client: Connecting to localhost/127.0.0.1:9000
18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
localhost/127.0.0.1:9000 from aaron: starting, having connections 1
18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
localhost/127.0.0.1:9000 from aaron sending #3
18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
localhost/127.0.0.1:9000 from aaron got value #3
18/09/04 14:45:10 DEBUG ProtobufRpcEngine: Call: getFileInfo took 6ms
18/09/04 14:45:10 DEBUG AbstractDFSCarbonFile: main Exception occurred:File
does not exist:
hdfs://localhost:9000/usr/carbon-meta/partition/default/carbon_table
18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
localhost/127.0.0.1:9000 from aaron sending #4
18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
localhost/127.0.0.1:9000 from aaron got value #4
18/09/04 14:45:10 DEBUG ProtobufRpcEngine: Call: getFileInfo took 3ms
18/09/04 14:45:10 ERROR CarbonLoadDataCommand: main Got exception
java.lang.ArrayIndexOutOfBoundsException when processing data. But this
command does not support undo yet, skipping the undo part.
Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
at org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
at
org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.acquireConcurrentLoadLock(CarbonLoadDataCommand.scala:399)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:259)
at
org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
at org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
at org.apache.carbondata.examples.S3Example$.main(S3Example.scala:91)
at org.apache.carbondata.examples.S3Example.main(S3Example.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
18/09/04 14:45:10 INFO SparkContext: Invoking stop() from shutdown hook
18/09/04 14:45:10 INFO SparkUI: Stopped Spark web UI at
http://localhost:4040
18/09/04 14:45:10 INFO MapOutputTrackerMasterEndpoint:
MapOutputTrackerMasterEndpoint stopped!
18/09/04 14:45:10 INFO MemoryStore: MemoryStore cleared
18/09/04 14:45:10 INFO BlockManager: BlockManager stopped
18/09/04 14:45:10 INFO BlockManagerMaster: BlockManagerMaster stopped
18/09/04 14:45:10 INFO
OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:
OutputCommitCoordinator stopped!
18/09/04 14:45:10 INFO SparkContext: Successfully stopped SparkContext
18/09/04 14:45:10 INFO ShutdownHookManager: Shutdown hook called
18/09/04 14:45:10 INFO ShutdownHookManager: Deleting directory
/private/var/folders/dd/n9pmb1nj0dncx5rd_s2rm9_40000gn/T/spark-f1e5dab8-a7db-4107-a3bf-c7253ba7ac06
18/09/04 14:45:10 DEBUG IdleConnectionReaper: Reaper thread:
java.lang.InterruptedException: sleep interrupted
at java.lang.Thread.sleep(Native Method)
at
com.amazonaws.http.IdleConnectionReaper.run(IdleConnectionReaper.java:112)
18/09/04 14:45:10 DEBUG IdleConnectionReaper: Shutting down reaper thread.
18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection manager
is shutting down
18/09/04 14:45:10 DEBUG DefaultClientConnection: Connection
0.0.0.0:59398<->54.231.82.12:443 closed
18/09/04 14:45:10 DEBUG DefaultClientConnection: Connection
0.0.0.0:59398<->54.231.82.12:443 closed
18/09/04 14:45:10 DEBUG PoolingClientConnectionManager: Connection manager
shut down
18/09/04 14:45:10 DEBUG Client: stopping client from cache:
org.apache.hadoop.ipc.Client@18ab86a2
18/09/04 14:45:10 DEBUG Client: removing client from cache:
org.apache.hadoop.ipc.Client@18ab86a2
18/09/04 14:45:10 DEBUG Client: stopping actual client because no more
references remain: org.apache.hadoop.ipc.Client@18ab86a2
18/09/04 14:45:10 DEBUG Client: Stopping client
18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
localhost/127.0.0.1:9000 from aaron: closed
18/09/04 14:45:10 DEBUG Client: IPC Client (777046609) connection to
localhost/127.0.0.1:9000 from aaron: stopped, remaining connections 0
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by Kunal Kapoor <ku...@gmail.com>.
Ok. Let me have a look
On Tue, Sep 4, 2018, 8:22 AM aaron <94...@qq.com> wrote:
> Hi kunalkapoor,
> It seems that error not fixed yet. Do you have any idea?
>
> thanks
> aaron
>
> aaron:2.2.1 aaron$ spark-shell --executor-memory 4g --driver-memory 2g
> Ivy Default Cache set to: /Users/aaron/.ivy2/cache
> The jars for the packages stored in: /Users/aaron/.ivy2/jars
> :: loading settings :: url =
>
> jar:file:/usr/local/Cellar/apache-spark/2.2.1/lib/apache-carbondata-1.5.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar!/org/apache/ivy/core/settings/ivysettings.xml
> com.amazonaws#aws-java-sdk added as a dependency
> org.apache.hadoop#hadoop-aws added as a dependency
> com.databricks#spark-avro_2.11 added as a dependency
> :: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0
> confs: [default]
> found com.amazonaws#aws-java-sdk;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-support;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-core;1.10.75.1 in central
> found commons-logging#commons-logging;1.1.3 in central
> found org.apache.httpcomponents#httpclient;4.3.6 in local-m2-cache
> found org.apache.httpcomponents#httpcore;4.3.3 in local-m2-cache
> found commons-codec#commons-codec;1.6 in local-m2-cache
> found com.fasterxml.jackson.core#jackson-databind;2.5.3 in central
> found com.fasterxml.jackson.core#jackson-annotations;2.5.0 in
> central
> found com.fasterxml.jackson.core#jackson-core;2.5.3 in central
> found
> com.fasterxml.jackson.dataformat#jackson-dataformat-cbor;2.5.3 in
> central
> found joda-time#joda-time;2.8.1 in central
> found com.amazonaws#aws-java-sdk-simpledb;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-simpleworkflow;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-storagegateway;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-route53;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-s3;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-kms;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-importexport;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-sts;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-sqs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-rds;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-redshift;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticbeanstalk;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-glacier;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-sns;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-iam;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-datapipeline;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticloadbalancing;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-emr;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticache;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elastictranscoder;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-ec2;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-dynamodb;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudtrail;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudwatch;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-logs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-events;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cognitoidentity;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-cognitosync;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-directconnect;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudformation;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-cloudfront;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-kinesis;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-opsworks;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ses;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-autoscaling;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudsearch;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudwatchmetrics;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-swf-libraries;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-codedeploy;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-codepipeline;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-config;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-lambda;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ecs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ecr;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-cloudhsm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-ssm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-workspaces;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-machinelearning;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-directory;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-efs;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-codecommit;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-devicefarm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-elasticsearch;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-waf;1.10.75.1 in central
> found
> com.amazonaws#aws-java-sdk-marketplacecommerceanalytics;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-inspector;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-iot;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-api-gateway;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-acm;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-gamelift;1.10.75.1 in central
> found com.amazonaws#aws-java-sdk-dms;1.10.75.1 in central
> found
> com.amazonaws#aws-java-sdk-marketplacemeteringservice;1.10.75.1 in
> central
> found com.amazonaws#aws-java-sdk-cognitoidp;1.10.75.1 in central
> found org.apache.hadoop#hadoop-aws;2.7.3 in central
> found org.apache.hadoop#hadoop-common;2.7.3 in local-m2-cache
> found org.apache.hadoop#hadoop-annotations;2.7.3 in local-m2-cache
> found com.google.guava#guava;11.0.2 in local-m2-cache
> found com.google.code.findbugs#jsr305;3.0.0 in local-m2-cache
> found commons-cli#commons-cli;1.2 in local-m2-cache
> found org.apache.commons#commons-math3;3.1.1 in local-m2-cache
> found xmlenc#xmlenc;0.52 in local-m2-cache
> found commons-httpclient#commons-httpclient;3.1 in local-m2-cache
> found commons-io#commons-io;2.4 in local-m2-cache
> found commons-net#commons-net;3.1 in local-m2-cache
> found commons-collections#commons-collections;3.2.2 in
> local-m2-cache
> found javax.servlet#servlet-api;2.5 in local-m2-cache
> found org.mortbay.jetty#jetty;6.1.26 in local-m2-cache
> found org.mortbay.jetty#jetty-util;6.1.26 in local-m2-cache
> found com.sun.jersey#jersey-core;1.9 in local-m2-cache
> found com.sun.jersey#jersey-json;1.9 in local-m2-cache
> found org.codehaus.jettison#jettison;1.1 in local-m2-cache
> found com.sun.xml.bind#jaxb-impl;2.2.3-1 in local-m2-cache
> found javax.xml.bind#jaxb-api;2.2.2 in local-m2-cache
> found javax.xml.stream#stax-api;1.0-2 in local-m2-cache
> found javax.activation#activation;1.1 in local-m2-cache
> found org.codehaus.jackson#jackson-core-asl;1.9.13 in
> local-m2-cache
> found org.codehaus.jackson#jackson-mapper-asl;1.9.13 in
> local-m2-cache
> found org.codehaus.jackson#jackson-jaxrs;1.9.13 in local-m2-cache
> found org.codehaus.jackson#jackson-xc;1.9.13 in local-m2-cache
> found com.sun.jersey#jersey-server;1.9 in local-m2-cache
> found asm#asm;3.2 in local-m2-cache
> found log4j#log4j;1.2.17 in local-m2-cache
> found net.java.dev.jets3t#jets3t;0.9.0 in local-m2-cache
> found com.jamesmurty.utils#java-xmlbuilder;0.4 in local-m2-cache
> found commons-lang#commons-lang;2.6 in central
> found commons-configuration#commons-configuration;1.6 in
> local-m2-cache
> found commons-digester#commons-digester;1.8 in local-m2-cache
> found commons-beanutils#commons-beanutils;1.7.0 in local-m2-cache
> found commons-beanutils#commons-beanutils-core;1.8.0 in
> local-m2-cache
> found org.slf4j#slf4j-api;1.7.10 in local-m2-cache
> found org.apache.avro#avro;1.7.4 in local-m2-cache
> found com.thoughtworks.paranamer#paranamer;2.3 in local-m2-cache
> found org.xerial.snappy#snappy-java;1.0.4.1 in local-m2-cache
> found org.apache.commons#commons-compress;1.4.1 in local-m2-cache
> found org.tukaani#xz;1.0 in local-m2-cache
> found com.google.protobuf#protobuf-java;2.5.0 in central
> found com.google.code.gson#gson;2.2.4 in local-m2-cache
> found org.apache.hadoop#hadoop-auth;2.7.3 in local-m2-cache
> found
> org.apache.directory.server#apacheds-kerberos-codec;2.0.0-M15 in
> local-m2-cache
> found org.apache.directory.server#apacheds-i18n;2.0.0-M15 in
> local-m2-cache
> found org.apache.directory.api#api-asn1-api;1.0.0-M20 in
> local-m2-cache
> found org.apache.directory.api#api-util;1.0.0-M20 in local-m2-cache
> found org.apache.zookeeper#zookeeper;3.4.6 in local-m2-cache
> found org.slf4j#slf4j-log4j12;1.7.10 in local-m2-cache
> found io.netty#netty;3.6.2.Final in local-m2-cache
> found org.apache.curator#curator-framework;2.7.1 in local-m2-cache
> found org.apache.curator#curator-client;2.7.1 in local-m2-cache
> found com.jcraft#jsch;0.1.42 in local-m2-cache
> found org.apache.curator#curator-recipes;2.7.1 in local-m2-cache
> found org.apache.htrace#htrace-core;3.1.0-incubating in
> local-m2-cache
> found javax.servlet.jsp#jsp-api;2.1 in local-m2-cache
> found jline#jline;0.9.94 in local-m2-cache
> found junit#junit;4.11 in local-m2-cache
> found org.hamcrest#hamcrest-core;1.3 in local-m2-cache
> found com.databricks#spark-avro_2.11;4.0.0 in central
> found org.apache.avro#avro;1.7.6 in central
> found org.xerial.snappy#snappy-java;1.0.5 in local-m2-cache
> :: resolution report :: resolve 3300ms :: artifacts dl 303ms
> :: modules in use:
> asm#asm;3.2 from local-m2-cache in [default]
> com.amazonaws#aws-java-sdk;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-acm;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-api-gateway;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-autoscaling;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudformation;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudfront;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudhsm;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudsearch;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudtrail;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudwatch;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cloudwatchmetrics;1.10.75.1 from
> central in
> [default]
> com.amazonaws#aws-java-sdk-codecommit;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-codedeploy;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-codepipeline;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cognitoidentity;1.10.75.1 from central
> in
> [default]
> com.amazonaws#aws-java-sdk-cognitoidp;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-cognitosync;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-config;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-core;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-datapipeline;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-devicefarm;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-directconnect;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-directory;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-dms;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-dynamodb;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-ec2;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ecr;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ecs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-efs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-elasticache;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-elasticbeanstalk;1.10.75.1 from central
> in
> [default]
> com.amazonaws#aws-java-sdk-elasticloadbalancing;1.10.75.1 from
> central in
> [default]
> com.amazonaws#aws-java-sdk-elasticsearch;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-elastictranscoder;1.10.75.1 from
> central in
> [default]
> com.amazonaws#aws-java-sdk-emr;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-events;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-gamelift;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-glacier;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-iam;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-importexport;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-inspector;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-iot;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-kinesis;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-kms;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-lambda;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-logs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-machinelearning;1.10.75.1 from central
> in
> [default]
> com.amazonaws#aws-java-sdk-marketplacecommerceanalytics;1.10.75.1
> from
> central in [default]
> com.amazonaws#aws-java-sdk-marketplacemeteringservice;1.10.75.1
> from
> central in [default]
> com.amazonaws#aws-java-sdk-opsworks;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-rds;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-redshift;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-route53;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-s3;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ses;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-simpledb;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-simpleworkflow;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-sns;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-sqs;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-ssm;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-storagegateway;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-sts;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-support;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-swf-libraries;1.10.75.1 from central in
> [default]
> com.amazonaws#aws-java-sdk-waf;1.10.75.1 from central in [default]
> com.amazonaws#aws-java-sdk-workspaces;1.10.75.1 from central in
> [default]
> com.databricks#spark-avro_2.11;4.0.0 from central in [default]
> com.fasterxml.jackson.core#jackson-annotations;2.5.0 from central
> in
> [default]
> com.fasterxml.jackson.core#jackson-core;2.5.3 from central in
> [default]
> com.fasterxml.jackson.core#jackson-databind;2.5.3 from central in
> [default]
> com.fasterxml.jackson.dataformat#jackson-dataformat-cbor;2.5.3
> from central
> in [default]
> com.google.code.findbugs#jsr305;3.0.0 from local-m2-cache in
> [default]
> com.google.code.gson#gson;2.2.4 from local-m2-cache in [default]
> com.google.guava#guava;11.0.2 from local-m2-cache in [default]
> com.google.protobuf#protobuf-java;2.5.0 from central in [default]
> com.jamesmurty.utils#java-xmlbuilder;0.4 from local-m2-cache in
> [default]
> com.jcraft#jsch;0.1.42 from local-m2-cache in [default]
> com.sun.jersey#jersey-core;1.9 from local-m2-cache in [default]
> com.sun.jersey#jersey-json;1.9 from local-m2-cache in [default]
> com.sun.jersey#jersey-server;1.9 from local-m2-cache in [default]
> com.sun.xml.bind#jaxb-impl;2.2.3-1 from local-m2-cache in [default]
> com.thoughtworks.paranamer#paranamer;2.3 from local-m2-cache in
> [default]
> commons-beanutils#commons-beanutils;1.7.0 from local-m2-cache in
> [default]
> commons-beanutils#commons-beanutils-core;1.8.0 from local-m2-cache
> in
> [default]
> commons-cli#commons-cli;1.2 from local-m2-cache in [default]
> commons-codec#commons-codec;1.6 from local-m2-cache in [default]
> commons-collections#commons-collections;3.2.2 from local-m2-cache
> in
> [default]
> commons-configuration#commons-configuration;1.6 from
> local-m2-cache in
> [default]
> commons-digester#commons-digester;1.8 from local-m2-cache in
> [default]
> commons-httpclient#commons-httpclient;3.1 from local-m2-cache in
> [default]
> commons-io#commons-io;2.4 from local-m2-cache in [default]
> commons-lang#commons-lang;2.6 from central in [default]
> commons-logging#commons-logging;1.1.3 from central in [default]
> commons-net#commons-net;3.1 from local-m2-cache in [default]
> io.netty#netty;3.6.2.Final from local-m2-cache in [default]
> javax.activation#activation;1.1 from local-m2-cache in [default]
> javax.servlet#servlet-api;2.5 from local-m2-cache in [default]
> javax.servlet.jsp#jsp-api;2.1 from local-m2-cache in [default]
> javax.xml.bind#jaxb-api;2.2.2 from local-m2-cache in [default]
> javax.xml.stream#stax-api;1.0-2 from local-m2-cache in [default]
> jline#jline;0.9.94 from local-m2-cache in [default]
> joda-time#joda-time;2.8.1 from central in [default]
> junit#junit;4.11 from local-m2-cache in [default]
> log4j#log4j;1.2.17 from local-m2-cache in [default]
> net.java.dev.jets3t#jets3t;0.9.0 from local-m2-cache in [default]
> org.apache.avro#avro;1.7.6 from central in [default]
> org.apache.commons#commons-compress;1.4.1 from local-m2-cache in
> [default]
> org.apache.commons#commons-math3;3.1.1 from local-m2-cache in
> [default]
> org.apache.curator#curator-client;2.7.1 from local-m2-cache in
> [default]
> org.apache.curator#curator-framework;2.7.1 from local-m2-cache in
> [default]
> org.apache.curator#curator-recipes;2.7.1 from local-m2-cache in
> [default]
> org.apache.directory.api#api-asn1-api;1.0.0-M20 from
> local-m2-cache in
> [default]
> org.apache.directory.api#api-util;1.0.0-M20 from local-m2-cache in
> [default]
> org.apache.directory.server#apacheds-i18n;2.0.0-M15 from
> local-m2-cache in
> [default]
> org.apache.directory.server#apacheds-kerberos-codec;2.0.0-M15 from
> local-m2-cache in [default]
> org.apache.hadoop#hadoop-annotations;2.7.3 from local-m2-cache in
> [default]
> org.apache.hadoop#hadoop-auth;2.7.3 from local-m2-cache in
> [default]
> org.apache.hadoop#hadoop-aws;2.7.3 from central in [default]
> org.apache.hadoop#hadoop-common;2.7.3 from local-m2-cache in
> [default]
> org.apache.htrace#htrace-core;3.1.0-incubating from local-m2-cache
> in
> [default]
> org.apache.httpcomponents#httpclient;4.3.6 from local-m2-cache in
> [default]
> org.apache.httpcomponents#httpcore;4.3.3 from local-m2-cache in
> [default]
> org.apache.zookeeper#zookeeper;3.4.6 from local-m2-cache in
> [default]
> org.codehaus.jackson#jackson-core-asl;1.9.13 from local-m2-cache in
> [default]
> org.codehaus.jackson#jackson-jaxrs;1.9.13 from local-m2-cache in
> [default]
> org.codehaus.jackson#jackson-mapper-asl;1.9.13 from local-m2-cache
> in
> [default]
> org.codehaus.jackson#jackson-xc;1.9.13 from local-m2-cache in
> [default]
> org.codehaus.jettison#jettison;1.1 from local-m2-cache in [default]
> org.hamcrest#hamcrest-core;1.3 from local-m2-cache in [default]
> org.mortbay.jetty#jetty;6.1.26 from local-m2-cache in [default]
> org.mortbay.jetty#jetty-util;6.1.26 from local-m2-cache in
> [default]
> org.slf4j#slf4j-api;1.7.10 from local-m2-cache in [default]
> org.slf4j#slf4j-log4j12;1.7.10 from local-m2-cache in [default]
> org.tukaani#xz;1.0 from local-m2-cache in [default]
> org.xerial.snappy#snappy-java;1.0.5 from local-m2-cache in
> [default]
> xmlenc#xmlenc;0.52 from local-m2-cache in [default]
> :: evicted modules:
> commons-codec#commons-codec;1.4 by
> [commons-codec#commons-codec;1.6] in
> [default]
> org.apache.httpcomponents#httpclient;4.2.5 by
> [org.apache.httpcomponents#httpclient;4.3.6] in [default]
> org.apache.httpcomponents#httpcore;4.2.5 by
> [org.apache.httpcomponents#httpcore;4.3.3] in [default]
> org.apache.avro#avro;1.7.4 by [org.apache.avro#avro;1.7.6] in
> [default]
> org.xerial.snappy#snappy-java;1.0.4.1 by
> [org.xerial.snappy#snappy-java;1.0.5] in [default]
> com.fasterxml.jackson.core#jackson-databind;2.2.3 by
> [com.fasterxml.jackson.core#jackson-databind;2.5.3] in [default]
> com.fasterxml.jackson.core#jackson-annotations;2.2.3 by
> [com.fasterxml.jackson.core#jackson-annotations;2.5.0] in [default]
> com.amazonaws#aws-java-sdk;1.7.4 by
> [com.amazonaws#aws-java-sdk;1.10.75.1]
> in [default]
> org.slf4j#slf4j-api;1.7.5 by [org.slf4j#slf4j-api;1.7.10] in
> [default]
> org.slf4j#slf4j-api;1.6.4 by [org.slf4j#slf4j-api;1.7.10] in
> [default]
>
> ---------------------------------------------------------------------
> | | modules || artifacts
> |
> | conf | number| search|dwnlded|evicted||
> number|dwnlded|
>
> ---------------------------------------------------------------------
> | default | 147 | 0 | 0 | 10 || 137 | 0
> |
>
> ---------------------------------------------------------------------
> :: retrieving :: org.apache.spark#spark-submit-parent
> confs: [default]
> 0 artifacts copied, 137 already retrieved (0kB/48ms)
> SLF4J: Class path contains multiple SLF4J bindings.
> SLF4J: Found binding in
>
> [jar:file:/usr/local/Cellar/apache-spark/2.2.1/lib/apache-carbondata-1.5.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: Found binding in
>
> [jar:file:/usr/local/Cellar/apache-spark/2.2.1/jars/slf4j-log4j12-1.7.16.jar!/org/slf4j/impl/StaticLoggerBinder.class]
> SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an
> explanation.
> SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
> Using Spark's default log4j profile:
> org/apache/spark/log4j-defaults.properties
> Setting default log level to "WARN".
> To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use
> setLogLevel(newLevel).
> 18/09/04 10:41:37 WARN NativeCodeLoader: Unable to load native-hadoop
> library for your platform... using builtin-java classes where applicable
> 18/09/04 10:41:37 WARN Utils: Your hostname, aaron.lan.appannie.com
> resolves
> to a loopback address: 127.0.0.1; using 192.168.2.212 instead (on interface
> en0)
> 18/09/04 10:41:37 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to
> another address
> Spark context Web UI available at http://192.168.2.212:4040
> Spark context available as 'sc' (master = local[*], app id =
> local-1536028898657).
> Spark session available as 'spark'.
> Welcome to
> ____ __
> / __/__ ___ _____/ /__
> _\ \/ _ \/ _ `/ __/ '_/
> /___/ .__/\_,_/_/ /_/\_\ version 2.2.1
> /_/
>
> Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java
> 1.8.0_144)
> Type in expressions to have them evaluated.
> Type :help for more information.
>
> scala> import org.apache.spark.sql.SparkSession
> import org.apache.spark.sql.SparkSession
>
> scala> import org.apache.spark.sql.CarbonSession._
> import org.apache.spark.sql.CarbonSession._
>
> scala> import org.apache.spark.sql.catalyst.util._
> import org.apache.spark.sql.catalyst.util._
>
> scala> val carbon =
>
> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
>
> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
> "xxx").config("spark.hadoop.fs.s3a.secret.key",
> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
> 18/09/04 10:41:58 WARN SparkContext: Using an existing SparkContext; some
> configuration may not take effect.
> 18/09/04 10:41:59 WARN CarbonProperties: main The enable off heap sort
> value
> "null" is invalid. Using the default value "true
> 18/09/04 10:41:59 WARN CarbonProperties: main The specified value for
> property sort.inmemory.size.inmbis Invalid. Taking the default value.1024
> 18/09/04 10:42:05 WARN ObjectStore: Failed to get database global_temp,
> returning NoSuchObjectException
> carbon: org.apache.spark.sql.SparkSession =
> org.apache.spark.sql.CarbonSession@2d492d46
>
> scala> carbon.sql("CREATE TABLE IF NOT EXISTS test_s3_table(id string, name
> string, city string, age Int) STORED BY 'carbondata' LOCATION
> 's3a://sdk-demo-sg/'")
> 18/09/04 10:42:52 AUDIT CarbonCreateTableCommand:
> [aaron.lan.appannie.com][aaron][Thread-1]Creating Table with Database name
> [default] and Table name [test_s3_table]
> 18/09/04 10:43:01 WARN HiveExternalCatalog: Couldn't find corresponding
> Hive
> SerDe for data source provider org.apache.spark.sql.CarbonSource.
> Persisting
> data source table `default`.`test_s3_table` into Hive metastore in Spark
> SQL
> specific format, which is NOT compatible with Hive.
> 18/09/04 10:43:01 AUDIT CarbonCreateTableCommand:
> [aaron.lan.appannie.com][aaron][Thread-1]Table created with Database name
> [default] and Table name [test_s3_table]
> res1: org.apache.spark.sql.DataFrame = []
>
> scala> carbon.sql("describe extended test_s3_table").show(200,
> truncate=false)
> 18/09/04 10:50:02 ERROR CarbonUtil: main Unable to unlock Table lock for
> table during table status updation
>
> +------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
> |col_name |data_type
>
> |comment |
>
> +------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
> |id |string
>
> |KEY COLUMN,null |
> |name |string
>
> |KEY COLUMN,null |
> |city |string
>
> |KEY COLUMN,null |
> |age |int
>
> |MEASURE,null |
> | |
>
> | |
> |##Detailed Table Information |
>
> | |
> |Database Name |default
>
> | |
> |Table Name |test_s3_table
>
> | |
> |CARBON Store Path |s3a://sdk-demo-sg/
>
> | |
> |Comment |
>
> | |
> |Table Block Size |1024 MB
>
> | |
> |Table Data Size |0
>
> | |
> |Table Index Size |0
>
> | |
> |Last Update Time |0
>
> | |
> |SORT_SCOPE |LOCAL_SORT
>
> |LOCAL_SORT |
> |CACHE_LEVEL |BLOCK
>
> | |
> |Streaming |false
>
> | |
> |Local Dictionary Enabled |false
>
> | |
> | |
>
> | |
> |##Detailed Column property |
>
> | |
> |ADAPTIVE |
>
> | |
> |SORT_COLUMNS |id,name,city
>
> | |
>
> +------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
>
> scala> carbon.sql("LOAD DATA INPATH
> 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE test_s3_table")
> 18/09/04 10:43:23 ERROR CarbonLoaderUtil: main Unable to unlock Table lock
> for tabledefault.test_s3_table during table status updation
> 18/09/04 10:43:23 ERROR CarbonLoadDataCommand: main
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at
> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
> at
> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:264)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
> at
>
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)
> at
>
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:42)
> at
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:44)
> at
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:46)
> at
> $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:48)
> at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:50)
> at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:52)
> at $line23.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:54)
> at $line23.$read$$iw$$iw$$iw$$iw.<init>(<console>:56)
> at $line23.$read$$iw$$iw$$iw.<init>(<console>:58)
> at $line23.$read$$iw$$iw.<init>(<console>:60)
> at $line23.$read$$iw.<init>(<console>:62)
> at $line23.$read.<init>(<console>:64)
> at $line23.$read$.<init>(<console>:68)
> at $line23.$read$.<clinit>(<console>)
> at $line23.$eval$.$print$lzycompute(<console>:7)
> at $line23.$eval$.$print(<console>:6)
> at $line23.$eval.$print(<console>)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
> at
> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
> at
>
> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
> at
>
> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
> at
> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
> at
>
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
>
> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
> at org.apache.spark.repl.Main$.doMain(Main.scala:74)
> at org.apache.spark.repl.Main$.main(Main.scala:54)
> at org.apache.spark.repl.Main.main(Main.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
>
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
> at
> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> at
> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 18/09/04 10:43:23 AUDIT CarbonLoadDataCommand:
> [aaron.lan.appannie.com][aaron][Thread-1]Dataload failure for
> default.test_s3_table. Please check the logs
> 18/09/04 10:43:23 ERROR CarbonLoadDataCommand: main Got exception
> java.lang.ArrayIndexOutOfBoundsException when processing data. But this
> command does not support undo yet, skipping the undo part.
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
> at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:264)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
> ... 52 elided
>
> scala>
>
>
>
> --
> Sent from:
> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
>
>
Re: error occur when I load data to s3
Posted by aaron <94...@qq.com>.
Hi kunalkapoor,
It seems that error not fixed yet. Do you have any idea?
thanks
aaron
aaron:2.2.1 aaron$ spark-shell --executor-memory 4g --driver-memory 2g
Ivy Default Cache set to: /Users/aaron/.ivy2/cache
The jars for the packages stored in: /Users/aaron/.ivy2/jars
:: loading settings :: url =
jar:file:/usr/local/Cellar/apache-spark/2.2.1/lib/apache-carbondata-1.5.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar!/org/apache/ivy/core/settings/ivysettings.xml
com.amazonaws#aws-java-sdk added as a dependency
org.apache.hadoop#hadoop-aws added as a dependency
com.databricks#spark-avro_2.11 added as a dependency
:: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0
confs: [default]
found com.amazonaws#aws-java-sdk;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-support;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-core;1.10.75.1 in central
found commons-logging#commons-logging;1.1.3 in central
found org.apache.httpcomponents#httpclient;4.3.6 in local-m2-cache
found org.apache.httpcomponents#httpcore;4.3.3 in local-m2-cache
found commons-codec#commons-codec;1.6 in local-m2-cache
found com.fasterxml.jackson.core#jackson-databind;2.5.3 in central
found com.fasterxml.jackson.core#jackson-annotations;2.5.0 in central
found com.fasterxml.jackson.core#jackson-core;2.5.3 in central
found com.fasterxml.jackson.dataformat#jackson-dataformat-cbor;2.5.3 in
central
found joda-time#joda-time;2.8.1 in central
found com.amazonaws#aws-java-sdk-simpledb;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-simpleworkflow;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-storagegateway;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-route53;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-s3;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-kms;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-importexport;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-sts;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-sqs;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-rds;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-redshift;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-elasticbeanstalk;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-glacier;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-sns;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-iam;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-datapipeline;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-elasticloadbalancing;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-emr;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-elasticache;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-elastictranscoder;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-ec2;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-dynamodb;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cloudtrail;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cloudwatch;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-logs;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-events;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cognitoidentity;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cognitosync;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-directconnect;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cloudformation;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cloudfront;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-kinesis;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-opsworks;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-ses;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-autoscaling;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cloudsearch;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cloudwatchmetrics;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-swf-libraries;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-codedeploy;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-codepipeline;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-config;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-lambda;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-ecs;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-ecr;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-cloudhsm;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-ssm;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-workspaces;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-machinelearning;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-directory;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-efs;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-codecommit;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-devicefarm;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-elasticsearch;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-waf;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-marketplacecommerceanalytics;1.10.75.1 in
central
found com.amazonaws#aws-java-sdk-inspector;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-iot;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-api-gateway;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-acm;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-gamelift;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-dms;1.10.75.1 in central
found com.amazonaws#aws-java-sdk-marketplacemeteringservice;1.10.75.1 in
central
found com.amazonaws#aws-java-sdk-cognitoidp;1.10.75.1 in central
found org.apache.hadoop#hadoop-aws;2.7.3 in central
found org.apache.hadoop#hadoop-common;2.7.3 in local-m2-cache
found org.apache.hadoop#hadoop-annotations;2.7.3 in local-m2-cache
found com.google.guava#guava;11.0.2 in local-m2-cache
found com.google.code.findbugs#jsr305;3.0.0 in local-m2-cache
found commons-cli#commons-cli;1.2 in local-m2-cache
found org.apache.commons#commons-math3;3.1.1 in local-m2-cache
found xmlenc#xmlenc;0.52 in local-m2-cache
found commons-httpclient#commons-httpclient;3.1 in local-m2-cache
found commons-io#commons-io;2.4 in local-m2-cache
found commons-net#commons-net;3.1 in local-m2-cache
found commons-collections#commons-collections;3.2.2 in local-m2-cache
found javax.servlet#servlet-api;2.5 in local-m2-cache
found org.mortbay.jetty#jetty;6.1.26 in local-m2-cache
found org.mortbay.jetty#jetty-util;6.1.26 in local-m2-cache
found com.sun.jersey#jersey-core;1.9 in local-m2-cache
found com.sun.jersey#jersey-json;1.9 in local-m2-cache
found org.codehaus.jettison#jettison;1.1 in local-m2-cache
found com.sun.xml.bind#jaxb-impl;2.2.3-1 in local-m2-cache
found javax.xml.bind#jaxb-api;2.2.2 in local-m2-cache
found javax.xml.stream#stax-api;1.0-2 in local-m2-cache
found javax.activation#activation;1.1 in local-m2-cache
found org.codehaus.jackson#jackson-core-asl;1.9.13 in local-m2-cache
found org.codehaus.jackson#jackson-mapper-asl;1.9.13 in local-m2-cache
found org.codehaus.jackson#jackson-jaxrs;1.9.13 in local-m2-cache
found org.codehaus.jackson#jackson-xc;1.9.13 in local-m2-cache
found com.sun.jersey#jersey-server;1.9 in local-m2-cache
found asm#asm;3.2 in local-m2-cache
found log4j#log4j;1.2.17 in local-m2-cache
found net.java.dev.jets3t#jets3t;0.9.0 in local-m2-cache
found com.jamesmurty.utils#java-xmlbuilder;0.4 in local-m2-cache
found commons-lang#commons-lang;2.6 in central
found commons-configuration#commons-configuration;1.6 in local-m2-cache
found commons-digester#commons-digester;1.8 in local-m2-cache
found commons-beanutils#commons-beanutils;1.7.0 in local-m2-cache
found commons-beanutils#commons-beanutils-core;1.8.0 in local-m2-cache
found org.slf4j#slf4j-api;1.7.10 in local-m2-cache
found org.apache.avro#avro;1.7.4 in local-m2-cache
found com.thoughtworks.paranamer#paranamer;2.3 in local-m2-cache
found org.xerial.snappy#snappy-java;1.0.4.1 in local-m2-cache
found org.apache.commons#commons-compress;1.4.1 in local-m2-cache
found org.tukaani#xz;1.0 in local-m2-cache
found com.google.protobuf#protobuf-java;2.5.0 in central
found com.google.code.gson#gson;2.2.4 in local-m2-cache
found org.apache.hadoop#hadoop-auth;2.7.3 in local-m2-cache
found org.apache.directory.server#apacheds-kerberos-codec;2.0.0-M15 in
local-m2-cache
found org.apache.directory.server#apacheds-i18n;2.0.0-M15 in local-m2-cache
found org.apache.directory.api#api-asn1-api;1.0.0-M20 in local-m2-cache
found org.apache.directory.api#api-util;1.0.0-M20 in local-m2-cache
found org.apache.zookeeper#zookeeper;3.4.6 in local-m2-cache
found org.slf4j#slf4j-log4j12;1.7.10 in local-m2-cache
found io.netty#netty;3.6.2.Final in local-m2-cache
found org.apache.curator#curator-framework;2.7.1 in local-m2-cache
found org.apache.curator#curator-client;2.7.1 in local-m2-cache
found com.jcraft#jsch;0.1.42 in local-m2-cache
found org.apache.curator#curator-recipes;2.7.1 in local-m2-cache
found org.apache.htrace#htrace-core;3.1.0-incubating in local-m2-cache
found javax.servlet.jsp#jsp-api;2.1 in local-m2-cache
found jline#jline;0.9.94 in local-m2-cache
found junit#junit;4.11 in local-m2-cache
found org.hamcrest#hamcrest-core;1.3 in local-m2-cache
found com.databricks#spark-avro_2.11;4.0.0 in central
found org.apache.avro#avro;1.7.6 in central
found org.xerial.snappy#snappy-java;1.0.5 in local-m2-cache
:: resolution report :: resolve 3300ms :: artifacts dl 303ms
:: modules in use:
asm#asm;3.2 from local-m2-cache in [default]
com.amazonaws#aws-java-sdk;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-acm;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-api-gateway;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-autoscaling;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cloudformation;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-cloudfront;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cloudhsm;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cloudsearch;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cloudtrail;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cloudwatch;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cloudwatchmetrics;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-codecommit;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-codedeploy;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-codepipeline;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cognitoidentity;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-cognitoidp;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-cognitosync;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-config;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-core;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-datapipeline;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-devicefarm;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-directconnect;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-directory;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-dms;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-dynamodb;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-ec2;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-ecr;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-ecs;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-efs;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-elasticache;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-elasticbeanstalk;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-elasticloadbalancing;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-elasticsearch;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-elastictranscoder;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-emr;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-events;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-gamelift;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-glacier;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-iam;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-importexport;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-inspector;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-iot;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-kinesis;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-kms;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-lambda;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-logs;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-machinelearning;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-marketplacecommerceanalytics;1.10.75.1 from
central in [default]
com.amazonaws#aws-java-sdk-marketplacemeteringservice;1.10.75.1 from
central in [default]
com.amazonaws#aws-java-sdk-opsworks;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-rds;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-redshift;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-route53;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-s3;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-ses;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-simpledb;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-simpleworkflow;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-sns;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-sqs;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-ssm;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-storagegateway;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-sts;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-support;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-swf-libraries;1.10.75.1 from central in
[default]
com.amazonaws#aws-java-sdk-waf;1.10.75.1 from central in [default]
com.amazonaws#aws-java-sdk-workspaces;1.10.75.1 from central in [default]
com.databricks#spark-avro_2.11;4.0.0 from central in [default]
com.fasterxml.jackson.core#jackson-annotations;2.5.0 from central in
[default]
com.fasterxml.jackson.core#jackson-core;2.5.3 from central in [default]
com.fasterxml.jackson.core#jackson-databind;2.5.3 from central in [default]
com.fasterxml.jackson.dataformat#jackson-dataformat-cbor;2.5.3 from central
in [default]
com.google.code.findbugs#jsr305;3.0.0 from local-m2-cache in [default]
com.google.code.gson#gson;2.2.4 from local-m2-cache in [default]
com.google.guava#guava;11.0.2 from local-m2-cache in [default]
com.google.protobuf#protobuf-java;2.5.0 from central in [default]
com.jamesmurty.utils#java-xmlbuilder;0.4 from local-m2-cache in [default]
com.jcraft#jsch;0.1.42 from local-m2-cache in [default]
com.sun.jersey#jersey-core;1.9 from local-m2-cache in [default]
com.sun.jersey#jersey-json;1.9 from local-m2-cache in [default]
com.sun.jersey#jersey-server;1.9 from local-m2-cache in [default]
com.sun.xml.bind#jaxb-impl;2.2.3-1 from local-m2-cache in [default]
com.thoughtworks.paranamer#paranamer;2.3 from local-m2-cache in [default]
commons-beanutils#commons-beanutils;1.7.0 from local-m2-cache in [default]
commons-beanutils#commons-beanutils-core;1.8.0 from local-m2-cache in
[default]
commons-cli#commons-cli;1.2 from local-m2-cache in [default]
commons-codec#commons-codec;1.6 from local-m2-cache in [default]
commons-collections#commons-collections;3.2.2 from local-m2-cache in
[default]
commons-configuration#commons-configuration;1.6 from local-m2-cache in
[default]
commons-digester#commons-digester;1.8 from local-m2-cache in [default]
commons-httpclient#commons-httpclient;3.1 from local-m2-cache in [default]
commons-io#commons-io;2.4 from local-m2-cache in [default]
commons-lang#commons-lang;2.6 from central in [default]
commons-logging#commons-logging;1.1.3 from central in [default]
commons-net#commons-net;3.1 from local-m2-cache in [default]
io.netty#netty;3.6.2.Final from local-m2-cache in [default]
javax.activation#activation;1.1 from local-m2-cache in [default]
javax.servlet#servlet-api;2.5 from local-m2-cache in [default]
javax.servlet.jsp#jsp-api;2.1 from local-m2-cache in [default]
javax.xml.bind#jaxb-api;2.2.2 from local-m2-cache in [default]
javax.xml.stream#stax-api;1.0-2 from local-m2-cache in [default]
jline#jline;0.9.94 from local-m2-cache in [default]
joda-time#joda-time;2.8.1 from central in [default]
junit#junit;4.11 from local-m2-cache in [default]
log4j#log4j;1.2.17 from local-m2-cache in [default]
net.java.dev.jets3t#jets3t;0.9.0 from local-m2-cache in [default]
org.apache.avro#avro;1.7.6 from central in [default]
org.apache.commons#commons-compress;1.4.1 from local-m2-cache in [default]
org.apache.commons#commons-math3;3.1.1 from local-m2-cache in [default]
org.apache.curator#curator-client;2.7.1 from local-m2-cache in [default]
org.apache.curator#curator-framework;2.7.1 from local-m2-cache in [default]
org.apache.curator#curator-recipes;2.7.1 from local-m2-cache in [default]
org.apache.directory.api#api-asn1-api;1.0.0-M20 from local-m2-cache in
[default]
org.apache.directory.api#api-util;1.0.0-M20 from local-m2-cache in
[default]
org.apache.directory.server#apacheds-i18n;2.0.0-M15 from local-m2-cache in
[default]
org.apache.directory.server#apacheds-kerberos-codec;2.0.0-M15 from
local-m2-cache in [default]
org.apache.hadoop#hadoop-annotations;2.7.3 from local-m2-cache in [default]
org.apache.hadoop#hadoop-auth;2.7.3 from local-m2-cache in [default]
org.apache.hadoop#hadoop-aws;2.7.3 from central in [default]
org.apache.hadoop#hadoop-common;2.7.3 from local-m2-cache in [default]
org.apache.htrace#htrace-core;3.1.0-incubating from local-m2-cache in
[default]
org.apache.httpcomponents#httpclient;4.3.6 from local-m2-cache in [default]
org.apache.httpcomponents#httpcore;4.3.3 from local-m2-cache in [default]
org.apache.zookeeper#zookeeper;3.4.6 from local-m2-cache in [default]
org.codehaus.jackson#jackson-core-asl;1.9.13 from local-m2-cache in
[default]
org.codehaus.jackson#jackson-jaxrs;1.9.13 from local-m2-cache in [default]
org.codehaus.jackson#jackson-mapper-asl;1.9.13 from local-m2-cache in
[default]
org.codehaus.jackson#jackson-xc;1.9.13 from local-m2-cache in [default]
org.codehaus.jettison#jettison;1.1 from local-m2-cache in [default]
org.hamcrest#hamcrest-core;1.3 from local-m2-cache in [default]
org.mortbay.jetty#jetty;6.1.26 from local-m2-cache in [default]
org.mortbay.jetty#jetty-util;6.1.26 from local-m2-cache in [default]
org.slf4j#slf4j-api;1.7.10 from local-m2-cache in [default]
org.slf4j#slf4j-log4j12;1.7.10 from local-m2-cache in [default]
org.tukaani#xz;1.0 from local-m2-cache in [default]
org.xerial.snappy#snappy-java;1.0.5 from local-m2-cache in [default]
xmlenc#xmlenc;0.52 from local-m2-cache in [default]
:: evicted modules:
commons-codec#commons-codec;1.4 by [commons-codec#commons-codec;1.6] in
[default]
org.apache.httpcomponents#httpclient;4.2.5 by
[org.apache.httpcomponents#httpclient;4.3.6] in [default]
org.apache.httpcomponents#httpcore;4.2.5 by
[org.apache.httpcomponents#httpcore;4.3.3] in [default]
org.apache.avro#avro;1.7.4 by [org.apache.avro#avro;1.7.6] in [default]
org.xerial.snappy#snappy-java;1.0.4.1 by
[org.xerial.snappy#snappy-java;1.0.5] in [default]
com.fasterxml.jackson.core#jackson-databind;2.2.3 by
[com.fasterxml.jackson.core#jackson-databind;2.5.3] in [default]
com.fasterxml.jackson.core#jackson-annotations;2.2.3 by
[com.fasterxml.jackson.core#jackson-annotations;2.5.0] in [default]
com.amazonaws#aws-java-sdk;1.7.4 by [com.amazonaws#aws-java-sdk;1.10.75.1]
in [default]
org.slf4j#slf4j-api;1.7.5 by [org.slf4j#slf4j-api;1.7.10] in [default]
org.slf4j#slf4j-api;1.6.4 by [org.slf4j#slf4j-api;1.7.10] in [default]
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 147 | 0 | 0 | 10 || 137 | 0 |
---------------------------------------------------------------------
:: retrieving :: org.apache.spark#spark-submit-parent
confs: [default]
0 artifacts copied, 137 already retrieved (0kB/48ms)
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in
[jar:file:/usr/local/Cellar/apache-spark/2.2.1/lib/apache-carbondata-1.5.0-SNAPSHOT-bin-spark2.2.1-hadoop2.7.2.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in
[jar:file:/usr/local/Cellar/apache-spark/2.2.1/jars/slf4j-log4j12-1.7.16.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an
explanation.
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
Using Spark's default log4j profile:
org/apache/spark/log4j-defaults.properties
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use
setLogLevel(newLevel).
18/09/04 10:41:37 WARN NativeCodeLoader: Unable to load native-hadoop
library for your platform... using builtin-java classes where applicable
18/09/04 10:41:37 WARN Utils: Your hostname, aaron.lan.appannie.com resolves
to a loopback address: 127.0.0.1; using 192.168.2.212 instead (on interface
en0)
18/09/04 10:41:37 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to
another address
Spark context Web UI available at http://192.168.2.212:4040
Spark context available as 'sc' (master = local[*], app id =
local-1536028898657).
Spark session available as 'spark'.
Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/___/ .__/\_,_/_/ /_/\_\ version 2.2.1
/_/
Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java
1.8.0_144)
Type in expressions to have them evaluated.
Type :help for more information.
scala> import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.SparkSession
scala> import org.apache.spark.sql.CarbonSession._
import org.apache.spark.sql.CarbonSession._
scala> import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.catalyst.util._
scala> val carbon =
SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
"xxx").config("spark.hadoop.fs.s3a.secret.key",
"xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
18/09/04 10:41:58 WARN SparkContext: Using an existing SparkContext; some
configuration may not take effect.
18/09/04 10:41:59 WARN CarbonProperties: main The enable off heap sort value
"null" is invalid. Using the default value "true
18/09/04 10:41:59 WARN CarbonProperties: main The specified value for
property sort.inmemory.size.inmbis Invalid. Taking the default value.1024
18/09/04 10:42:05 WARN ObjectStore: Failed to get database global_temp,
returning NoSuchObjectException
carbon: org.apache.spark.sql.SparkSession =
org.apache.spark.sql.CarbonSession@2d492d46
scala> carbon.sql("CREATE TABLE IF NOT EXISTS test_s3_table(id string, name
string, city string, age Int) STORED BY 'carbondata' LOCATION
's3a://sdk-demo-sg/'")
18/09/04 10:42:52 AUDIT CarbonCreateTableCommand:
[aaron.lan.appannie.com][aaron][Thread-1]Creating Table with Database name
[default] and Table name [test_s3_table]
18/09/04 10:43:01 WARN HiveExternalCatalog: Couldn't find corresponding Hive
SerDe for data source provider org.apache.spark.sql.CarbonSource. Persisting
data source table `default`.`test_s3_table` into Hive metastore in Spark SQL
specific format, which is NOT compatible with Hive.
18/09/04 10:43:01 AUDIT CarbonCreateTableCommand:
[aaron.lan.appannie.com][aaron][Thread-1]Table created with Database name
[default] and Table name [test_s3_table]
res1: org.apache.spark.sql.DataFrame = []
scala> carbon.sql("describe extended test_s3_table").show(200,
truncate=false)
18/09/04 10:50:02 ERROR CarbonUtil: main Unable to unlock Table lock for
table during table status updation
+------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
|col_name |data_type
|comment |
+------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
|id |string
|KEY COLUMN,null |
|name |string
|KEY COLUMN,null |
|city |string
|KEY COLUMN,null |
|age |int
|MEASURE,null |
| |
| |
|##Detailed Table Information |
| |
|Database Name |default
| |
|Table Name |test_s3_table
| |
|CARBON Store Path |s3a://sdk-demo-sg/
| |
|Comment |
| |
|Table Block Size |1024 MB
| |
|Table Data Size |0
| |
|Table Index Size |0
| |
|Last Update Time |0
| |
|SORT_SCOPE |LOCAL_SORT
|LOCAL_SORT |
|CACHE_LEVEL |BLOCK
| |
|Streaming |false
| |
|Local Dictionary Enabled |false
| |
| |
| |
|##Detailed Column property |
| |
|ADAPTIVE |
| |
|SORT_COLUMNS |id,name,city
| |
+------------------------------------+--------------------------------------------------------------------------------+------------------------------------------------------------------------+
scala> carbon.sql("LOAD DATA INPATH
'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE test_s3_table")
18/09/04 10:43:23 ERROR CarbonLoaderUtil: main Unable to unlock Table lock
for tabledefault.test_s3_table during table status updation
18/09/04 10:43:23 ERROR CarbonLoadDataCommand: main
java.lang.ArrayIndexOutOfBoundsException
at java.lang.System.arraycopy(Native Method)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
at org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
at
org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:264)
at
org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
at org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
at
$line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:37)
at
$line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:42)
at
$line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:44)
at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:46)
at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:48)
at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:50)
at $line23.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:52)
at $line23.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:54)
at $line23.$read$$iw$$iw$$iw$$iw.<init>(<console>:56)
at $line23.$read$$iw$$iw$$iw.<init>(<console>:58)
at $line23.$read$$iw$$iw.<init>(<console>:60)
at $line23.$read$$iw.<init>(<console>:62)
at $line23.$read.<init>(<console>:64)
at $line23.$read$.<init>(<console>:68)
at $line23.$read$.<clinit>(<console>)
at $line23.$eval$.$print$lzycompute(<console>:7)
at $line23.$eval$.$print(<console>:6)
at $line23.$eval.$print(<console>)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
at
scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
at
scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
at
scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
at
scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
at
scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at
scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
at
scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
at org.apache.spark.repl.Main$.doMain(Main.scala:74)
at org.apache.spark.repl.Main$.main(Main.scala:54)
at org.apache.spark.repl.Main.main(Main.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
18/09/04 10:43:23 AUDIT CarbonLoadDataCommand:
[aaron.lan.appannie.com][aaron][Thread-1]Dataload failure for
default.test_s3_table. Please check the logs
18/09/04 10:43:23 ERROR CarbonLoadDataCommand: main Got exception
java.lang.ArrayIndexOutOfBoundsException when processing data. But this
command does not support undo yet, skipping the undo part.
java.lang.ArrayIndexOutOfBoundsException
at java.lang.System.arraycopy(Native Method)
at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
at
org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
at
org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
at java.io.DataOutputStream.write(DataOutputStream.java:107)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
at
org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
at
org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:289)
at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
at
org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
at
org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
at
org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:264)
at
org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
at
org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:106)
at
org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:95)
at
org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:153)
at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:93)
... 52 elided
scala>
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by aaron <94...@qq.com>.
Thanks, you're right. Succeed already!
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by xuchuanyin <xu...@hust.edu.cn>.
Did you build carbon with -Pbuild-with-format? it introduced Map datatype and changed the thrift, so you need to add it. On 09/04/2018 09:10, aaron wrote: Compile failed. My env is, aaron:carbondata aaron$ java -version java version "1.8.0_144" Java(TM) SE Runtime Environment (build 1.8.0_144-b01) Java HotSpot(TM) 64-Bit Server VM (build 25.144-b01, mixed mode) aaron:carbondata aaron$ mvn -v Apache Maven 3.5.2 (138edd61fd100ec658bfa2d307c43b76940a5d7d; 2017-10-18T15:58:13+08:00) Maven home: /usr/local/Cellar/maven/3.5.2/libexec Java version: 1.8.0_144, vendor: Oracle Corporation Java home: /Library/Java/JavaVirtualMachines/jdk1.8.0_144.jdk/Contents/Home/jre Default locale: en_US, platform encoding: UTF-8 OS name: "mac os x", version: "10.13.6", arch: "x86_64", family: "mac" aaron:carbondata aaron$ scala -version Scala code runner version 2.11.8 -- Copyright 2002-2016, LAMP/EPFL Error info is, [ERROR] COMPILATION ERROR : [INFO] ------------------------------------------------------------- [ERROR] /Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java:[2230,12] an enum switch case label must be the unqualified name of an enumeration constant [ERROR] /Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[160,51] cannot find symbol symbol: variable MAP location: class org.apache.carbondata.format.DataType [ERROR] /Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[501,12] an enum switch case label must be the unqualified name of an enumeration constant [INFO] 3 errors [INFO] ------------------------------------------------------------- [INFO] ------------------------------------------------------------------------ [INFO] Reactor Summary: [INFO] [INFO] Apache CarbonData :: Parent ........................ SUCCESS [ 3.251 s] [INFO] Apache CarbonData :: Common ........................ SUCCESS [ 9.868 s] [INFO] Apache CarbonData :: Core .......................... FAILURE [ 5.734 s] [INFO] Apache CarbonData :: Processing .................... SKIPPED [INFO] Apache CarbonData :: Hadoop ........................ SKIPPED [INFO] Apache CarbonData :: Streaming ..................... SKIPPED [INFO] Apache CarbonData :: Store SDK ..................... SKIPPED [INFO] Apache CarbonData :: Spark Datasource .............. SKIPPED [INFO] Apache CarbonData :: Spark Common .................. SKIPPED [INFO] Apache CarbonData :: Search ........................ SKIPPED [INFO] Apache CarbonData :: Lucene Index DataMap .......... SKIPPED [INFO] Apache CarbonData :: Bloom Index DataMap ........... SKIPPED [INFO] Apache CarbonData :: Spark2 ........................ SKIPPED [INFO] Apache CarbonData :: Spark Common Test ............. SKIPPED [INFO] Apache CarbonData :: DataMap Examples .............. SKIPPED [INFO] Apache CarbonData :: Assembly ...................... SKIPPED [INFO] Apache CarbonData :: Hive .......................... SKIPPED [INFO] Apache CarbonData :: presto ........................ SKIPPED [INFO] Apache CarbonData :: Spark2 Examples ............... SKIPPED [INFO] ------------------------------------------------------------------------ [INFO] BUILD FAILURE [INFO] ------------------------------------------------------------------------ [INFO] Total time: 19.595 s [INFO] Finished at: 2018-09-04T09:06:59+08:00 [INFO] Final Memory: 56M/583M [INFO] ------------------------------------------------------------------------ [ERROR] Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.2:compile (default-compile) on project carbondata-core: Compilation failure: Compilation failure: [ERROR] /Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java:[2230,12] an enum switch case label must be the unqualified name of an enumeration constant [ERROR] /Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[160,51] cannot find symbol [ERROR] symbol: variable MAP [ERROR] location: class org.apache.carbondata.format.DataType [ERROR] /Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[501,12] an enum switch case label must be the unqualified name of an enumeration constant [ERROR] -> [Help 1] [ERROR] [ERROR] To see the full stack trace of the errors, re-run Maven with the -e switch. [ERROR] Re-run Maven using the -X switch to enable full debug logging. [ERROR] [ERROR] For more information about the errors and possible solutions, please read the following articles: [ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException [ERROR] [ERROR] After correcting the problems, you can resume the build with the command [ERROR] mvn <goals> -rf :carbondata-core -- Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by aaron <94...@qq.com>.
Compile failed.
My env is,
aaron:carbondata aaron$ java -version
java version "1.8.0_144"
Java(TM) SE Runtime Environment (build 1.8.0_144-b01)
Java HotSpot(TM) 64-Bit Server VM (build 25.144-b01, mixed mode)
aaron:carbondata aaron$ mvn -v
Apache Maven 3.5.2 (138edd61fd100ec658bfa2d307c43b76940a5d7d;
2017-10-18T15:58:13+08:00)
Maven home: /usr/local/Cellar/maven/3.5.2/libexec
Java version: 1.8.0_144, vendor: Oracle Corporation
Java home:
/Library/Java/JavaVirtualMachines/jdk1.8.0_144.jdk/Contents/Home/jre
Default locale: en_US, platform encoding: UTF-8
OS name: "mac os x", version: "10.13.6", arch: "x86_64", family: "mac"
aaron:carbondata aaron$ scala -version
Scala code runner version 2.11.8 -- Copyright 2002-2016, LAMP/EPFL
Error info is,
[ERROR] COMPILATION ERROR :
[INFO] -------------------------------------------------------------
[ERROR]
/Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java:[2230,12]
an enum switch case label must be the unqualified name of an enumeration
constant
[ERROR]
/Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[160,51]
cannot find symbol
symbol: variable MAP
location: class org.apache.carbondata.format.DataType
[ERROR]
/Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[501,12]
an enum switch case label must be the unqualified name of an enumeration
constant
[INFO] 3 errors
[INFO] -------------------------------------------------------------
[INFO]
------------------------------------------------------------------------
[INFO] Reactor Summary:
[INFO]
[INFO] Apache CarbonData :: Parent ........................ SUCCESS [ 3.251
s]
[INFO] Apache CarbonData :: Common ........................ SUCCESS [ 9.868
s]
[INFO] Apache CarbonData :: Core .......................... FAILURE [ 5.734
s]
[INFO] Apache CarbonData :: Processing .................... SKIPPED
[INFO] Apache CarbonData :: Hadoop ........................ SKIPPED
[INFO] Apache CarbonData :: Streaming ..................... SKIPPED
[INFO] Apache CarbonData :: Store SDK ..................... SKIPPED
[INFO] Apache CarbonData :: Spark Datasource .............. SKIPPED
[INFO] Apache CarbonData :: Spark Common .................. SKIPPED
[INFO] Apache CarbonData :: Search ........................ SKIPPED
[INFO] Apache CarbonData :: Lucene Index DataMap .......... SKIPPED
[INFO] Apache CarbonData :: Bloom Index DataMap ........... SKIPPED
[INFO] Apache CarbonData :: Spark2 ........................ SKIPPED
[INFO] Apache CarbonData :: Spark Common Test ............. SKIPPED
[INFO] Apache CarbonData :: DataMap Examples .............. SKIPPED
[INFO] Apache CarbonData :: Assembly ...................... SKIPPED
[INFO] Apache CarbonData :: Hive .......................... SKIPPED
[INFO] Apache CarbonData :: presto ........................ SKIPPED
[INFO] Apache CarbonData :: Spark2 Examples ............... SKIPPED
[INFO]
------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO]
------------------------------------------------------------------------
[INFO] Total time: 19.595 s
[INFO] Finished at: 2018-09-04T09:06:59+08:00
[INFO] Final Memory: 56M/583M
[INFO]
------------------------------------------------------------------------
[ERROR] Failed to execute goal
org.apache.maven.plugins:maven-compiler-plugin:3.2:compile (default-compile)
on project carbondata-core: Compilation failure: Compilation failure:
[ERROR]
/Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java:[2230,12]
an enum switch case label must be the unqualified name of an enumeration
constant
[ERROR]
/Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[160,51]
cannot find symbol
[ERROR] symbol: variable MAP
[ERROR] location: class org.apache.carbondata.format.DataType
[ERROR]
/Users/aaron/workspace/carbondata/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java:[501,12]
an enum switch case label must be the unqualified name of an enumeration
constant
[ERROR] -> [Help 1]
[ERROR]
[ERROR] To see the full stack trace of the errors, re-run Maven with the -e
switch.
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please
read the following articles:
[ERROR] [Help 1]
http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException
[ERROR]
[ERROR] After correcting the problems, you can resume the build with the
command
[ERROR] mvn <goals> -rf :carbondata-core
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by aaron <94...@qq.com>.
Thanks, I will have a try!
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by Liang Chen <ch...@gmail.com>.
Hi kunal
Can you list all S3 issues PR, we may need to give 1.4.2 patch release.
Because aaron plan to use carbondata in production this month.
To arron : First please you try master, see if can solve your problems.
Regards
Liang
kunalkapoor wrote
> Hi aaron,
> Many issues like this have been identified in 1.4 version. Most of the
> issues have been fixed in the master code and will be released in 1.5
> version.
> Remaing fixes are in progress.
> Can you try the same scenario in 1.5(master branch).
>
> Thanks
> Kunal Kapoor
>
> On Mon, Sep 3, 2018, 5:57 AM aaron <
> 949835961@
>> wrote:
>
>> *update the aws-java-sdk and hadoop-aws to below version, then
>> authorization
>> works.
>> com.amazonaws:aws-java-sdk:1.10.75.1,org.apache.hadoop:hadoop-aws:2.7.3*
>>
>> *But we still can not load data, the exception is same.
>> carbon.sql("LOAD DATA INPATH
>> 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
>> test_s3_table")*
>>
>> 18/09/02 21:49:47 ERROR CarbonLoaderUtil: main Unable to unlock Table
>> lock
>> for tabledefault.test_s3_table during table status updation
>> 18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main
>> java.lang.ArrayIndexOutOfBoundsException
>> at java.lang.System.arraycopy(Native Method)
>> at
>> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
>> at
>> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
>> at
>>
>> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
>> at java.io.DataOutputStream.write(DataOutputStream.java:107)
>> at
>>
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
>> at
>>
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
>> at
>>
>> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
>> at
>> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
>> at
>>
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
>> at
>>
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
>> at
>>
>> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
>> at
>>
>> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
>> at
>>
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
>> at
>>
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
>> at
>>
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
>> at org.apache.spark.sql.Dataset.
> <init>
> (Dataset.scala:183)
>> at
>>
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
>> at
>>
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
>> at
>> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
>> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
>> at
>>
>> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :34)
>> at
>>
>> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :39)
>> at
>> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :41)
>> at
>> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :43)
>> at
>> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :45)
>> at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :47)
>> at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :49)
>> at $line25.$read$$iw$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :51)
>> at $line25.$read$$iw$$iw$$iw$$iw.
> <init>
> (
> <console>
> :53)
>> at $line25.$read$$iw$$iw$$iw.
> <init>
> (
> <console>
> :55)
>> at $line25.$read$$iw$$iw.
> <init>
> (
> <console>
> :57)
>> at $line25.$read$$iw.
> <init>
> (
> <console>
> :59)
>> at $line25.$read.
> <init>
> (
> <console>
> :61)
>> at $line25.$read$.
> <init>
> (
> <console>
> :65)
>> at $line25.$read$.
> <clinit>
> (
> <console>
> )
>> at $line25.$eval$.$print$lzycompute(
> <console>
> :7)
>> at $line25.$eval$.$print(
> <console>
> :6)
>> at $line25.$eval.$print(
> <console>
> )
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at
>>
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> at
>>
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:498)
>> at
>> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
>> at
>> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
>> at
>>
>> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
>> at
>>
>> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
>> at
>>
>> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
>> at
>>
>> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
>> at
>>
>> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
>> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
>> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
>> at
>> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
>> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
>> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
>> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
>> at
>>
>> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
>> at
>> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
>> at
>> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
>> at
>>
>> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
>> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
>> at org.apache.spark.repl.Main$.doMain(Main.scala:74)
>> at org.apache.spark.repl.Main$.main(Main.scala:54)
>> at org.apache.spark.repl.Main.main(Main.scala)
>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> at
>>
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> at
>>
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:498)
>> at
>>
>> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
>> at
>> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
>> at
>> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
>> at
>> org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
>> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
>> 18/09/02 21:49:47 AUDIT CarbonLoadDataCommand:
>> [aaron.local][aaron][Thread-1]Dataload failure for default.test_s3_table.
>> Please check the logs
>> 18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main Got exception
>> java.lang.ArrayIndexOutOfBoundsException when processing data. But this
>> command does not support undo yet, skipping the undo part.
>> java.lang.ArrayIndexOutOfBoundsException
>> at java.lang.System.arraycopy(Native Method)
>> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
>> at
>> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
>> at
>>
>> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
>> at java.io.DataOutputStream.write(DataOutputStream.java:107)
>> at
>>
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
>> at
>>
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
>> at
>>
>> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
>> at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
>> at
>>
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
>> at
>>
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
>> at
>>
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
>> at
>>
>> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
>> at
>>
>> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
>> at
>>
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
>> at
>>
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
>> at
>>
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
>> at org.apache.spark.sql.Dataset.
> <init>
> (Dataset.scala:183)
>> at
>>
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
>> at
>>
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
>> at
>> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
>> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
>> ... 52 elided
>>
>> aaron wrote
>> > Hi dear community, could anybody please kindly tell me what happened?
>> >
>> > *Env*:
>> >
>> > 1.spark 2.2.1 + carbon1.4.1
>> > 2.spark.jars.packages
>> > com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.2
>> > 3.spark.driver.extraClassPath
>> > file:///usr/local/Cellar/apache-spark/2.2.1/lib/*
>> > spark.executor.extraClassPath
>> > file:///usr/local/Cellar/apache-spark/2.2.1/lib/*
>> > lib folder include below jars
>> > -rw-r--r--@ 1 aaron staff 52M Aug 29 20:50
>> > apache-carbondata-1.4.1-bin-spark2.2.1-hadoop2.7.2.jar
>> > -rw-r--r-- 1 aaron staff 764K Aug 29 21:33 httpclient-4.5.4.jar
>> > -rw-r--r-- 1 aaron staff 314K Aug 29 21:40 httpcore-4.4.jar
>> >
>> >
>> > *Code*:
>> >
>> > import org.apache.spark.sql.SparkSession
>> > import org.apache.spark.sql.CarbonSession._
>> > import org.apache.spark.sql.catalyst.util._
>> > import org.apache.carbondata.core.util.CarbonProperties
>> > import org.apache.carbondata.core.constants.CarbonCommonConstants
>> >
>> CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE,
>> > "HDFSLOCK")
>> > val carbon =
>> >
>> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
>> >
>> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
>> > "xxx").config("spark.hadoop.fs.s3a.secret.key",
>> >
>> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
>> >
>> > carbon.sql("CREATE TABLE IF NOT EXISTS test_s3_table(id string, name
>> > string,
>> > city string, age Int) STORED BY 'carbondata' LOCATION
>> > 's3a://key:password@aaron-s3-poc/'")
>> > carbon.sql("LOAD DATA INPATH
>> > 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
>> > test_s3_table")
>> >
>> > *s3 files,*
>> >
>> > aws s3 ls s3://aaron-s3-poc/ --human --recursive
>> > 2018-08-29 22:13:32 0 Bytes LockFiles/tablestatus.lock
>> > 2018-08-29 21:41:36 616 Bytes Metadata/schema
>> >
>> >
>> > *Issue 1,* when I create table, carbondata raise Exception
>> > "com.amazonaws.AmazonClientException: Unable to load AWS credentials
>> from
>> > any provider in the chain" even if
>> > a. I set related properties in spark-default.conf like
>> > spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
>> > spark.hadoop.fs.s3a.awsAccessKeyId=xxx
>> > spark.hadoop.fs.s3a.awsSecretAccessKey=xxx
>> > spark.hadoop.fs.s3a.access.key=xxx
>> > spark.hadoop.fs.s3a.secret.key=xxx
>> > b.config in code
>> > val carbon =
>> >
>> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
>> >
>> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
>> > "xxx").config("spark.hadoop.fs.s3a.secret.key",
>> >
>> "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
>> > c. spark-submit conf
>> > Finally I succeed when I put credentials in LOCATION
>> > 's3a://key:password@aaron-s3-poc/'", But it's very strange. Who could
>> tell
>> > me why?
>> >
>> >
>> > *Issue 2,* Load data failed
>> >
>> > scala> carbon.sql("LOAD DATA INPATH
>> > 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
>> > test_s3_table")
>> > 18/08/29 22:13:35 ERROR CarbonLoaderUtil: main Unable to unlock Table
>> lock
>> > for tabledefault.test_s3_table during table status updation
>> > 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main
>> > java.lang.ArrayIndexOutOfBoundsException
>> > at java.lang.System.arraycopy(Native Method)
>> > at
>> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
>> > at
>> >
>> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
>> > at
>> >
>> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
>> > at java.io.DataOutputStream.write(DataOutputStream.java:107)
>> > at
>> >
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
>> > at
>> >
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
>> > at
>> >
>> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
>> > at
>> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
>> > at
>> >
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
>> > at
>> >
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
>> > at
>> >
>> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
>> > at
>> >
>> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
>> > at
>> >
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
>> > at
>> >
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
>> > at
>> >
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
>> > at org.apache.spark.sql.Dataset.
>> >
> <init>
>> > (Dataset.scala:183)
>> > at
>> >
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
>> > at
>> >
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
>> > at
>> >
>> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
>> > at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
>> > at
>> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :36)
>> > at
>> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :41)
>> > at
>> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :43)
>> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :45)
>> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :47)
>> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :49)
>> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :51)
>> > at $line21.$read$$iw$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :53)
>> > at $line21.$read$$iw$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :55)
>> > at $line21.$read$$iw$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :57)
>> > at $line21.$read$$iw$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :59)
>> > at $line21.$read$$iw.
>> >
> <init>
>> > (
>> >
> <console>
>> > :61)
>> > at $line21.$read.
>> >
> <init>
>> > (
>> >
> <console>
>> > :63)
>> > at $line21.$read$.
>> >
> <init>
>> > (
>> >
> <console>
>> > :67)
>> > at $line21.$read$.
>> >
> <clinit>
>> > (
>> >
> <console>
>> > )
>> > at $line21.$eval$.$print$lzycompute(
>> >
> <console>
>> > :7)
>> > at $line21.$eval$.$print(
>> >
> <console>
>> > :6)
>> > at $line21.$eval.$print(
>> >
> <console>
>> > )
>> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > at
>> >
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> > at
>> >
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> > at java.lang.reflect.Method.invoke(Method.java:498)
>> > at
>> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
>> > at
>> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
>> > at
>> >
>> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
>> > at
>> >
>> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
>> > at
>> >
>> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
>> > at
>> >
>> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
>> > at
>> >
>> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
>> > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
>> > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
>> > at
>> >
>> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
>> > at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
>> > at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
>> > at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
>> > at
>> >
>> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
>> > at
>> >
>> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
>> > at
>> >
>> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
>> > at
>> >
>> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
>> > at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
>> > at org.apache.spark.repl.Main$.doMain(Main.scala:74)
>> > at org.apache.spark.repl.Main$.main(Main.scala:54)
>> > at org.apache.spark.repl.Main.main(Main.scala)
>> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > at
>> >
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> > at
>> >
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> > at java.lang.reflect.Method.invoke(Method.java:498)
>> > at
>> >
>> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
>> > at
>> > org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
>> > at
>> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
>> > at
>> org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
>> > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
>> > 18/08/29 22:13:35 AUDIT CarbonLoadDataCommand:
>> > [aaron.local][aaron][Thread-1]Dataload failure for
>> default.test_s3_table.
>> > Please check the logs
>> > 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main Got exception
>> > java.lang.ArrayIndexOutOfBoundsException when processing data. But this
>> > command does not support undo yet, skipping the undo part.
>> > java.lang.ArrayIndexOutOfBoundsException
>> > at java.lang.System.arraycopy(Native Method)
>> > at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
>> > at
>> >
>> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
>> > at
>> >
>> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
>> > at java.io.DataOutputStream.write(DataOutputStream.java:107)
>> > at
>> >
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
>> > at
>> >
>> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
>> > at
>> >
>> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
>> > at
>> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
>> > at
>> >
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
>> > at
>> >
>> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
>> > at
>> >
>> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
>> > at
>> >
>> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
>> > at
>> >
>> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
>> > at
>> >
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
>> > at
>> >
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
>> > at
>> >
>> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
>> > at org.apache.spark.sql.Dataset.
>> >
> <init>
>> > (Dataset.scala:183)
>> > at
>> >
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
>> > at
>> >
>> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
>> > at
>> >
>> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
>> > at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
>> > ... 52 elided
>> >
>> >
>> > Thanks
>> > Aaron
>> >
>> >
>> >
>> > --
>> > Sent from:
>> >
>> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
>>
>>
>>
>>
>>
>> --
>> Sent from:
>> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
>>
--
Sent from: http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
Re: error occur when I load data to s3
Posted by Kunal Kapoor <ku...@gmail.com>.
Hi aaron,
Many issues like this have been identified in 1.4 version. Most of the
issues have been fixed in the master code and will be released in 1.5
version.
Remaing fixes are in progress.
Can you try the same scenario in 1.5(master branch).
Thanks
Kunal Kapoor
On Mon, Sep 3, 2018, 5:57 AM aaron <94...@qq.com> wrote:
> *update the aws-java-sdk and hadoop-aws to below version, then
> authorization
> works.
> com.amazonaws:aws-java-sdk:1.10.75.1,org.apache.hadoop:hadoop-aws:2.7.3*
>
> *But we still can not load data, the exception is same.
> carbon.sql("LOAD DATA INPATH
> 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
> test_s3_table")*
>
> 18/09/02 21:49:47 ERROR CarbonLoaderUtil: main Unable to unlock Table lock
> for tabledefault.test_s3_table during table status updation
> 18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at
> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
> at
> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
> at
>
> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:34)
> at
>
> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:39)
> at
> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:41)
> at
> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:43)
> at
> $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:45)
> at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:47)
> at $line25.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:49)
> at $line25.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:51)
> at $line25.$read$$iw$$iw$$iw$$iw.<init>(<console>:53)
> at $line25.$read$$iw$$iw$$iw.<init>(<console>:55)
> at $line25.$read$$iw$$iw.<init>(<console>:57)
> at $line25.$read$$iw.<init>(<console>:59)
> at $line25.$read.<init>(<console>:61)
> at $line25.$read$.<init>(<console>:65)
> at $line25.$read$.<clinit>(<console>)
> at $line25.$eval$.$print$lzycompute(<console>:7)
> at $line25.$eval$.$print(<console>:6)
> at $line25.$eval.$print(<console>)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
> at
> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
> at
>
> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
> at
>
> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
> at
>
> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
> at
> scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
> at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
> at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
> at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
> at
>
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> at
>
> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
> at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
> at org.apache.spark.repl.Main$.doMain(Main.scala:74)
> at org.apache.spark.repl.Main$.main(Main.scala:54)
> at org.apache.spark.repl.Main.main(Main.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
>
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
> at
> org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> at
> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> 18/09/02 21:49:47 AUDIT CarbonLoadDataCommand:
> [aaron.local][aaron][Thread-1]Dataload failure for default.test_s3_table.
> Please check the logs
> 18/09/02 21:49:47 ERROR CarbonLoadDataCommand: main Got exception
> java.lang.ArrayIndexOutOfBoundsException when processing data. But this
> command does not support undo yet, skipping the undo part.
> java.lang.ArrayIndexOutOfBoundsException
> at java.lang.System.arraycopy(Native Method)
> at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> at
> org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> at
>
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> at java.io.DataOutputStream.write(DataOutputStream.java:107)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> at
>
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> at
>
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
> at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> at
>
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> at
>
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> at
>
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
> at
>
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
>
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:183)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
> at
>
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
> at
> org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
> at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
> ... 52 elided
>
> aaron wrote
> > Hi dear community, could anybody please kindly tell me what happened?
> >
> > *Env*:
> >
> > 1.spark 2.2.1 + carbon1.4.1
> > 2.spark.jars.packages
> > com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.2
> > 3.spark.driver.extraClassPath
> > file:///usr/local/Cellar/apache-spark/2.2.1/lib/*
> > spark.executor.extraClassPath
> > file:///usr/local/Cellar/apache-spark/2.2.1/lib/*
> > lib folder include below jars
> > -rw-r--r--@ 1 aaron staff 52M Aug 29 20:50
> > apache-carbondata-1.4.1-bin-spark2.2.1-hadoop2.7.2.jar
> > -rw-r--r-- 1 aaron staff 764K Aug 29 21:33 httpclient-4.5.4.jar
> > -rw-r--r-- 1 aaron staff 314K Aug 29 21:40 httpcore-4.4.jar
> >
> >
> > *Code*:
> >
> > import org.apache.spark.sql.SparkSession
> > import org.apache.spark.sql.CarbonSession._
> > import org.apache.spark.sql.catalyst.util._
> > import org.apache.carbondata.core.util.CarbonProperties
> > import org.apache.carbondata.core.constants.CarbonCommonConstants
> >
> CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE,
> > "HDFSLOCK")
> > val carbon =
> >
> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
> >
> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
> > "xxx").config("spark.hadoop.fs.s3a.secret.key",
> > "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
> >
> > carbon.sql("CREATE TABLE IF NOT EXISTS test_s3_table(id string, name
> > string,
> > city string, age Int) STORED BY 'carbondata' LOCATION
> > 's3a://key:password@aaron-s3-poc/'")
> > carbon.sql("LOAD DATA INPATH
> > 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
> > test_s3_table")
> >
> > *s3 files,*
> >
> > aws s3 ls s3://aaron-s3-poc/ --human --recursive
> > 2018-08-29 22:13:32 0 Bytes LockFiles/tablestatus.lock
> > 2018-08-29 21:41:36 616 Bytes Metadata/schema
> >
> >
> > *Issue 1,* when I create table, carbondata raise Exception
> > "com.amazonaws.AmazonClientException: Unable to load AWS credentials from
> > any provider in the chain" even if
> > a. I set related properties in spark-default.conf like
> > spark.hadoop.fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem
> > spark.hadoop.fs.s3a.awsAccessKeyId=xxx
> > spark.hadoop.fs.s3a.awsSecretAccessKey=xxx
> > spark.hadoop.fs.s3a.access.key=xxx
> > spark.hadoop.fs.s3a.secret.key=xxx
> > b.config in code
> > val carbon =
> >
> SparkSession.builder().config(sc.getConf).config("spark.hadoop.fs.s3a.impl",
> >
> "org.apache.hadoop.fs.s3a.S3AFileSystem").config("spark.hadoop.fs.s3a.access.key",
> > "xxx").config("spark.hadoop.fs.s3a.secret.key",
> > "xxx").getOrCreateCarbonSession("hdfs://localhost:9000/usr/carbon-meta")
> > c. spark-submit conf
> > Finally I succeed when I put credentials in LOCATION
> > 's3a://key:password@aaron-s3-poc/'", But it's very strange. Who could
> tell
> > me why?
> >
> >
> > *Issue 2,* Load data failed
> >
> > scala> carbon.sql("LOAD DATA INPATH
> > 'hdfs://localhost:9000/usr/carbon-s3/sample.csv' INTO TABLE
> > test_s3_table")
> > 18/08/29 22:13:35 ERROR CarbonLoaderUtil: main Unable to unlock Table
> lock
> > for tabledefault.test_s3_table during table status updation
> > 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main
> > java.lang.ArrayIndexOutOfBoundsException
> > at java.lang.System.arraycopy(Native Method)
> > at
> java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> > at
> > org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> > at
> >
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> > at java.io.DataOutputStream.write(DataOutputStream.java:107)
> > at
> >
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> > at
> >
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> > at
> >
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
> > at
> org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> > at
> >
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> > at
> >
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> > at
> >
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
> > at
> >
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> > at
> >
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> > at
> >
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> > at
> >
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> > at org.apache.spark.sql.Dataset.
> > <init>
> > (Dataset.scala:183)
> > at
> >
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
> > at
> >
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
> > at
> > org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
> > at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
> > at
> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :36)
> > at
> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :41)
> > at
> > $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :43)
> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :45)
> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :47)
> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :49)
> > at $line21.$read$$iw$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :51)
> > at $line21.$read$$iw$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :53)
> > at $line21.$read$$iw$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :55)
> > at $line21.$read$$iw$$iw$$iw.
> > <init>
> > (
> > <console>
> > :57)
> > at $line21.$read$$iw$$iw.
> > <init>
> > (
> > <console>
> > :59)
> > at $line21.$read$$iw.
> > <init>
> > (
> > <console>
> > :61)
> > at $line21.$read.
> > <init>
> > (
> > <console>
> > :63)
> > at $line21.$read$.
> > <init>
> > (
> > <console>
> > :67)
> > at $line21.$read$.
> > <clinit>
> > (
> > <console>
> > )
> > at $line21.$eval$.$print$lzycompute(
> > <console>
> > :7)
> > at $line21.$eval$.$print(
> > <console>
> > :6)
> > at $line21.$eval.$print(
> > <console>
> > )
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > at java.lang.reflect.Method.invoke(Method.java:498)
> > at
> scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
> > at
> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
> > at
> >
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
> > at
> >
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
> > at
> >
> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
> > at
> >
> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
> > at
> >
> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
> > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
> > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
> > at
> > scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807)
> > at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681)
> > at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395)
> > at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415)
> > at
> >
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923)
> > at
> >
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> > at
> >
> scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909)
> > at
> >
> scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97)
> > at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909)
> > at org.apache.spark.repl.Main$.doMain(Main.scala:74)
> > at org.apache.spark.repl.Main$.main(Main.scala:54)
> > at org.apache.spark.repl.Main.main(Main.scala)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > at
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > at java.lang.reflect.Method.invoke(Method.java:498)
> > at
> >
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:775)
> > at
> > org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
> > at
> org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
> > at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
> > at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> > 18/08/29 22:13:35 AUDIT CarbonLoadDataCommand:
> > [aaron.local][aaron][Thread-1]Dataload failure for default.test_s3_table.
> > Please check the logs
> > 18/08/29 22:13:35 ERROR CarbonLoadDataCommand: main Got exception
> > java.lang.ArrayIndexOutOfBoundsException when processing data. But this
> > command does not support undo yet, skipping the undo part.
> > java.lang.ArrayIndexOutOfBoundsException
> > at java.lang.System.arraycopy(Native Method)
> > at java.io.BufferedOutputStream.write(BufferedOutputStream.java:128)
> > at
> > org.apache.hadoop.fs.s3a.S3AOutputStream.write(S3AOutputStream.java:164)
> > at
> >
> org.apache.hadoop.fs.FSDataOutputStream$PositionCache.write(FSDataOutputStream.java:58)
> > at java.io.DataOutputStream.write(DataOutputStream.java:107)
> > at
> >
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStream(S3CarbonFile.java:111)
> > at
> >
> org.apache.carbondata.core.datastore.filesystem.S3CarbonFile.getDataOutputStreamUsingAppend(S3CarbonFile.java:93)
> > at
> >
> org.apache.carbondata.core.datastore.impl.FileFactory.getDataOutputStreamUsingAppend(FileFactory.java:276)
> > at org.apache.carbondata.core.locks.S3FileLock.lock(S3FileLock.java:96)
> > at
> >
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:41)
> > at
> >
> org.apache.carbondata.core.locks.AbstractCarbonLock.lockWithRetries(AbstractCarbonLock.java:59)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:247)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.recordNewLoadMetadata(CarbonLoaderUtil.java:204)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:437)
> > at
> >
> org.apache.carbondata.processing.util.CarbonLoaderUtil.readAndUpdateLoadProgressInTableMeta(CarbonLoaderUtil.java:446)
> > at
> >
> org.apache.spark.sql.execution.command.management.CarbonLoadDataCommand.processData(CarbonLoadDataCommand.scala:263)
> > at
> >
> org.apache.spark.sql.execution.command.AtomicRunnableCommand.run(package.scala:92)
> > at
> >
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> > at
> >
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> > at
> >
> org.apache.spark.sql.execution.command.ExecutedCommandExec.executeCollect(commands.scala:67)
> > at org.apache.spark.sql.Dataset.
> > <init>
> > (Dataset.scala:183)
> > at
> >
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:107)
> > at
> >
> org.apache.spark.sql.CarbonSession$$anonfun$sql$1.apply(CarbonSession.scala:96)
> > at
> > org.apache.spark.sql.CarbonSession.withProfiler(CarbonSession.scala:154)
> > at org.apache.spark.sql.CarbonSession.sql(CarbonSession.scala:94)
> > ... 52 elided
> >
> >
> > Thanks
> > Aaron
> >
> >
> >
> > --
> > Sent from:
> > http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
>
>
>
>
>
> --
> Sent from:
> http://apache-carbondata-dev-mailing-list-archive.1130556.n5.nabble.com/
>