You are viewing a plain text version of this content. The canonical link for it is here.

Posted to user-zh@flink.apache.org by Zhou Zach <wa...@163.com> on 2020/07/16 04:07:08 UTC

flink sql 1.11 create hive table error

Hi all,
flink sql 1.11 create table 是不是 不支持 IF NOT EXISTS


Query：
    val hiveConfDir = "/etc/hive/conf" 
    val hiveVersion = "2.1.1"

    val odsCatalog = "odsCatalog"
    val odsHiveCatalog = new HiveCatalog(odsCatalog, "ods", hiveConfDir, hiveVersion)
    streamTableEnv.registerCatalog(odsCatalog, odsHiveCatalog)

    streamTableEnv.getConfig.setSqlDialect(SqlDialect.HIVE)
    streamTableEnv.executeSql(
      """
        |
        |CREATE TABLE IF NOT EXISTS odsCatalog.ods.hive_table (
        |  user_id STRING,
        |  age INT
        |) PARTITIONED BY (dt STRING, hr STRING) STORED AS parquet TBLPROPERTIES (
        |  'partition.time-extractor.timestamp-pattern'='$dt $hr:00:00',
        |  'sink.partition-commit.trigger'='partition-time',
        |  'sink.partition-commit.delay'='0s',
        |  'sink.partition-commit.policy.kind'='metastore'
        |)
        |
        |""".stripMargin)












java.util.concurrent.CompletionException: org.apache.flink.client.deployment.application.ApplicationExecutionException: Could not execute application.
	at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_161]
	at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_161]
	at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:943) ~[?:1.8.0_161]
	at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926) ~[?:1.8.0_161]
	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_161]
	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_161]
	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:245) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAsync$1(ApplicationDispatcherBootstrap.java:199) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_161]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_161]
	at org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:154) [data-flow-1.0.jar:?]
	at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40) [data-flow-1.0.jar:?]
	at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:44) [data-flow-1.0.jar:?]
	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [data-flow-1.0.jar:?]
	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [data-flow-1.0.jar:?]
	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [data-flow-1.0.jar:?]
	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [data-flow-1.0.jar:?]
Caused by: org.apache.flink.client.deployment.application.ApplicationExecutionException: Could not execute application.
	... 11 more
Caused by: org.apache.flink.client.program.ProgramInvocationException: The main method caused an error: SQL parse failed. Encountered "NOT" at line 3, column 17.
Was expecting one of:
    <EOF> 
    "ROW" ...
    "COMMENT" ...
    "LOCATION" ...
    "PARTITIONED" ...
    "STORED" ...
    "TBLPROPERTIES" ...
    "(" ...
    "." ...
    
	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:302) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	... 10 more
Caused by: org.apache.flink.table.api.SqlParserException: SQL parse failed. Encountered "NOT" at line 3, column 17.
Was expecting one of:
    <EOF> 
    "ROW" ...
    "COMMENT" ...
    "LOCATION" ...
    "PARTITIONED" ...
    "STORED" ...
    "TBLPROPERTIES" ...
    "(" ...
    "." ...
    
	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:56) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[qile-data-flow-1.0.jar:?]
	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[qile-data-flow-1.0.jar:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	... 10 more
Caused by: org.apache.calcite.sql.parser.SqlParseException: Encountered "NOT" at line 3, column 17.
Was expecting one of:
    <EOF> 
    "ROW" ...
    "COMMENT" ...
    "LOCATION" ...
    "PARTITIONED" ...
    "STORED" ...
    "TBLPROPERTIES" ...
    "(" ...
    "." ...
    
	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.convertException(FlinkHiveSqlParserImpl.java:435) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.normalizeException(FlinkHiveSqlParserImpl.java:220) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.calcite.sql.parser.SqlParser.handleException(SqlParser.java:148) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.calcite.sql.parser.SqlParser.parseQuery(SqlParser.java:163) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.calcite.sql.parser.SqlParser.parseStmt(SqlParser.java:188) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:54) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[data-flow-1.0.jar:?]
	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[data-flow-1.0.jar:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	... 10 more
Caused by: org.apache.flink.sql.parser.hive.impl.ParseException: Encountered "NOT" at line 3, column 17.
Was expecting one of:
    <EOF> 
    "ROW" ...
    "COMMENT" ...
    "LOCATION" ...
    "PARTITIONED" ...
    "STORED" ...
    "TBLPROPERTIES" ...
    "(" ...
    "." ...
    
	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.generateParseException(FlinkHiveSqlParserImpl.java:37347) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.jj_consume_token(FlinkHiveSqlParserImpl.java:37158) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.SqlStmtEof(FlinkHiveSqlParserImpl.java:3962) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.parseSqlStmtEof(FlinkHiveSqlParserImpl.java:267) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.calcite.sql.parser.SqlParser.parseQuery(SqlParser.java:161) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.calcite.sql.parser.SqlParser.parseStmt(SqlParser.java:188) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:54) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[data-flow-1.0.jar:?]
	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[data-flow-1.0.jar:?]
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]

Re:Re: flink sql 1.11 create hive table error

Posted by Zhou Zach <wa...@163.com>.

hi Leonard,
感谢答疑！

















At 2020-07-16 12:16:40, "Leonard Xu" <xb...@gmail.com> wrote:
>Hello, Zach
>
>是的, 1.12 会支持，PR[1]已经开了，在review中。
>
>祝好，
>Leonard Xu
>[1] https://issues.apache.org/jira/browse/FLINK-18588 <https://issues.apache.org/jira/browse/FLINK-18588>
>> 在 2020年7月16日，12:07，Zhou Zach <wa...@163.com> 写道：
>> 
>> Hi all,
>> flink sql 1.11 create table 是不是 不支持 IF NOT EXISTS
>> 
>> 
>> Query：
>>    val hiveConfDir = "/etc/hive/conf" 
>>    val hiveVersion = "2.1.1"
>> 
>>    val odsCatalog = "odsCatalog"
>>    val odsHiveCatalog = new HiveCatalog(odsCatalog, "ods", hiveConfDir, hiveVersion)
>>    streamTableEnv.registerCatalog(odsCatalog, odsHiveCatalog)
>> 
>>    streamTableEnv.getConfig.setSqlDialect(SqlDialect.HIVE)
>>    streamTableEnv.executeSql(
>>      """
>>        |
>>        |CREATE TABLE IF NOT EXISTS odsCatalog.ods.hive_table (
>>        |  user_id STRING,
>>        |  age INT
>>        |) PARTITIONED BY (dt STRING, hr STRING) STORED AS parquet TBLPROPERTIES (
>>        |  'partition.time-extractor.timestamp-pattern'='$dt $hr:00:00',
>>        |  'sink.partition-commit.trigger'='partition-time',
>>        |  'sink.partition-commit.delay'='0s',
>>        |  'sink.partition-commit.policy.kind'='metastore'
>>        |)
>>        |
>>        |""".stripMargin)
>> 
>> 
>> 
>> 
>> 
>> 
>> 
>> 
>> 
>> 
>> 
>> 
>> java.util.concurrent.CompletionException: org.apache.flink.client.deployment.application.ApplicationExecutionException: Could not execute application.
>> 	at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_161]
>> 	at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_161]
>> 	at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:943) ~[?:1.8.0_161]
>> 	at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926) ~[?:1.8.0_161]
>> 	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_161]
>> 	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_161]
>> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:245) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAsync$1(ApplicationDispatcherBootstrap.java:199) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_161]
>> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_161]
>> 	at org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:154) [data-flow-1.0.jar:?]
>> 	at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40) [data-flow-1.0.jar:?]
>> 	at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:44) [data-flow-1.0.jar:?]
>> 	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [data-flow-1.0.jar:?]
>> 	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [data-flow-1.0.jar:?]
>> 	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [data-flow-1.0.jar:?]
>> 	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [data-flow-1.0.jar:?]
>> Caused by: org.apache.flink.client.deployment.application.ApplicationExecutionException: Could not execute application.
>> 	... 11 more
>> Caused by: org.apache.flink.client.program.ProgramInvocationException: The main method caused an error: SQL parse failed. Encountered "NOT" at line 3, column 17.
>> Was expecting one of:
>>    <EOF> 
>>    "ROW" ...
>>    "COMMENT" ...
>>    "LOCATION" ...
>>    "PARTITIONED" ...
>>    "STORED" ...
>>    "TBLPROPERTIES" ...
>>    "(" ...
>>    "." ...
>> 
>> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:302) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	... 10 more
>> Caused by: org.apache.flink.table.api.SqlParserException: SQL parse failed. Encountered "NOT" at line 3, column 17.
>> Was expecting one of:
>>    <EOF> 
>>    "ROW" ...
>>    "COMMENT" ...
>>    "LOCATION" ...
>>    "PARTITIONED" ...
>>    "STORED" ...
>>    "TBLPROPERTIES" ...
>>    "(" ...
>>    "." ...
>> 
>> 	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:56) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[qile-data-flow-1.0.jar:?]
>> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[qile-data-flow-1.0.jar:?]
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
>> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
>> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	... 10 more
>> Caused by: org.apache.calcite.sql.parser.SqlParseException: Encountered "NOT" at line 3, column 17.
>> Was expecting one of:
>>    <EOF> 
>>    "ROW" ...
>>    "COMMENT" ...
>>    "LOCATION" ...
>>    "PARTITIONED" ...
>>    "STORED" ...
>>    "TBLPROPERTIES" ...
>>    "(" ...
>>    "." ...
>> 
>> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.convertException(FlinkHiveSqlParserImpl.java:435) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.normalizeException(FlinkHiveSqlParserImpl.java:220) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.calcite.sql.parser.SqlParser.handleException(SqlParser.java:148) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.calcite.sql.parser.SqlParser.parseQuery(SqlParser.java:163) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.calcite.sql.parser.SqlParser.parseStmt(SqlParser.java:188) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:54) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[data-flow-1.0.jar:?]
>> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[data-flow-1.0.jar:?]
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
>> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
>> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	... 10 more
>> Caused by: org.apache.flink.sql.parser.hive.impl.ParseException: Encountered "NOT" at line 3, column 17.
>> Was expecting one of:
>>    <EOF> 
>>    "ROW" ...
>>    "COMMENT" ...
>>    "LOCATION" ...
>>    "PARTITIONED" ...
>>    "STORED" ...
>>    "TBLPROPERTIES" ...
>>    "(" ...
>>    "." ...
>> 
>> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.generateParseException(FlinkHiveSqlParserImpl.java:37347) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.jj_consume_token(FlinkHiveSqlParserImpl.java:37158) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.SqlStmtEof(FlinkHiveSqlParserImpl.java:3962) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.parseSqlStmtEof(FlinkHiveSqlParserImpl.java:267) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.calcite.sql.parser.SqlParser.parseQuery(SqlParser.java:161) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.calcite.sql.parser.SqlParser.parseStmt(SqlParser.java:188) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:54) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
>> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[data-flow-1.0.jar:?]
>> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[data-flow-1.0.jar:?]
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
>> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
>> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
>> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
>

Re: flink sql 1.11 create hive table error

Posted by Leonard Xu <xb...@gmail.com>.

Hello, Zach

是的, 1.12 会支持，PR[1]已经开了，在review中。

祝好，
Leonard Xu
[1] https://issues.apache.org/jira/browse/FLINK-18588 <https://issues.apache.org/jira/browse/FLINK-18588>
> 在 2020年7月16日，12:07，Zhou Zach <wa...@163.com> 写道：
> 
> Hi all,
> flink sql 1.11 create table 是不是 不支持 IF NOT EXISTS
> 
> 
> Query：
>    val hiveConfDir = "/etc/hive/conf" 
>    val hiveVersion = "2.1.1"
> 
>    val odsCatalog = "odsCatalog"
>    val odsHiveCatalog = new HiveCatalog(odsCatalog, "ods", hiveConfDir, hiveVersion)
>    streamTableEnv.registerCatalog(odsCatalog, odsHiveCatalog)
> 
>    streamTableEnv.getConfig.setSqlDialect(SqlDialect.HIVE)
>    streamTableEnv.executeSql(
>      """
>        |
>        |CREATE TABLE IF NOT EXISTS odsCatalog.ods.hive_table (
>        |  user_id STRING,
>        |  age INT
>        |) PARTITIONED BY (dt STRING, hr STRING) STORED AS parquet TBLPROPERTIES (
>        |  'partition.time-extractor.timestamp-pattern'='$dt $hr:00:00',
>        |  'sink.partition-commit.trigger'='partition-time',
>        |  'sink.partition-commit.delay'='0s',
>        |  'sink.partition-commit.policy.kind'='metastore'
>        |)
>        |
>        |""".stripMargin)
> 
> 
> 
> 
> 
> 
> 
> 
> 
> 
> 
> 
> java.util.concurrent.CompletionException: org.apache.flink.client.deployment.application.ApplicationExecutionException: Could not execute application.
> 	at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292) ~[?:1.8.0_161]
> 	at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308) ~[?:1.8.0_161]
> 	at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:943) ~[?:1.8.0_161]
> 	at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:926) ~[?:1.8.0_161]
> 	at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:474) ~[?:1.8.0_161]
> 	at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1977) ~[?:1.8.0_161]
> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:245) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.lambda$runApplicationAsync$1(ApplicationDispatcherBootstrap.java:199) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_161]
> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_161]
> 	at org.apache.flink.runtime.concurrent.akka.ActorSystemScheduledExecutorAdapter$ScheduledFutureTask.run(ActorSystemScheduledExecutorAdapter.java:154) [data-flow-1.0.jar:?]
> 	at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:40) [data-flow-1.0.jar:?]
> 	at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:44) [data-flow-1.0.jar:?]
> 	at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [data-flow-1.0.jar:?]
> 	at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [data-flow-1.0.jar:?]
> 	at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [data-flow-1.0.jar:?]
> 	at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [data-flow-1.0.jar:?]
> Caused by: org.apache.flink.client.deployment.application.ApplicationExecutionException: Could not execute application.
> 	... 11 more
> Caused by: org.apache.flink.client.program.ProgramInvocationException: The main method caused an error: SQL parse failed. Encountered "NOT" at line 3, column 17.
> Was expecting one of:
>    <EOF> 
>    "ROW" ...
>    "COMMENT" ...
>    "LOCATION" ...
>    "PARTITIONED" ...
>    "STORED" ...
>    "TBLPROPERTIES" ...
>    "(" ...
>    "." ...
> 
> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:302) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	... 10 more
> Caused by: org.apache.flink.table.api.SqlParserException: SQL parse failed. Encountered "NOT" at line 3, column 17.
> Was expecting one of:
>    <EOF> 
>    "ROW" ...
>    "COMMENT" ...
>    "LOCATION" ...
>    "PARTITIONED" ...
>    "STORED" ...
>    "TBLPROPERTIES" ...
>    "(" ...
>    "." ...
> 
> 	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:56) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[qile-data-flow-1.0.jar:?]
> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[qile-data-flow-1.0.jar:?]
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	... 10 more
> Caused by: org.apache.calcite.sql.parser.SqlParseException: Encountered "NOT" at line 3, column 17.
> Was expecting one of:
>    <EOF> 
>    "ROW" ...
>    "COMMENT" ...
>    "LOCATION" ...
>    "PARTITIONED" ...
>    "STORED" ...
>    "TBLPROPERTIES" ...
>    "(" ...
>    "." ...
> 
> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.convertException(FlinkHiveSqlParserImpl.java:435) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.normalizeException(FlinkHiveSqlParserImpl.java:220) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.calcite.sql.parser.SqlParser.handleException(SqlParser.java:148) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.calcite.sql.parser.SqlParser.parseQuery(SqlParser.java:163) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.calcite.sql.parser.SqlParser.parseStmt(SqlParser.java:188) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:54) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[data-flow-1.0.jar:?]
> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[data-flow-1.0.jar:?]
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	... 10 more
> Caused by: org.apache.flink.sql.parser.hive.impl.ParseException: Encountered "NOT" at line 3, column 17.
> Was expecting one of:
>    <EOF> 
>    "ROW" ...
>    "COMMENT" ...
>    "LOCATION" ...
>    "PARTITIONED" ...
>    "STORED" ...
>    "TBLPROPERTIES" ...
>    "(" ...
>    "." ...
> 
> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.generateParseException(FlinkHiveSqlParserImpl.java:37347) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.jj_consume_token(FlinkHiveSqlParserImpl.java:37158) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.SqlStmtEof(FlinkHiveSqlParserImpl.java:3962) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.sql.parser.hive.impl.FlinkHiveSqlParserImpl.parseSqlStmtEof(FlinkHiveSqlParserImpl.java:267) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.calcite.sql.parser.SqlParser.parseQuery(SqlParser.java:161) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.calcite.sql.parser.SqlParser.parseStmt(SqlParser.java:188) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.planner.calcite.CalciteParser.parse(CalciteParser.java:54) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.planner.delegation.ParserImpl.parse(ParserImpl.java:76) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.table.api.internal.TableEnvironmentImpl.executeSql(TableEnvironmentImpl.java:678) ~[flink-table-blink_2.11-1.11.0.jar:1.11.0]
> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase$.main(FromKafkaSinkHiveAndHbase.scala:31) ~[data-flow-1.0.jar:?]
> 	at cn.ibobei.qile.dataflow.sql.FromKafkaSinkHiveAndHbase.main(FromKafkaSinkHiveAndHbase.scala) ~[data-flow-1.0.jar:?]
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_161]
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_161]
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_161]
> 	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_161]
> 	at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:149) ~[flink-clients_2.11-1.11.0.jar:1.11.0]
> 	at org.apache.flink.client.deployment.application.ApplicationDispatcherBootstrap.runApplicationEntryPoint(ApplicationDispatcherBootstrap.java:230) ~[flink-clients_2.11-1.11.0.jar:1.11.0]