You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@phoenix.apache.org by "Samarth Jain (JIRA)" <ji...@apache.org> on 2017/09/06 20:19:00 UTC
[jira] [Updated] (PHOENIX-4171) Creating immutable index is timing
out intermittently
[ https://issues.apache.org/jira/browse/PHOENIX-4171?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Samarth Jain updated PHOENIX-4171:
----------------------------------
Description:
In PHOENIX-4151, I converted all the tests extending BaseQueryIT to not use current_scn anymore when creating tables and indices. This was done with the assumption that somehow current_scn is causing index creation to timeout. However, even after that change, I am seeing that the tests are still flapping. And they are failing because creating immutable indexes is timing out.
Sample run: https://builds.apache.org/job/PreCommit-PHOENIX-Build/1379/
Stacktrace:
{code}
2017-09-06 02:44:13,297 ERROR [main] org.apache.phoenix.end2end.BaseQueryIT(141): Exception while creating index: CREATE INDEX T000205 ON T000204 (a_integer DESC) INCLUDE ( A_STRING, B_STRING, A_DATE) KEEP_DELETED_CELLS=false
java.sql.SQLTimeoutException: Operation timed out.
at org.apache.phoenix.exception.SQLExceptionCode$15.newException(SQLExceptionCode.java:399)
at org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:932)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:846)
at org.apache.phoenix.iterate.ConcatResultIterator.getIterators(ConcatResultIterator.java:50)
at org.apache.phoenix.iterate.ConcatResultIterator.currentIterator(ConcatResultIterator.java:97)
at org.apache.phoenix.iterate.ConcatResultIterator.next(ConcatResultIterator.java:117)
at org.apache.phoenix.iterate.BaseGroupedAggregatingResultIterator.next(BaseGroupedAggregatingResultIterator.java:64)
at org.apache.phoenix.iterate.UngroupedAggregatingResultIterator.next(UngroupedAggregatingResultIterator.java:39)
at org.apache.phoenix.compile.UpsertCompiler$1.execute(UpsertCompiler.java:734)
at org.apache.phoenix.compile.DelegateMutationPlan.execute(DelegateMutationPlan.java:31)
at org.apache.phoenix.compile.PostIndexDDLCompiler$1.execute(PostIndexDDLCompiler.java:117)
at org.apache.phoenix.query.ConnectionQueryServicesImpl.updateData(ConnectionQueryServicesImpl.java:3359)
at org.apache.phoenix.schema.MetaDataClient.buildIndex(MetaDataClient.java:1282)
at org.apache.phoenix.schema.MetaDataClient.buildIndexAtTimeStamp(MetaDataClient.java:1222)
at org.apache.phoenix.schema.MetaDataClient.createIndex(MetaDataClient.java:1588)
at org.apache.phoenix.compile.CreateIndexCompiler$1.execute(CreateIndexCompiler.java:85)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:393)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:376)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:374)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:363)
at org.apache.phoenix.jdbc.PhoenixStatement.execute(PhoenixStatement.java:1707)
at org.apache.phoenix.end2end.BaseQueryIT.<init>(BaseQueryIT.java:139)
at org.apache.phoenix.end2end.NotQueryIT.<init>(NotQueryIT.java:56)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
{code}
{code}
2017-09-06 02:57:28,819 ERROR [main] org.apache.phoenix.end2end.BaseQueryIT(141): Exception while creating index: CREATE INDEX T000350 ON T000349 (a_integer, a_string) INCLUDE ( B_STRING, A_DATE) KEEP_DELETED_CELLS=false
java.sql.SQLTimeoutException: Operation timed out.
at org.apache.phoenix.exception.SQLExceptionCode$15.newException(SQLExceptionCode.java:399)
at org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:932)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:846)
at org.apache.phoenix.iterate.ConcatResultIterator.getIterators(ConcatResultIterator.java:50)
at org.apache.phoenix.iterate.ConcatResultIterator.currentIterator(ConcatResultIterator.java:97)
at org.apache.phoenix.iterate.ConcatResultIterator.next(ConcatResultIterator.java:117)
at org.apache.phoenix.iterate.BaseGroupedAggregatingResultIterator.next(BaseGroupedAggregatingResultIterator.java:64)
at org.apache.phoenix.iterate.UngroupedAggregatingResultIterator.next(UngroupedAggregatingResultIterator.java:39)
at org.apache.phoenix.compile.UpsertCompiler$1.execute(UpsertCompiler.java:734)
at org.apache.phoenix.compile.DelegateMutationPlan.execute(DelegateMutationPlan.java:31)
at org.apache.phoenix.compile.PostIndexDDLCompiler$1.execute(PostIndexDDLCompiler.java:117)
at org.apache.phoenix.query.ConnectionQueryServicesImpl.updateData(ConnectionQueryServicesImpl.java:3359)
at org.apache.phoenix.schema.MetaDataClient.buildIndex(MetaDataClient.java:1282)
at org.apache.phoenix.schema.MetaDataClient.buildIndexAtTimeStamp(MetaDataClient.java:1222)
at org.apache.phoenix.schema.MetaDataClient.createIndex(MetaDataClient.java:1588)
at org.apache.phoenix.compile.CreateIndexCompiler$1.execute(CreateIndexCompiler.java:85)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:393)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:376)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:374)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:363)
at org.apache.phoenix.jdbc.PhoenixStatement.execute(PhoenixStatement.java:1707)
at org.apache.phoenix.end2end.BaseQueryIT.<init>(BaseQueryIT.java:139)
at org.apache.phoenix.end2end.QueryIT.<init>(QueryIT.java:66)
at sun.reflect.GeneratedConstructorAccessor118.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
{code}
was:
In PHOENIX-4151, I converted all the tests extending BaseQueryIT to not use current_scn anymore when creating tables and indices. This was done with the assumption that somehow current_scn is causing index creation to timeout. However, even after that change, I am seeing that the tests are still flapping. And they are failing because creating immutable indexes is timing out.
Sample run: https://builds.apache.org/job/PreCommit-PHOENIX-Build/1379/
Stacktrace:
{code}
2017-09-06 02:44:13,297 ERROR [main] org.apache.phoenix.end2end.BaseQueryIT(141): Exception while creating index: CREATE INDEX T000205 ON T000204 (a_integer DESC) INCLUDE ( A_STRING, B_STRING, A_DATE) KEEP_DELETED_CELLS=false
java.sql.SQLTimeoutException: Operation timed out.
at org.apache.phoenix.exception.SQLExceptionCode$15.newException(SQLExceptionCode.java:399)
at org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:932)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:846)
at org.apache.phoenix.iterate.ConcatResultIterator.getIterators(ConcatResultIterator.java:50)
at org.apache.phoenix.iterate.ConcatResultIterator.currentIterator(ConcatResultIterator.java:97)
at org.apache.phoenix.iterate.ConcatResultIterator.next(ConcatResultIterator.java:117)
at org.apache.phoenix.iterate.BaseGroupedAggregatingResultIterator.next(BaseGroupedAggregatingResultIterator.java:64)
at org.apache.phoenix.iterate.UngroupedAggregatingResultIterator.next(UngroupedAggregatingResultIterator.java:39)
at org.apache.phoenix.compile.UpsertCompiler$1.execute(UpsertCompiler.java:734)
at org.apache.phoenix.compile.DelegateMutationPlan.execute(DelegateMutationPlan.java:31)
at org.apache.phoenix.compile.PostIndexDDLCompiler$1.execute(PostIndexDDLCompiler.java:117)
at org.apache.phoenix.query.ConnectionQueryServicesImpl.updateData(ConnectionQueryServicesImpl.java:3359)
at org.apache.phoenix.schema.MetaDataClient.buildIndex(MetaDataClient.java:1282)
at org.apache.phoenix.schema.MetaDataClient.buildIndexAtTimeStamp(MetaDataClient.java:1222)
at org.apache.phoenix.schema.MetaDataClient.createIndex(MetaDataClient.java:1588)
at org.apache.phoenix.compile.CreateIndexCompiler$1.execute(CreateIndexCompiler.java:85)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:393)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:376)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:374)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:363)
at org.apache.phoenix.jdbc.PhoenixStatement.execute(PhoenixStatement.java:1707)
at org.apache.phoenix.end2end.BaseQueryIT.<init>(BaseQueryIT.java:139)
at org.apache.phoenix.end2end.NotQueryIT.<init>(NotQueryIT.java:56)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
{code}
{code}
2017-09-06 02:57:28,819 ERROR [main] org.apache.phoenix.end2end.BaseQueryIT(141): Exception while creating index: CREATE INDEX T000350 ON T000349 (a_integer, a_string) INCLUDE ( B_STRING, A_DATE) KEEP_DELETED_CELLS=false
java.sql.SQLTimeoutException: Operation timed out.
at org.apache.phoenix.exception.SQLExceptionCode$15.newException(SQLExceptionCode.java:399)
at org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:932)
at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:846)
at org.apache.phoenix.iterate.ConcatResultIterator.getIterators(ConcatResultIterator.java:50)
at org.apache.phoenix.iterate.ConcatResultIterator.currentIterator(ConcatResultIterator.java:97)
at org.apache.phoenix.iterate.ConcatResultIterator.next(ConcatResultIterator.java:117)
at org.apache.phoenix.iterate.BaseGroupedAggregatingResultIterator.next(BaseGroupedAggregatingResultIterator.java:64)
at org.apache.phoenix.iterate.UngroupedAggregatingResultIterator.next(UngroupedAggregatingResultIterator.java:39)
at org.apache.phoenix.compile.UpsertCompiler$1.execute(UpsertCompiler.java:734)
at org.apache.phoenix.compile.DelegateMutationPlan.execute(DelegateMutationPlan.java:31)
at org.apache.phoenix.compile.PostIndexDDLCompiler$1.execute(PostIndexDDLCompiler.java:117)
at org.apache.phoenix.query.ConnectionQueryServicesImpl.updateData(ConnectionQueryServicesImpl.java:3359)
at org.apache.phoenix.schema.MetaDataClient.buildIndex(MetaDataClient.java:1282)
at org.apache.phoenix.schema.MetaDataClient.buildIndexAtTimeStamp(MetaDataClient.java:1222)
at org.apache.phoenix.schema.MetaDataClient.createIndex(MetaDataClient.java:1588)
at org.apache.phoenix.compile.CreateIndexCompiler$1.execute(CreateIndexCompiler.java:85)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:393)
at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:376)
at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:374)
at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:363)
at org.apache.phoenix.jdbc.PhoenixStatement.execute(PhoenixStatement.java:1707)
at org.apache.phoenix.end2end.BaseQueryIT.<init>(BaseQueryIT.java:139)
at org.apache.phoenix.end2end.QueryIT.<init>(QueryIT.java:66)
at sun.reflect.GeneratedConstructorAccessor118.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
{code}
I have mostly seen this happening for KEEP_DELETED_CELLS = false attribute. But I am not too sure if it is related to index creation failing. Still worth pointing out.
> Creating immutable index is timing out intermittently
> -----------------------------------------------------
>
> Key: PHOENIX-4171
> URL: https://issues.apache.org/jira/browse/PHOENIX-4171
> Project: Phoenix
> Issue Type: Bug
> Reporter: Samarth Jain
>
> In PHOENIX-4151, I converted all the tests extending BaseQueryIT to not use current_scn anymore when creating tables and indices. This was done with the assumption that somehow current_scn is causing index creation to timeout. However, even after that change, I am seeing that the tests are still flapping. And they are failing because creating immutable indexes is timing out.
> Sample run: https://builds.apache.org/job/PreCommit-PHOENIX-Build/1379/
> Stacktrace:
> {code}
> 2017-09-06 02:44:13,297 ERROR [main] org.apache.phoenix.end2end.BaseQueryIT(141): Exception while creating index: CREATE INDEX T000205 ON T000204 (a_integer DESC) INCLUDE ( A_STRING, B_STRING, A_DATE) KEEP_DELETED_CELLS=false
> java.sql.SQLTimeoutException: Operation timed out.
> at org.apache.phoenix.exception.SQLExceptionCode$15.newException(SQLExceptionCode.java:399)
> at org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
> at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:932)
> at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:846)
> at org.apache.phoenix.iterate.ConcatResultIterator.getIterators(ConcatResultIterator.java:50)
> at org.apache.phoenix.iterate.ConcatResultIterator.currentIterator(ConcatResultIterator.java:97)
> at org.apache.phoenix.iterate.ConcatResultIterator.next(ConcatResultIterator.java:117)
> at org.apache.phoenix.iterate.BaseGroupedAggregatingResultIterator.next(BaseGroupedAggregatingResultIterator.java:64)
> at org.apache.phoenix.iterate.UngroupedAggregatingResultIterator.next(UngroupedAggregatingResultIterator.java:39)
> at org.apache.phoenix.compile.UpsertCompiler$1.execute(UpsertCompiler.java:734)
> at org.apache.phoenix.compile.DelegateMutationPlan.execute(DelegateMutationPlan.java:31)
> at org.apache.phoenix.compile.PostIndexDDLCompiler$1.execute(PostIndexDDLCompiler.java:117)
> at org.apache.phoenix.query.ConnectionQueryServicesImpl.updateData(ConnectionQueryServicesImpl.java:3359)
> at org.apache.phoenix.schema.MetaDataClient.buildIndex(MetaDataClient.java:1282)
> at org.apache.phoenix.schema.MetaDataClient.buildIndexAtTimeStamp(MetaDataClient.java:1222)
> at org.apache.phoenix.schema.MetaDataClient.createIndex(MetaDataClient.java:1588)
> at org.apache.phoenix.compile.CreateIndexCompiler$1.execute(CreateIndexCompiler.java:85)
> at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:393)
> at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:376)
> at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
> at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:374)
> at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:363)
> at org.apache.phoenix.jdbc.PhoenixStatement.execute(PhoenixStatement.java:1707)
> at org.apache.phoenix.end2end.BaseQueryIT.<init>(BaseQueryIT.java:139)
> at org.apache.phoenix.end2end.NotQueryIT.<init>(NotQueryIT.java:56)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
> at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
> {code}
> {code}
> 2017-09-06 02:57:28,819 ERROR [main] org.apache.phoenix.end2end.BaseQueryIT(141): Exception while creating index: CREATE INDEX T000350 ON T000349 (a_integer, a_string) INCLUDE ( B_STRING, A_DATE) KEEP_DELETED_CELLS=false
> java.sql.SQLTimeoutException: Operation timed out.
> at org.apache.phoenix.exception.SQLExceptionCode$15.newException(SQLExceptionCode.java:399)
> at org.apache.phoenix.exception.SQLExceptionInfo.buildException(SQLExceptionInfo.java:150)
> at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:932)
> at org.apache.phoenix.iterate.BaseResultIterators.getIterators(BaseResultIterators.java:846)
> at org.apache.phoenix.iterate.ConcatResultIterator.getIterators(ConcatResultIterator.java:50)
> at org.apache.phoenix.iterate.ConcatResultIterator.currentIterator(ConcatResultIterator.java:97)
> at org.apache.phoenix.iterate.ConcatResultIterator.next(ConcatResultIterator.java:117)
> at org.apache.phoenix.iterate.BaseGroupedAggregatingResultIterator.next(BaseGroupedAggregatingResultIterator.java:64)
> at org.apache.phoenix.iterate.UngroupedAggregatingResultIterator.next(UngroupedAggregatingResultIterator.java:39)
> at org.apache.phoenix.compile.UpsertCompiler$1.execute(UpsertCompiler.java:734)
> at org.apache.phoenix.compile.DelegateMutationPlan.execute(DelegateMutationPlan.java:31)
> at org.apache.phoenix.compile.PostIndexDDLCompiler$1.execute(PostIndexDDLCompiler.java:117)
> at org.apache.phoenix.query.ConnectionQueryServicesImpl.updateData(ConnectionQueryServicesImpl.java:3359)
> at org.apache.phoenix.schema.MetaDataClient.buildIndex(MetaDataClient.java:1282)
> at org.apache.phoenix.schema.MetaDataClient.buildIndexAtTimeStamp(MetaDataClient.java:1222)
> at org.apache.phoenix.schema.MetaDataClient.createIndex(MetaDataClient.java:1588)
> at org.apache.phoenix.compile.CreateIndexCompiler$1.execute(CreateIndexCompiler.java:85)
> at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:393)
> at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:376)
> at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)
> at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:374)
> at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:363)
> at org.apache.phoenix.jdbc.PhoenixStatement.execute(PhoenixStatement.java:1707)
> at org.apache.phoenix.end2end.BaseQueryIT.<init>(BaseQueryIT.java:139)
> at org.apache.phoenix.end2end.QueryIT.<init>(QueryIT.java:66)
> at sun.reflect.GeneratedConstructorAccessor118.newInstance(Unknown Source)
> at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)