You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@drill.apache.org by "Chun Chang (JIRA)" <ji...@apache.org> on 2014/11/13 19:58:34 UTC
[jira] [Commented] (DRILL-1700) caught a memory assertion on an
order by
[ https://issues.apache.org/jira/browse/DRILL-1700?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14210154#comment-14210154 ]
Chun Chang commented on DRILL-1700:
-----------------------------------
plan without limit:
0: jdbc:drill:schema=dfs> explain plan for select columns[0], columns[1], columns[2] from `aggregate_1m.csv` order by cast(columns[0] as int);
+------------+------------+
| text | json |
+------------+------------+
| 00-00 Screen
00-01 Project(EXPR$0=[$0], EXPR$1=[$1], EXPR$2=[$2])
00-02 SelectionVectorRemover
00-03 Sort(sort0=[$3], dir0=[ASC])
00-04 Project(EXPR$0=[ITEM($0, 0)], EXPR$1=[ITEM($0, 1)], EXPR$2=[ITEM($0, 2)], EXPR$3=[CAST(ITEM($0, 0)):INTEGER NOT NULL])
00-05 Scan(groupscan=[EasyGroupScan [selectionRoot=/drill/testdata/mondrian/aggregate_1m.csv, numFiles=1, columns=[`columns`[0], `columns`[1], `columns`[2]], files=[maprfs:/drill/testdata/mondrian/aggregate_1m.csv]]])
| {
"head" : {
"version" : 1,
"generator" : {
"type" : "ExplainHandler",
"info" : ""
},
"type" : "APACHE_DRILL_PHYSICAL",
"options" : [ ],
"queue" : 0,
"resultMode" : "EXEC"
},
"graph" : [ {
"pop" : "fs-scan",
"@id" : 5,
"files" : [ "maprfs:/drill/testdata/mondrian/aggregate_1m.csv" ],
"storage" : {
"type" : "file",
"enabled" : true,
"connection" : "maprfs:///",
"workspaces" : {
"root" : {
"location" : "/drill/testdata",
"writable" : false,
"defaultInputFormat" : null
},
"default" : {
"location" : "/drill/testdata/mondrian",
"writable" : true,
"defaultInputFormat" : null
},
"tmp" : {
"location" : "/tmp",
"writable" : true,
"defaultInputFormat" : null
}
},
"formats" : {
"psv" : {
"type" : "text",
"extensions" : [ "tbl" ],
"delimiter" : "|"
},
"csv" : {
"type" : "text",
"extensions" : [ "csv" ],
"delimiter" : ","
},
"tsv" : {
"type" : "text",
"extensions" : [ "tsv" ],
"delimiter" : "\t"
},
"parquet" : {
"type" : "parquet"
},
"json" : {
"type" : "json"
}
}
},
"format" : {
"type" : "text",
"extensions" : [ "csv" ],
"delimiter" : ","
},
"columns" : [ "`columns`[0]", "`columns`[1]", "`columns`[2]" ],
"selectionRoot" : "/drill/testdata/mondrian/aggregate_1m.csv",
"cost" : 52083.0
}, {
"pop" : "project",
"@id" : 4,
"exprs" : [ {
"ref" : "`EXPR$0`",
"expr" : "`columns`[0]"
}, {
"ref" : "`EXPR$1`",
"expr" : "`columns`[1]"
}, {
"ref" : "`EXPR$2`",
"expr" : "`columns`[2]"
}, {
"ref" : "`EXPR$3`",
"expr" : "cast( (`columns`[0] ) as INT )"
} ],
"child" : 5,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "external-sort",
"@id" : 3,
"child" : 4,
"orderings" : [ {
"order" : "ASC",
"expr" : "`EXPR$3`",
"nullDirection" : "UNSPECIFIED"
} ],
"reverse" : false,
"initialAllocation" : 20000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "selection-vector-remover",
"@id" : 2,
"child" : 3,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "project",
"@id" : 1,
"exprs" : [ {
"ref" : "`EXPR$0`",
"expr" : "`EXPR$0`"
}, {
"ref" : "`EXPR$1`",
"expr" : "`EXPR$1`"
}, {
"ref" : "`EXPR$2`",
"expr" : "`EXPR$2`"
} ],
"child" : 2,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "screen",
"@id" : 0,
"child" : 1,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
} ]
} |
+------------+------------+
1 row selected (8.539 seconds)
plan with limit 100:
0: jdbc:drill:schema=dfs> explain plan for select columns[0], columns[1], columns[2] from `aggregate_1m.csv` order by cast(columns[0] as int) limit 100;
+------------+------------+
| text | json |
+------------+------------+
| 00-00 Screen
00-01 Project(EXPR$0=[$0], EXPR$1=[$1], EXPR$2=[$2])
00-02 SelectionVectorRemover
00-03 Limit(fetch=[100])
00-04 SelectionVectorRemover
00-05 TopN(limit=[100])
00-06 Project(EXPR$0=[ITEM($0, 0)], EXPR$1=[ITEM($0, 1)], EXPR$2=[ITEM($0, 2)], EXPR$3=[CAST(ITEM($0, 0)):INTEGER NOT NULL])
00-07 Scan(groupscan=[EasyGroupScan [selectionRoot=/drill/testdata/mondrian/aggregate_1m.csv, numFiles=1, columns=[`columns`[0], `columns`[1], `columns`[2]], files=[maprfs:/drill/testdata/mondrian/aggregate_1m.csv]]])
| {
"head" : {
"version" : 1,
"generator" : {
"type" : "ExplainHandler",
"info" : ""
},
"type" : "APACHE_DRILL_PHYSICAL",
"options" : [ ],
"queue" : 0,
"resultMode" : "EXEC"
},
"graph" : [ {
"pop" : "fs-scan",
"@id" : 7,
"files" : [ "maprfs:/drill/testdata/mondrian/aggregate_1m.csv" ],
"storage" : {
"type" : "file",
"enabled" : true,
"connection" : "maprfs:///",
"workspaces" : {
"root" : {
"location" : "/drill/testdata",
"writable" : false,
"defaultInputFormat" : null
},
"default" : {
"location" : "/drill/testdata/mondrian",
"writable" : true,
"defaultInputFormat" : null
},
"tmp" : {
"location" : "/tmp",
"writable" : true,
"defaultInputFormat" : null
}
},
"formats" : {
"psv" : {
"type" : "text",
"extensions" : [ "tbl" ],
"delimiter" : "|"
},
"csv" : {
"type" : "text",
"extensions" : [ "csv" ],
"delimiter" : ","
},
"tsv" : {
"type" : "text",
"extensions" : [ "tsv" ],
"delimiter" : "\t"
},
"parquet" : {
"type" : "parquet"
},
"json" : {
"type" : "json"
}
}
},
"format" : {
"type" : "text",
"extensions" : [ "csv" ],
"delimiter" : ","
},
"columns" : [ "`columns`[0]", "`columns`[1]", "`columns`[2]" ],
"selectionRoot" : "/drill/testdata/mondrian/aggregate_1m.csv",
"cost" : 52083.0
}, {
"pop" : "project",
"@id" : 6,
"exprs" : [ {
"ref" : "`EXPR$0`",
"expr" : "`columns`[0]"
}, {
"ref" : "`EXPR$1`",
"expr" : "`columns`[1]"
}, {
"ref" : "`EXPR$2`",
"expr" : "`columns`[2]"
}, {
"ref" : "`EXPR$3`",
"expr" : "cast( (`columns`[0] ) as INT )"
} ],
"child" : 7,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "top-n",
"@id" : 5,
"child" : 6,
"orderings" : [ {
"order" : "ASC",
"expr" : "`EXPR$3`",
"nullDirection" : "UNSPECIFIED"
} ],
"reverse" : false,
"limit" : 100,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "selection-vector-remover",
"@id" : 4,
"child" : 5,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "limit",
"@id" : 3,
"child" : 4,
"first" : 0,
"last" : 100,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "selection-vector-remover",
"@id" : 2,
"child" : 3,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "project",
"@id" : 1,
"exprs" : [ {
"ref" : "`EXPR$0`",
"expr" : "`EXPR$0`"
}, {
"ref" : "`EXPR$1`",
"expr" : "`EXPR$1`"
}, {
"ref" : "`EXPR$2`",
"expr" : "`EXPR$2`"
} ],
"child" : 2,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
}, {
"pop" : "screen",
"@id" : 0,
"child" : 1,
"initialAllocation" : 1000000,
"maxAllocation" : 10000000000,
"cost" : 52083.0
} ]
} |
+------------+------------+
1 row selected (0.793 seconds)
> caught a memory assertion on an order by
> ----------------------------------------
>
> Key: DRILL-1700
> URL: https://issues.apache.org/jira/browse/DRILL-1700
> Project: Apache Drill
> Issue Type: Bug
> Components: Query Planning & Optimization
> Affects Versions: 0.7.0
> Reporter: Chun Chang
> Assignee: Parth Chandra
> Priority: Blocker
>
> #Wed Nov 12 13:06:45 EST 2014
> git.commit.id.abbrev=1e21045
> The following query on a 1 million row data caused a memory assertion.
> 0: jdbc:drill:schema=dfs> select columns[0], columns[1], columns[2] from `aggregate_1m.csv` order by cast(columns[0] as int);
> +------------+------------+------------+
> | EXPR$0 | EXPR$1 | EXPR$2 |
> +------------+------------+------------+
> Query failed: Failure while running fragment.[ 7bc42b6c-c00a-41c2-af43-331655e5a178 on qa-node119.qa.lab:31010 ]
> java.lang.RuntimeException: java.sql.SQLException: Failure while executing query.
> at sqlline.SqlLine$IncrementalRows.hasNext(SqlLine.java:2514)
> at sqlline.SqlLine$TableOutputFormat.print(SqlLine.java:2148)
> at sqlline.SqlLine.print(SqlLine.java:1809)
> The same query adding a limit (say limit 100) worked. Here is the assertion stack in drill bit log:
> 2014-11-12 15:16:22,864 [f89183d7-f94d-4ece-818f-8e597859d529:frag:0:0] WARN o.a.d.e.w.fragment.FragmentExecutor - Error while initializing or executing fragment
> java.lang.AssertionError: null
> at org.apache.drill.exec.memory.AtomicRemainder.get(AtomicRemainder.java:126) ~[drill-java-exec-0.7.0-incubating-SNAPSHOT-rebuffed.jar:0.7.0-incubating-SNAPSHOT]
> at org.apache.drill.exec.memory.AtomicRemainder.forceGet(AtomicRemainder.java:85) ~[drill-java-exec-0.7.0-incubating-SNAPSHOT-rebuffed.jar:0.7.0-incubating-SNAPSHOT]
> at org.apache.drill.exec.memory.Accountor.forceAdditionalReservation(Accountor.java:142) ~[drill-java-exec-0.7.0-incubating-SNAPSHOT-rebuffed.jar:0.7.0-incubating-SNAPSHOT]
> at org.apache.drill.exec.memory.Accountor.transferTo(Accountor.java:111) ~[drill-java-exec-0.7.0-incubating-SNAPSHOT-rebuffed.jar:0.7.0-incubating-SNAPSHOT]
> at io.netty.buffer.DrillBuf.transferAccounting(DrillBuf.java:182) ~[drill-java-exec-0.7.0-incubating-SNAPSHOT-rebuffed.jar:4.0.24.Final]
> at org.apache.drill.exec.memory.TopLevelAllocator$ChildAllocator.takeOwnership(TopLevelAllocator.java:192) ~[drill-java-exec-0.7.0-incubating-SNAPSHOT-rebuffed.jar:0.7.0-incubating-SNAPSHOT]
> at org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch.takeOwnership(ExternalSortBatch.java:464) ~[drill-java-exec-0.7.0-incubating-SNAPSHOT-rebuffed.jar:0.7.0-incubating-SNAPSHOT]
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)