You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@hbase.apache.org by "Nihal Jain (Jira)" <ji...@apache.org> on 2021/11/18 06:37:00 UTC
[jira] [Commented] (HBASE-26464) WALPrettyPrinter: JSON output is broken/malformed
[ https://issues.apache.org/jira/browse/HBASE-26464?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17445666#comment-17445666 ]
Nihal Jain commented on HBASE-26464:
------------------------------------
Handling edit heap size and position fields are simple, we can just move it inside each edit.
But, the writer classes and cell codec classes in current JSON adds a little complexity. I have 2 proposal for fix.
# Keep Json schema same by adding these 2 fields in each edit entry
{code:java}
[
{
"sequence": "3",
"cellCodecClass": "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec",
"writerClasses": [
"ProtobufLogWriter",
"AsyncProtobufLogWriter"
],
"editHeapSize": "328",
"position": "389",
"region": "f0153c5d890d96f06ec304eebc4aacf9",
"actions": [
{
"qualifier": "HBASE::REGION_EVENT::REGION_OPEN",
"vlen": 176,
"row": "\\x00",
"type": "Put",
"family": "METAFAMILY",
"timestamp": "1637170532553",
"total_size_sum": "288"
}
],
"table": {
"name": [
104,
98,
97,
115,
101,
58,
110,
97,
109,
101,
115,
112,
97,
99,
101
],
"nameAsString": "hbase:namespace",
"namespace": [
104,
98,
97,
115,
101
],
"namespaceAsString": "hbase",
"qualifier": [
110,
97,
109,
101,
115,
112,
97,
99,
101
],
"qualifierAsString": "namespace",
"systemTable": true,
"hashCode": 771659354
}
},
{
"sequence": "26",
"cellCodecClass": "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec",
"writerClasses": [
"ProtobufLogWriter",
"AsyncProtobufLogWriter"
],
"editHeapSize": "376",
"position": "726",
"region": "e42fbcf354fcf7e4e425f2d06130797c",
"actions": [
{
"qualifier": "HBASE::REGION_EVENT::REGION_OPEN",
"vlen": 218,
"row": "\\x00",
"type": "Put",
"family": "METAFAMILY",
"timestamp": "1637170533215",
"total_size_sum": "336"
}
],
"table": {
"name": [
116,
101,
115,
116
],
"nameAsString": "test",
"namespace": [
100,
101,
102,
97,
117,
108,
116
],
"namespaceAsString": "default",
"qualifier": [
116,
101,
115,
116
],
"qualifierAsString": "test",
"systemTable": false,
"hashCode": 3556498
}
},
{
"sequence": "3",
"cellCodecClass": "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec",
"writerClasses": [
"ProtobufLogWriter",
"AsyncProtobufLogWriter"
],
"editHeapSize": "328",
"position": "389",
"region": "f0153c5d890d96f06ec304eebc4aacf9",
"actions": [
{
"qualifier": "HBASE::REGION_EVENT::REGION_OPEN",
"vlen": 176,
"row": "\\x00",
"type": "Put",
"family": "METAFAMILY",
"timestamp": "1637170532553",
"total_size_sum": "288"
}
],
"table": {
"name": [
104,
98,
97,
115,
101,
58,
110,
97,
109,
101,
115,
112,
97,
99,
101
],
"nameAsString": "hbase:namespace",
"namespace": [
104,
98,
97,
115,
101
],
"namespaceAsString": "hbase",
"qualifier": [
110,
97,
109,
101,
115,
112,
97,
99,
101
],
"qualifierAsString": "namespace",
"systemTable": true,
"hashCode": 771659354
}
},
{
"sequence": "26",
"cellCodecClass": "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec",
"writerClasses": [
"ProtobufLogWriter",
"AsyncProtobufLogWriter"
],
"editHeapSize": "376",
"position": "726",
"region": "e42fbcf354fcf7e4e425f2d06130797c",
"actions": [
{
"qualifier": "HBASE::REGION_EVENT::REGION_OPEN",
"vlen": 218,
"row": "\\x00",
"type": "Put",
"family": "METAFAMILY",
"timestamp": "1637170533215",
"total_size_sum": "336"
}
],
"table": {
"name": [
116,
101,
115,
116
],
"nameAsString": "test",
"namespace": [
100,
101,
102,
97,
117,
108,
116
],
"namespaceAsString": "default",
"qualifier": [
116,
101,
115,
116
],
"qualifierAsString": "test",
"systemTable": false,
"hashCode": 3556498
}
}
]
{code}
# Restructure schema to have a list of wal file and corresponding entries, where entry is a list of edits of that wal file
{code:java}
[
{
"fileName": "/apps/hbase/data/WALs/hostname1,16020,1636451511104/hostname1%2C16020%2C1636451511104.1637138938339",
"writerClasses": [
"ProtobufLogWriter",
"AsyncProtobufLogWriter"
],
"cellCodecClass": "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec",
"edits": [
{
"sequence": "31",
"region": "9792343a024b19611352546a28720e8e",
"actions": [
{
"qualifier": "",
"vlen": 3,
"row": "user4",
"type": "Put",
"family": "cf",
"timestamp": "1637141077993",
"total_size_sum": "80"
}
],
"table": {
"name": [
110,
106,
58,
119,
97,
108,
80,
80
],
"nameAsString": "nj:walPP",
"namespace": [
110,
106
],
"namespaceAsString": "nj",
"qualifier": [
119,
97,
108,
80,
80
],
"qualifierAsString": "walPP",
"systemTable": false,
"hashCode": -204895964
},
"editHeapSize": 120,
"position": 173
},
{
"sequence": "32",
"region": "9792343a024b19611352546a28720e8e",
"actions": [
{
"qualifier": "",
"vlen": 3,
"row": "user5",
"type": "Put",
"family": "cf",
"timestamp": "1637141077996",
"total_size_sum": "80"
}
],
"table": {
"name": [
110,
106,
58,
119,
97,
108,
80,
80
],
"nameAsString": "nj:walPP",
"namespace": [
110,
106
],
"namespaceAsString": "nj",
"qualifier": [
119,
97,
108,
80,
80
],
"qualifierAsString": "walPP",
"systemTable": false,
"hashCode": -204895964
},
"editHeapSize": 120,
"position": 263
}
]
},
{
"fileName": "/apps/hbase/data/WALs/hostname2,16020,1636451511106/hostname2%2C16020%2C1636451511106.1637138954906 ",
"writerClasses": [
"ProtobufLogWriter",
"AsyncProtobufLogWriter"
],
"cellCodecClass": "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec",
"edits": [
{
"sequence": "31",
"region": "24a0775fe33056988afc8cedf9caab98",
"actions": [
{
"qualifier": "",
"vlen": 3,
"row": "user8",
"type": "Put",
"family": "cf",
"timestamp": "1637141078031",
"total_size_sum": "80"
}
],
"table": {
"name": [
110,
106,
58,
119,
97,
108,
80,
80
],
"nameAsString": "nj:walPP",
"namespace": [
110,
106
],
"namespaceAsString": "nj",
"qualifier": [
119,
97,
108,
80,
80
],
"qualifierAsString": "walPP",
"systemTable": false,
"hashCode": -204895964
},
"editHeapSize": 120,
"position": 173
},
{
"sequence": "54",
"region": "24a0775fe33056988afc8cedf9caab98",
"actions": [
{
"qualifier": "",
"vlen": 4,
"row": "user99",
"type": "Put",
"family": "cf",
"timestamp": "1637141078178",
"total_size_sum": "80"
}
],
"table": {
"name": [
110,
106,
58,
119,
97,
108,
80,
80
],
"nameAsString": "nj:walPP",
"namespace": [
110,
106
],
"namespaceAsString": "nj",
"qualifier": [
119,
97,
108,
80,
80
],
"qualifierAsString": "walPP",
"systemTable": false,
"hashCode": -204895964
},
"editHeapSize": 120,
"position": 2287
}
]
}
]
{code}
I already have a patch for proposal 1 ready. Let me if it sounds good.
Also, we could also provide an option to skip printing writer and codec classes.
> WALPrettyPrinter: JSON output is broken/malformed
> -------------------------------------------------
>
> Key: HBASE-26464
> URL: https://issues.apache.org/jira/browse/HBASE-26464
> Project: HBase
> Issue Type: Bug
> Components: tooling
> Reporter: Nihal Jain
> Assignee: Nihal Jain
> Priority: Major
>
> The JSON output of wal pretty printer is malformed and cannot be parsed.
> There are 2 major issue:
> * The writer classes and cell codec classes are added (as normal text) whenever a new WAL file printing starts (NOTE: WALPrettyPrinter can take multiple WAL files as input)
> * The edit heap size and position are outside json formatting
> These are side effects of some patches done in past which missed to cover json output scenarios.
> Current sample output, which is malformed JSON:
> {code:java}
> [Writer Classes: ProtobufLogWriter AsyncProtobufLogWriter
> Cell Codec Class: org.apache.hadoop.hbase.regionserver.wal.WALCellCodec
> {"sequence":"3","region":"f0153c5d890d96f06ec304eebc4aacf9","actions":[{"qualifier":"HBASE::REGION_EVENT::REGION_OPEN","vlen":176,"row":"\\x00","type":"Put","family":"METAFAMILY","timestamp":"1637170532553","total_size_sum":"288"}],"table":{"name":[104,98,97,115,101,58,110,97,109,101,115,112,97,99,101],"nameAsString":"hbase:namespace","namespace":[104,98,97,115,101],"namespaceAsString":"hbase","qualifier":[110,97,109,101,115,112,97,99,101],"qualifierAsString":"namespace","systemTable":true,"hashCode":771659354}}edit heap size: 328
> position: 389
> ,{"sequence":"26","region":"e42fbcf354fcf7e4e425f2d06130797c","actions":[{"qualifier":"HBASE::REGION_EVENT::REGION_OPEN","vlen":218,"row":"\\x00","type":"Put","family":"METAFAMILY","timestamp":"1637170533215","total_size_sum":"336"}],"table":{"name":[116,101,115,116],"nameAsString":"test","namespace":[100,101,102,97,117,108,116],"namespaceAsString":"default","qualifier":[116,101,115,116],"qualifierAsString":"test","systemTable":false,"hashCode":3556498}}edit heap size: 376
> position: 726
> ,{"sequence":"4","region":"f0153c5d890d96f06ec304eebc4aacf9","actions":[{"qualifier":"d","vlen":9,"row":"default","type":"Put","family":"info","timestamp":"1637170533278","total_size_sum":"96"}],"table":{"name":[104,98,97,115,101,58,110,97,109,101,115,112,97,99,101],"nameAsString":"hbase:namespace","namespace":[104,98,97,115,101],"namespaceAsString":"hbase","qualifier":[110,97,109,101,115,112,97,99,101],"qualifierAsString":"namespace","systemTable":true,"hashCode":771659354}}edit heap size: 136
> position: 834
> ,{"sequence":"5","region":"f0153c5d890d96f06ec304eebc4aacf9","actions":[{"qualifier":"d","vlen":7,"row":"hbase","type":"Put","family":"info","timestamp":"1637170533340","total_size_sum":"88"}],"table":{"name":[104,98,97,115,101,58,110,97,109,101,115,112,97,99,101],"nameAsString":"hbase:namespace","namespace":[104,98,97,115,101],"namespaceAsString":"hbase","qualifier":[110,97,109,101,115,112,97,99,101],"qualifierAsString":"namespace","systemTable":true,"hashCode":771659354}}edit heap size: 128
> position: 938
> ,{"sequence":"27","region":"e42fbcf354fcf7e4e425f2d06130797c","actions":[{"qualifier":"","vlen":2,"row":"r6","type":"Put","family":"cf","timestamp":"1637170714545","total_size_sum":"80"}],"table":{"name":[116,101,115,116],"nameAsString":"test","namespace":[100,101,102,97,117,108,116],"namespaceAsString":"default","qualifier":[116,101,115,116],"qualifierAsString":"test","systemTable":false,"hashCode":3556498}}edit heap size: 120
> position: 1020
> Writer Classes: ProtobufLogWriter AsyncProtobufLogWriter
> Cell Codec Class: org.apache.hadoop.hbase.regionserver.wal.WALCellCodec
> ,{"sequence":"3","region":"f0153c5d890d96f06ec304eebc4aacf9","actions":[{"qualifier":"HBASE::REGION_EVENT::REGION_OPEN","vlen":176,"row":"\\x00","type":"Put","family":"METAFAMILY","timestamp":"1637170532553","total_size_sum":"288"}],"table":{"name":[104,98,97,115,101,58,110,97,109,101,115,112,97,99,101],"nameAsString":"hbase:namespace","namespace":[104,98,97,115,101],"namespaceAsString":"hbase","qualifier":[110,97,109,101,115,112,97,99,101],"qualifierAsString":"namespace","systemTable":true,"hashCode":771659354}}edit heap size: 328
> position: 389
> ,{"sequence":"26","region":"e42fbcf354fcf7e4e425f2d06130797c","actions":[{"qualifier":"HBASE::REGION_EVENT::REGION_OPEN","vlen":218,"row":"\\x00","type":"Put","family":"METAFAMILY","timestamp":"1637170533215","total_size_sum":"336"}],"table":{"name":[116,101,115,116],"nameAsString":"test","namespace":[100,101,102,97,117,108,116],"namespaceAsString":"default","qualifier":[116,101,115,116],"qualifierAsString":"test","systemTable":false,"hashCode":3556498}}edit heap size: 376
> position: 726
> ,{"sequence":"4","region":"f0153c5d890d96f06ec304eebc4aacf9","actions":[{"qualifier":"d","vlen":9,"row":"default","type":"Put","family":"info","timestamp":"1637170533278","total_size_sum":"96"}],"table":{"name":[104,98,97,115,101,58,110,97,109,101,115,112,97,99,101],"nameAsString":"hbase:namespace","namespace":[104,98,97,115,101],"namespaceAsString":"hbase","qualifier":[110,97,109,101,115,112,97,99,101],"qualifierAsString":"namespace","systemTable":true,"hashCode":771659354}}edit heap size: 136
> position: 834
> ,{"sequence":"5","region":"f0153c5d890d96f06ec304eebc4aacf9","actions":[{"qualifier":"d","vlen":7,"row":"hbase","type":"Put","family":"info","timestamp":"1637170533340","total_size_sum":"88"}],"table":{"name":[104,98,97,115,101,58,110,97,109,101,115,112,97,99,101],"nameAsString":"hbase:namespace","namespace":[104,98,97,115,101],"namespaceAsString":"hbase","qualifier":[110,97,109,101,115,112,97,99,101],"qualifierAsString":"namespace","systemTable":true,"hashCode":771659354}}edit heap size: 128
> position: 938
> ,{"sequence":"27","region":"e42fbcf354fcf7e4e425f2d06130797c","actions":[{"qualifier":"","vlen":2,"row":"r6","type":"Put","family":"cf","timestamp":"1637170714545","total_size_sum":"80"}],"table":{"name":[116,101,115,116],"nameAsString":"test","namespace":[100,101,102,97,117,108,116],"namespaceAsString":"default","qualifier":[116,101,115,116],"qualifierAsString":"test","systemTable":false,"hashCode":3556498}}edit heap size: 120
> position: 1020
> ]
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)