You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hive.apache.org by Laszlo Pinter via Review Board <no...@reviews.apache.org> on 2020/01/04 09:06:54 UTC
Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/
-----------------------------------------------------------
Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
Repository: hive-git
Description
-------
HIVE-20934: ACID: Query based compactor for minor compaction
Diffs
-----
itests/hive-unit/pom.xml bc20cd6168dd61222c75fb866deada26328986dd
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 445e39c260edc68f511550271a7ac471fae908fe
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 3c508ec6cf620aee6a7791c6ab52c331ad5ec6bd
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 20b0ccd94b5f08aa2c1dace1301a8315bd202bf7
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 2b2cc1a2ba8377aa3681b1a3454a0d64369eef64
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 7a0e32463d28007cff5526ae037cc1447e50a50b
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
Diff: https://reviews.apache.org/r/71949/diff/1/
Testing
-------
Thanks,
Laszlo Pinter
Re: Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
Posted by Peter Vary via Review Board <no...@reviews.apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/#review219177
-----------------------------------------------------------
Ship it!
Ship It!
- Peter Vary
On jan. 8, 2020, 10:40 de, Laszlo Pinter wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/71949/
> -----------------------------------------------------------
>
> (Updated jan. 8, 2020, 10:40 de)
>
>
> Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-20934: ACID: Query based compactor for minor compaction
>
>
> Diffs
> -----
>
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 7ae33fadf7c8da83924e317287d160faf5364e3d
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
> ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java dde8878769e10191c5ba61bd1ba44d9b16b172c1
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
> ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ad6817c32bbfad1d27023b25912b1204f069a66a
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java e26794355ca1e73be5c103f8405c471da870bbe3
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 2253fda6c6ed5e2f70ef7c1166895eb49f600ea9
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
>
>
> Diff: https://reviews.apache.org/r/71949/diff/5/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Laszlo Pinter
>
>
Re: Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
Posted by Laszlo Pinter via Review Board <no...@reviews.apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/
-----------------------------------------------------------
(Updated Jan. 8, 2020, 10:40 a.m.)
Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
Changes
-------
Fixed code review findings.
Repository: hive-git
Description
-------
HIVE-20934: ACID: Query based compactor for minor compaction
Diffs (updated)
-----
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 445e39c260edc68f511550271a7ac471fae908fe
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java dde8878769e10191c5ba61bd1ba44d9b16b172c1
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ad6817c32bbfad1d27023b25912b1204f069a66a
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 2b2cc1a2ba8377aa3681b1a3454a0d64369eef64
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 2253fda6c6ed5e2f70ef7c1166895eb49f600ea9
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
Diff: https://reviews.apache.org/r/71949/diff/3/
Changes: https://reviews.apache.org/r/71949/diff/2-3/
Testing
-------
Thanks,
Laszlo Pinter
Re: Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
Posted by Laszlo Pinter via Review Board <no...@reviews.apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/#review219172
-----------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
Lines 247 (patched)
<https://reviews.apache.org/r/71949/#comment307289>
I didn't want to make this string globally avalaible for other components as well.
- Laszlo Pinter
On Jan. 7, 2020, 2:24 p.m., Laszlo Pinter wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/71949/
> -----------------------------------------------------------
>
> (Updated Jan. 7, 2020, 2:24 p.m.)
>
>
> Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-20934: ACID: Query based compactor for minor compaction
>
>
> Diffs
> -----
>
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 445e39c260edc68f511550271a7ac471fae908fe
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
> ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 3c508ec6cf620aee6a7791c6ab52c331ad5ec6bd
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
> ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ad6817c32bbfad1d27023b25912b1204f069a66a
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 2b2cc1a2ba8377aa3681b1a3454a0d64369eef64
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 7a0e32463d28007cff5526ae037cc1447e50a50b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
>
>
> Diff: https://reviews.apache.org/r/71949/diff/2/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Laszlo Pinter
>
>
Re: Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
Posted by Peter Vary via Review Board <no...@reviews.apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/#review219171
-----------------------------------------------------------
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 1597 (original), 1602 (patched)
<https://reviews.apache.org/r/71949/#comment307279>
Question: Is it intentional, that this writeBatch is not moved to the util?
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 1798 (original), 1675 (patched)
<https://reviews.apache.org/r/71949/#comment307280>
nit: still some only formatting change?
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 1831 (original), 1685 (patched)
<https://reviews.apache.org/r/71949/#comment307281>
nit: Only formatting change?
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
Line 64 (original), 63 (patched)
<https://reviews.apache.org/r/71949/#comment307283>
nit: Only formatting change?
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
Line 70 (original), 68 (patched)
<https://reviews.apache.org/r/71949/#comment307284>
why move this to static, if the initialization is still done in @Before?
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
Line 113 (original), 111 (patched)
<https://reviews.apache.org/r/71949/#comment307285>
nit: only formatting?
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
Line 174 (original), 171 (patched)
<https://reviews.apache.org/r/71949/#comment307286>
Only formatting, and some more - will not mark the others
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
Lines 247 (patched)
<https://reviews.apache.org/r/71949/#comment307287>
Is it worth to use a constant instead for "queryminorcomp"?
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
Line 287 (original), 319 (patched)
<https://reviews.apache.org/r/71949/#comment307288>
formatting only changes....
- Peter Vary
On jan. 7, 2020, 2:24 du, Laszlo Pinter wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/71949/
> -----------------------------------------------------------
>
> (Updated jan. 7, 2020, 2:24 du)
>
>
> Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-20934: ACID: Query based compactor for minor compaction
>
>
> Diffs
> -----
>
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 445e39c260edc68f511550271a7ac471fae908fe
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
> ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 3c508ec6cf620aee6a7791c6ab52c331ad5ec6bd
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
> ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ad6817c32bbfad1d27023b25912b1204f069a66a
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 2b2cc1a2ba8377aa3681b1a3454a0d64369eef64
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 7a0e32463d28007cff5526ae037cc1447e50a50b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
>
>
> Diff: https://reviews.apache.org/r/71949/diff/2/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Laszlo Pinter
>
>
Re: Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
Posted by Laszlo Pinter via Review Board <no...@reviews.apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/
-----------------------------------------------------------
(Updated Jan. 7, 2020, 2:24 p.m.)
Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
Changes
-------
Fixed code review findings.
Repository: hive-git
Description
-------
HIVE-20934: ACID: Query based compactor for minor compaction
Diffs (updated)
-----
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 445e39c260edc68f511550271a7ac471fae908fe
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 3c508ec6cf620aee6a7791c6ab52c331ad5ec6bd
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ad6817c32bbfad1d27023b25912b1204f069a66a
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 2b2cc1a2ba8377aa3681b1a3454a0d64369eef64
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 7a0e32463d28007cff5526ae037cc1447e50a50b
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
Diff: https://reviews.apache.org/r/71949/diff/2/
Changes: https://reviews.apache.org/r/71949/diff/1-2/
Testing
-------
Thanks,
Laszlo Pinter
Re: Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
Posted by Laszlo Pinter via Review Board <no...@reviews.apache.org>.
> On Jan. 7, 2020, 10:21 a.m., Peter Vary wrote:
> > ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
> > Lines 282 (patched)
> > <https://reviews.apache.org/r/71949/diff/1/?file=2193164#file2193164line283>
> >
> > I do not see this in the original code. What is this for?
I made a bit of simplification here. The OrcSplit.parse() call does some metadata preparation steps, like fetching the writeID, transactionID and bucketID and setting it to the split. Originally, this was done in ComparatorCompactor, but my point of view is that the comparator should be used only for comparing OrcSplits and not to prepare data. Since, we are already iterating over the splits, it made sense to move the call here.
> On Jan. 7, 2020, 10:21 a.m., Peter Vary wrote:
> > ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
> > Lines 1228 (patched)
> > <https://reviews.apache.org/r/71949/diff/1/?file=2193166#file2193166line1228>
> >
> > Do we need this? With delete_delta we do not supposed to have syntehetic rowIDs...
That is correct, but I had to introduce this condition as well, to avoid IllegalStateException. Without this check, the splits in a delete delta directory would be considered as original files but without the correct parent.
- Laszlo
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/#review219140
-----------------------------------------------------------
On Jan. 4, 2020, 9:06 a.m., Laszlo Pinter wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/71949/
> -----------------------------------------------------------
>
> (Updated Jan. 4, 2020, 9:06 a.m.)
>
>
> Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-20934: ACID: Query based compactor for minor compaction
>
>
> Diffs
> -----
>
> itests/hive-unit/pom.xml bc20cd6168dd61222c75fb866deada26328986dd
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 445e39c260edc68f511550271a7ac471fae908fe
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
> ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 3c508ec6cf620aee6a7791c6ab52c331ad5ec6bd
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
> ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 20b0ccd94b5f08aa2c1dace1301a8315bd202bf7
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 2b2cc1a2ba8377aa3681b1a3454a0d64369eef64
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 7a0e32463d28007cff5526ae037cc1447e50a50b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
>
>
> Diff: https://reviews.apache.org/r/71949/diff/1/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Laszlo Pinter
>
>
Re: Review Request 71949: HIVE-20934: ACID: Query based compactor for
minor compaction
Posted by Peter Vary via Review Board <no...@reviews.apache.org>.
-----------------------------------------------------------
This is an automatically generated e-mail. To reply, visit:
https://reviews.apache.org/r/71949/#review219140
-----------------------------------------------------------
itests/hive-unit/pom.xml
Lines 440 (patched)
<https://reviews.apache.org/r/71949/#comment307237>
nit: formatting?
question: Do we really need guava? I hate this dependency as a general rule try to avoid it.
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 1 (original), 1 (patched)
<https://reviews.apache.org/r/71949/#comment307244>
Hard to review the changes because of the formatting differences... Let's talk
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 10 (original), 10 (patched)
<https://reviews.apache.org/r/71949/#comment307238>
nit: Do you know what is this change?
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 116 (original), 112 (patched)
<https://reviews.apache.org/r/71949/#comment307243>
' ' is needed after ','
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 198 (original), 191 (patched)
<https://reviews.apache.org/r/71949/#comment307239>
nit: unnecessary '+' in the middle
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 213 (original), 207 (patched)
<https://reviews.apache.org/r/71949/#comment307240>
nit: unnecessary '+'
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 259 (original), 252 (patched)
<https://reviews.apache.org/r/71949/#comment307241>
nit: unnecessary '+'
itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
Line 264 (original), 256 (patched)
<https://reviews.apache.org/r/71949/#comment307242>
'+' again
ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java
Lines 282 (patched)
<https://reviews.apache.org/r/71949/#comment307245>
I do not see this in the original code. What is this for?
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java
Lines 1228 (patched)
<https://reviews.apache.org/r/71949/#comment307246>
Do we need this? With delete_delta we do not supposed to have syntehetic rowIDs...
ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java
Lines 619 (patched)
<https://reviews.apache.org/r/71949/#comment307247>
nit: spaces...
- Peter Vary
On jan. 4, 2020, 9:06 de, Laszlo Pinter wrote:
>
> -----------------------------------------------------------
> This is an automatically generated e-mail. To reply, visit:
> https://reviews.apache.org/r/71949/
> -----------------------------------------------------------
>
> (Updated jan. 4, 2020, 9:06 de)
>
>
> Review request for hive, Denys Kuzmenko, Karen Coppage, and Peter Vary.
>
>
> Repository: hive-git
>
>
> Description
> -------
>
> HIVE-20934: ACID: Query based compactor for minor compaction
>
>
> Diffs
> -----
>
> itests/hive-unit/pom.xml bc20cd6168dd61222c75fb866deada26328986dd
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorTestUtil.java PRE-CREATION
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java 445e39c260edc68f511550271a7ac471fae908fe
> itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b7245e2c3570b362a00b65b23f3f84616d0a3d1e
> ql/src/java/org/apache/hadoop/hive/ql/exec/tez/SplitGrouper.java 33d723a02e28d69a69b88281038f69b5aecfe6a2
> ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java 3c508ec6cf620aee6a7791c6ab52c331ad5ec6bd
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java 2ac6232460fedb8351b5f0cfae2ce2d0f2e2d948
> ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java 0a96fc30b359043293017b235a36cd044ddb176e
> ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java 20b0ccd94b5f08aa2c1dace1301a8315bd202bf7
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java 2b2cc1a2ba8377aa3681b1a3454a0d64369eef64
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java 7a0e32463d28007cff5526ae037cc1447e50a50b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MajorQueryCompactor.java 38689ef86c607a36f8ec961a88578c13bfcd5b01
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MinorQueryCompactor.java PRE-CREATION
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MmMajorQueryCompactor.java 9b8420902fb688b218fa432d70f71302f9f180e6
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactor.java 1eab5b888deef2d0fb5c097941a1dafa51c7d46b
> ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/QueryCompactorFactory.java 41cb4b64fbc79dcf81919769c567b26a2e18cfe5
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForMmTable.java d4c9121c9f17f8d083f1e1af1caf840678a3559d
> ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommandsForOrcMmTable.java d6435342aa1f56ba5495a657b4a43327fdc49645
>
>
> Diff: https://reviews.apache.org/r/71949/diff/1/
>
>
> Testing
> -------
>
>
> Thanks,
>
> Laszlo Pinter
>
>