You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mh...@apache.org on 2018/11/07 07:15:55 UTC
[10/10] asterixdb git commit: [ASTERIXDB-2422][STO] Introduce
compressed storage
[ASTERIXDB-2422][STO] Introduce compressed storage
- user model changes: yes
- Add new configuration in the with-caluse to enable compression
- Add new nc configuration in the config file
- storage format changes: yes
- Pages of the primary index can be compressed
- Add a companion file (Look Aside File) with the compressed index
- Allow optional values in the LocalResource
- Add compression information in Metadata.Dataset
- interface changes: yes
- ICCApplicationContext:
- Add getCompressionManager()
- IBufferCache:
- Add getCompressedFileWriter(int fileId)
- ICachedPageInternal:
- Add setCompressedPageOffset(long offset)
- Add getCompressedPageOffset()
- Add setCompressedPageSize(int size)
- Add getCompressedPageSize()
Details:
- Add new integration test for this patch
- Fix ASTERIXDB-2464
- Add ddl-with-clause type validator
Additional details in the design document:
https://cwiki.apache.org/confluence/display/ASTERIXDB/Compression+in+AsterixDB
Change-Id: Idde6f37c810c30c7f1a5ee8bcbc1e3e5f4410031
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2857
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/5aeba9b4
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/5aeba9b4
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/5aeba9b4
Branch: refs/heads/master
Commit: 5aeba9b475fc714edf953bd88ada7281a2d4937e
Parents: 8076fe9
Author: Wail Alkowaileet <wa...@gmail.com>
Authored: Tue Nov 6 18:52:34 2018 -0800
Committer: Wail Alkowaileet <wa...@gmail.com>
Committed: Tue Nov 6 23:12:28 2018 -0800
----------------------------------------------------------------------
.../compression/incompressible_large_object.adm | 1 +
.../data/compression/incompressible_object.adm | 2 +
.../app/io/PersistedResourceRegistry.java | 28 ++
.../asterix/app/translator/QueryTranslator.java | 4 +-
.../SqlppExecutionWithCompresisionTest.java | 64 +++
.../src/test/resources/cc-compression.conf | 57 +++
.../large-page/large-page.1.ddl.sqlpp | 36 ++
.../large-page/large-page.2.update.sqlpp | 25 ++
.../large-page/large-page.3.query.sqlpp | 24 ++
.../small-page/small-page.1.ddl.sqlpp | 36 ++
.../small-page/small-page.2.update.sqlpp | 25 ++
.../small-page/small-page.3.query.sqlpp | 24 ++
.../invalid-compression-scheme.1.ddl.sqlpp | 36 ++
.../scheme-none/scheme-none.1.ddl.sqlpp | 42 ++
.../scheme-none/scheme-none.2.update.sqlpp | 25 ++
.../scheme-none/scheme-none.3.query.sqlpp | 27 ++
.../scheme-snappy/scheme-snappy.1.ddl.sqlpp | 43 ++
.../scheme-snappy/scheme-snappy.2.update.sqlpp | 25 ++
.../scheme-snappy/scheme-snappy.3.query.sqlpp | 27 ++
.../cross-dv02/cross-dv02.3.query.sqlpp | 14 +-
.../cross-dv04/cross-dv04.3.query.sqlpp | 14 +-
.../cross-dv19/cross-dv19.3.query.sqlpp | 15 +-
.../missing-non-optional.1.ddl.sqlpp | 41 ++
.../type-mismatch/type-mismatch.1.ddl.sqlpp | 42 ++
.../unsupported-field.1.ddl.sqlpp | 43 ++
.../unsupported-subfield.1.ddl.sqlpp | 43 ++
.../all_datasets_compressed.1.ddl.sqlpp | 66 +++
.../all_datasets_compressed.10.post.http | 20 +
.../all_datasets_compressed.11.get.http | 20 +
.../all_datasets_compressed.12.get.http | 20 +
.../all_datasets_compressed.13.query.sqlpp | 35 ++
.../all_datasets_compressed.14.query.sqlpp | 22 +
.../all_datasets_compressed.15.query.sqlpp | 22 +
.../all_datasets_compressed.16.query.sqlpp | 22 +
.../all_datasets_compressed.2.update.sqlpp | 24 ++
.../all_datasets_compressed.3.query.sqlpp | 35 ++
.../all_datasets_compressed.4.post.http | 20 +
.../all_datasets_compressed.5.get.http | 20 +
.../all_datasets_compressed.6.get.http | 20 +
.../all_datasets_compressed.7.query.sqlpp | 35 ++
.../all_datasets_compressed.8.query.sqlpp | 22 +
.../all_datasets_compressed.9.query.sqlpp | 22 +
.../single_dataset_compressed.1.ddl.sqlpp | 49 +++
.../single_dataset_compressed.10.query.sqlpp | 22 +
.../single_dataset_compressed.11.query.sqlpp | 22 +
.../single_dataset_compressed.2.update.sqlpp | 23 ++
.../single_dataset_compressed.3.query.sqlpp | 22 +
.../single_dataset_compressed.4.post.http | 20 +
.../single_dataset_compressed.5.get.http | 20 +
.../single_dataset_compressed.6.query.sqlpp | 22 +
.../single_dataset_compressed.7.query.sqlpp | 22 +
.../single_dataset_compressed.8.post.http | 20 +
.../single_dataset_compressed.9.get.http | 20 +
...le_dataset_with_index_compressed.1.ddl.sqlpp | 51 +++
...le_dataset_with_index_compressed.10.get.http | 20 +
...dataset_with_index_compressed.11.query.sqlpp | 25 ++
...dataset_with_index_compressed.12.query.sqlpp | 22 +
...dataset_with_index_compressed.13.query.sqlpp | 25 ++
...dataset_with_index_compressed.2.update.sqlpp | 23 ++
..._dataset_with_index_compressed.3.query.sqlpp | 25 ++
...le_dataset_with_index_compressed.4.post.http | 20 +
...gle_dataset_with_index_compressed.5.get.http | 20 +
..._dataset_with_index_compressed.6.query.sqlpp | 25 ++
..._dataset_with_index_compressed.7.query.sqlpp | 22 +
..._dataset_with_index_compressed.8.query.sqlpp | 25 ++
...le_dataset_with_index_compressed.9.post.http | 20 +
.../single_dataverse_compressed.1.ddl.sqlpp | 65 +++
.../single_dataverse_compressed.10.post.http | 20 +
.../single_dataverse_compressed.11.get.http | 20 +
.../single_dataverse_compressed.12.get.http | 20 +
.../single_dataverse_compressed.13.query.sqlpp | 36 ++
.../single_dataverse_compressed.14.query.sqlpp | 22 +
.../single_dataverse_compressed.15.query.sqlpp | 22 +
.../single_dataverse_compressed.2.update.sqlpp | 26 ++
.../single_dataverse_compressed.3.query.sqlpp | 36 ++
.../single_dataverse_compressed.4.post.http | 20 +
.../single_dataverse_compressed.5.get.http | 20 +
.../single_dataverse_compressed.6.get.http | 20 +
.../single_dataverse_compressed.7.query.sqlpp | 36 ++
.../single_dataverse_compressed.8.query.sqlpp | 22 +
.../single_dataverse_compressed.9.query.sqlpp | 22 +
.../bulkload.1.sto.cmd | 19 +
.../bulkload.10.post.http | 19 +
.../bulkload.11.pollget.http | 21 +
.../bulkload.12.query.sqlpp | 22 +
.../bulkload.13.sto.cmd | 19 +
.../bulkload.14.sto.cmd | 19 +
.../bulkload.2.sto.cmd | 19 +
.../bulkload.3.pollget.http | 21 +
.../bulkload.4.pollget.http | 21 +
.../bulkload.5.ddl.sqlpp | 48 +++
.../bulkload.6.update.sqlpp | 23 ++
.../bulkload.7.sto.cmd | 19 +
.../bulkload.8.sto.cmd | 19 +
.../bulkload.9.post.http | 19 +
.../flushed_component_compressed.1.sto.cmd | 19 +
.../flushed_component_compressed.10.post.http | 19 +
.../flushed_component_compressed.11.post.http | 19 +
...flushed_component_compressed.12.pollget.http | 21 +
.../flushed_component_compressed.13.query.sqlpp | 22 +
.../flushed_component_compressed.14.sto.cmd | 19 +
.../flushed_component_compressed.15.sto.cmd | 19 +
.../flushed_component_compressed.2.sto.cmd | 19 +
.../flushed_component_compressed.3.pollget.http | 21 +
.../flushed_component_compressed.4.pollget.http | 21 +
.../flushed_component_compressed.5.ddl.sqlpp | 57 +++
.../flushed_component_compressed.6.update.sqlpp | 22 +
.../flushed_component_compressed.7.sleep.sqlpp | 19 +
.../flushed_component_compressed.8.sto.cmd | 19 +
.../flushed_component_compressed.9.sto.cmd | 19 +
.../src/test/resources/runtimets/rebalance.xml | 21 +
.../test/resources/runtimets/replication.xml | 11 +
.../large-page/large-page.1.adm | 1 +
.../small-page/small-page.1.adm | 2 +
.../compression/scheme-none/scheme-none.1.adm | 100 +++++
.../scheme-snappy/scheme-snappy.1.adm | 100 +++++
.../cross-dataverse/cross-dv02/cross-dv02.3.ast | 68 +++-
.../cross-dataverse/cross-dv04/cross-dv04.3.ast | 68 +++-
.../cross-dataverse/cross-dv19/cross-dv19.3.ast | 73 +++-
.../resources/runtimets/testsuite_sqlpp.xml | 59 +++
.../common/config/StorageProperties.java | 10 +-
.../common/dataflow/ICcApplicationContext.java | 8 +-
.../asterix/common/exceptions/ErrorCode.java | 4 +-
.../common/storage/ICompressionManager.java | 49 +++
.../main/resources/asx_errormsg/en.properties | 10 +-
asterixdb/asterix-lang-common/pom.xml | 4 +
.../common/statement/CreateFeedStatement.java | 8 +-
.../lang/common/statement/DatasetDecl.java | 92 ++---
.../common/util/ConfigurationTypeValidator.java | 191 +++++++++
.../lang/common/util/ConfigurationUtil.java | 74 ++++
.../common/util/DatasetDeclParametersUtil.java | 97 +++++
.../lang/common/util/ExpressionUtils.java | 9 +-
.../lang/common/util/MergePolicyUtils.java | 67 ----
.../metadata/bootstrap/MetadataBootstrap.java | 4 +-
.../metadata/bootstrap/MetadataRecordTypes.java | 5 +
.../metadata/dataset/hints/DatasetHints.java | 3 +
.../declared/BTreeResourceFactoryProvider.java | 13 +-
.../metadata/declared/MetadataProvider.java | 5 +
.../asterix/metadata/entities/Dataset.java | 28 +-
.../DatasetTupleTranslator.java | 73 +++-
.../DatasetTupleTranslatorTest.java | 8 +-
.../IndexTupleTranslatorTest.java | 8 +-
.../asterix/object/base/AdmObjectNode.java | 8 +
.../runtime/compression/CompressionManager.java | 128 ++++++
.../runtime/utils/CcApplicationContext.java | 12 +-
.../compression/ICompressorDecompressor.java | 64 +++
.../ICompressorDecompressorFactory.java | 39 ++
.../apache/hyracks/api/io/FileReference.java | 12 +-
.../api/io/IPersistedResourceRegistry.java | 12 +
.../lsm/btree/LSMBTreeOperatorTestHelper.java | 3 +-
.../hyracks/storage/am/btree/impls/BTree.java | 8 +-
.../AppendOnlyLinkedMetadataPageManager.java | 7 +-
.../am/common/impls/AbstractTreeIndex.java | 16 +-
.../dataflow/ExternalBTreeLocalResource.java | 6 +-
.../ExternalBTreeLocalResourceFactory.java | 3 +-
.../ExternalBTreeWithBuddyLocalResource.java | 6 +-
...ernalBTreeWithBuddyLocalResourceFactory.java | 3 +-
.../btree/dataflow/LSMBTreeLocalResource.java | 20 +-
.../dataflow/LSMBTreeLocalResourceFactory.java | 8 +-
.../lsm/btree/impls/LSMBTreeDiskComponent.java | 9 +-
.../am/lsm/btree/impls/LSMBTreeFileManager.java | 20 +-
.../am/lsm/btree/utils/LSMBTreeUtil.java | 11 +-
.../impls/AbstractLSMIndexFileManager.java | 34 +-
.../hyracks/storage/am/rtree/impls/RTree.java | 9 +-
.../hyracks/hyracks-storage-common/pom.xml | 5 +
.../AbstractBufferedFileIOManager.java | 287 +++++++++++++
.../storage/common/buffercache/BufferCache.java | 238 ++++-------
.../buffercache/BufferCacheHeaderHelper.java | 90 +++++
.../storage/common/buffercache/CachedPage.java | 21 +
.../common/buffercache/IBufferCache.java | 8 +
.../common/buffercache/ICachedPageInternal.java | 8 +
.../compression/NoOpCompressorDecompressor.java | 46 +++
.../NoOpCompressorDecompressorFactory.java | 47 +++
.../SnappyCompressorDecompressor.java | 66 +++
.../SnappyCompressorDecompressorFactory.java | 48 +++
.../compression/file/CompressedFileManager.java | 285 +++++++++++++
.../file/CompressedFileReference.java | 86 ++++
.../compression/file/ICompressedPageWriter.java | 48 +++
.../common/compression/file/LAFWriter.java | 260 ++++++++++++
.../common/compression/file/NoOpLAFWriter.java | 44 ++
.../storage/common/file/BufferedFileHandle.java | 133 +++++-
.../file/CompressedBufferedFileHandle.java | 261 ++++++++++++
.../am/lsm/btree/LSMBTreeExamplesTest.java | 4 +-
...MBTreeModificationOperationCallbackTest.java | 4 +-
.../LSMBTreeSearchOperationCallbackTest.java | 4 +-
.../am/lsm/btree/LSMBTreeUpdateInPlaceTest.java | 4 +-
.../btree/impl/TestLsmBtreeLocalResource.java | 6 +-
.../impl/TestLsmBtreeLocalResourceFactory.java | 3 +-
.../multithread/LSMBTreeMultiThreadTest.java | 4 +-
.../am/lsm/btree/perf/LSMTreeRunner.java | 5 +-
.../am/lsm/btree/util/LSMBTreeTestContext.java | 10 +-
.../common/BufferCacheWithCompressionTest.java | 401 +++++++++++++++++++
192 files changed, 6536 insertions(+), 424 deletions(-)
----------------------------------------------------------------------