You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2016/05/13 02:41:02 UTC
[12/12] incubator-asterixdb git commit: ASTERIXDB-1436: Big Object
Support For Storage
ASTERIXDB-1436: Big Object Support For Storage
Preliminary submit for big object storage support (values only).
Enable hyracks frames up to Integer.MAX_VALUE
Change-Id: Ie0a3cb855768dfd0cd1da4a5fc9f62aedebcbc5f
Reviewed-on: https://asterix-gerrit.ics.uci.edu/840
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/1defc92a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/1defc92a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/1defc92a
Branch: refs/heads/master
Commit: 1defc92ae43cb36e1c06eb884c5359d5c1d6b0e5
Parents: a98d0de
Author: Michael Blow <mb...@apache.org>
Authored: Thu May 12 18:31:47 2016 -0700
Committer: Michael Blow <mi...@couchbase.com>
Committed: Thu May 12 19:40:40 2016 -0700
----------------------------------------------------------------------
.../big_object_bulkload.1.ddl.aql | 35 +++
.../big_object_bulkload.2.update.aql | 119 +++++++++
.../big_object_bulkload.3.query.aql | 23 ++
.../big_object_insert.1.ddl.aql | 35 +++
.../big_object_insert.2.update.aql | 119 +++++++++
.../big_object_insert.3.query.aql | 23 ++
.../big_object_bulkload.1.adm | 100 +++++++
.../big_object_insert/big_object_insert.1.adm | 100 +++++++
.../feed-with-external-parser.1.adm | 3 +-
.../feed-with-multiple-indexes.1.adm | 3 +-
.../src/test/resources/runtimets/testsuite.xml | 10 +
.../asterix/common/transactions/ILogRecord.java | 9 +-
.../asterix/common/transactions/LogRecord.java | 45 ++--
.../apache/asterix/test/aql/TestExecutor.java | 3 +
.../dataflow/FeedRecordDataFlowController.java | 5 -
.../external/dataflow/FeedTupleForwarder.java | 16 --
.../external/util/ExternalDataConstants.java | 1 -
.../apache/asterix/external/util/FeedUtils.java | 2 +-
.../aql/statement/SubscribeFeedStatement.java | 2 +-
.../logging/ReplicationLogBuffer.java | 8 +
.../logging/ReplicationLogFlusher.java | 5 +-
.../management/ReplicationManager.java | 18 +-
.../management/service/logging/LogBuffer.java | 4 +
.../service/logging/LogBufferTailReader.java | 6 +-
.../management/service/logging/LogManager.java | 22 +-
.../logging/LogManagerWithReplication.java | 6 +-
.../management/service/logging/LogReader.java | 93 ++++---
.../service/logging/RemoteLogReader.java | 57 ++--
hyracks-fullstack/hyracks/hyracks-api/pom.xml | 5 +
.../apache/hyracks/api/comm/FrameConstants.java | 12 +-
.../apache/hyracks/api/comm/FrameHelper.java | 16 +-
.../nc/resources/memory/FrameManager.java | 18 +-
.../common/comm/io/AbstractFrameAppender.java | 2 +-
.../common/comm/io/FrameFixedFieldAppender.java | 2 +-
.../common/comm/io/FrameTupleAccessor.java | 2 +-
.../common/comm/io/FrameTupleAppender.java | 2 +-
.../comm/io/FrameTupleAppenderAccessor.java | 2 +-
.../common/io/MessagingFrameTupleAppender.java | 2 +-
.../dataflow/common/util/IntSerDeUtils.java | 42 ---
.../dataflow/common/util/ReflectionUtils.java | 33 ---
.../VariableFrameMemoryManager.java | 2 +-
.../std/sort/AbstractExternalSortRunMerger.java | 2 +-
.../dataflow/std/sort/AbstractFrameSorter.java | 2 +-
.../sort/util/DeletableFrameTupleAppender.java | 2 +-
.../AbstractTupleMemoryManagerTest.java | 2 +-
.../VPartitionTupleBufferManagerTest.java | 2 +-
.../VariableTupleMemoryManagerTest.java | 2 +-
.../util/DeletableFrameTupleAppenderTest.java | 2 +-
.../storage/am/btree/api/IBTreeFrame.java | 2 +
.../storage/am/btree/api/IBTreeLeafFrame.java | 14 +-
.../frames/BTreeFieldPrefixNSMLeafFrame.java | 37 ++-
.../BTreeFieldPrefixNSMLeafFrameFactory.java | 6 +
.../am/btree/frames/BTreeLargeFrameHelper.java | 44 ++++
.../am/btree/frames/BTreeNSMInteriorFrame.java | 50 ++--
.../frames/BTreeNSMInteriorFrameFactory.java | 6 +
.../am/btree/frames/BTreeNSMLeafFrame.java | 189 +++++++++++---
.../btree/frames/BTreeNSMLeafFrameFactory.java | 8 +-
.../hyracks/storage/am/btree/impls/BTree.java | 258 ++++++++++++-------
.../btree/impls/BTreeCountingSearchCursor.java | 5 +-
.../am/btree/impls/BTreeRangeSearchCursor.java | 4 +-
.../storage/am/btree/util/BTreeUtils.java | 3 +
.../am/common/api/IMetaDataPageManager.java | 15 +-
.../storage/am/common/api/ITreeIndexFrame.java | 8 +-
.../am/common/api/ITreeIndexFrameFactory.java | 7 +-
.../am/common/frames/FrameOpSpaceStatus.java | 7 +-
.../am/common/frames/TreeIndexNSMFrame.java | 58 ++++-
.../freepage/LinkedMetaDataPageManager.java | 15 ++
.../am/common/impls/AbstractTreeIndex.java | 34 ++-
.../impls/TreeIndexDiskOrderScanCursor.java | 3 +-
.../am/common/util/TreeIndexStatsGatherer.java | 2 +-
.../freepage/VirtualMetaDataPageManager.java | 12 +-
.../impls/MultitenantVirtualBufferCache.java | 16 ++
.../am/lsm/common/impls/VirtualBufferCache.java | 60 ++++-
.../storage/am/rtree/frames/RTreeNSMFrame.java | 20 +-
.../am/rtree/frames/RTreeNSMInteriorFrame.java | 5 +-
.../frames/RTreeNSMInteriorFrameFactory.java | 10 +-
.../am/rtree/frames/RTreeNSMLeafFrame.java | 2 +-
.../rtree/frames/RTreeNSMLeafFrameFactory.java | 10 +-
.../hyracks/storage/am/rtree/impls/RTree.java | 14 +-
.../buffercache/AsyncFIFOPageQueueManager.java | 37 +--
.../storage/common/buffercache/BufferCache.java | 182 +++++++++++--
.../storage/common/buffercache/CachedPage.java | 12 +-
.../ClockPageReplacementStrategy.java | 136 +++++++---
.../common/buffercache/DebugBufferCache.java | 17 ++
.../buffercache/DirectBufferAllocator.java | 4 +
.../common/buffercache/FIFOLocalWriter.java | 1 -
.../common/buffercache/HeapBufferAllocator.java | 4 +
.../common/buffercache/IBufferCache.java | 6 +-
.../buffercache/IBufferCacheInternal.java | 2 +
.../buffercache/ICacheMemoryAllocator.java | 11 +
.../common/buffercache/ILargePageHelper.java | 25 ++
.../buffercache/IPageReplacementStrategy.java | 6 +
.../ResourceHeapBufferAllocator.java | 7 +-
.../storage/common/file/BufferedFileHandle.java | 4 +-
.../am/btree/OrderedIndexExamplesTest.java | 8 +-
.../storage/am/btree/FieldPrefixNSMTest.java | 7 +-
.../storage/am/btree/StorageManagerTest.java | 4 +-
.../org/apache/hyracks/util/IntSerDeUtils.java | 42 +++
.../apache/hyracks/util/ReflectionUtils.java | 33 +++
99 files changed, 1965 insertions(+), 533 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql
new file mode 100644
index 0000000..6670a05
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/big-object/big_object_bulkload/big_object_bulkload.1.ddl.aql
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : big_object_bulkload.aql
+ * Description : bulkload insert of large objects
+ * Expected Result : Success
+ * Date : 20th April 2016
+ */
+
+drop dataverse testdv2 if exists;
+create dataverse testdv2;
+use dataverse testdv2;
+
+create type testtype as open {
+ id: int64,
+ name: string
+}
+
+create dataset testds(testtype) primary key id;