You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ji...@apache.org on 2016/02/26 06:54:01 UTC
[04/11] incubator-asterixdb-hyracks git commit: Implemented the
memory-bounded HashGroupby and HashJoin for BigObject
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java
new file mode 100644
index 0000000..9d4a9a1
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.buffermanager;
+
+import static org.apache.hyracks.dataflow.std.buffermanager.Common.BUDGET;
+import static org.apache.hyracks.dataflow.std.buffermanager.Common.MIN_FRAME_SIZE;
+import static org.apache.hyracks.dataflow.std.buffermanager.Common.NUM_MIN_FRAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hyracks.api.comm.FixedSizeFrame;
+import org.apache.hyracks.api.comm.FrameHelper;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameTupleAppender;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+
+public class VariableFramesMemoryManagerTest {
+ VariableFrameMemoryManager framesMemoryManager;
+ FrameTupleAccessor fta;
+ Random random;
+ List<IFrame> frameList;
+
+ @Before
+ public void setUp() throws Exception {
+ VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, BUDGET);
+ FrameFreeSlotLastFit policy = new FrameFreeSlotLastFit(NUM_MIN_FRAME);
+ framesMemoryManager = new VariableFrameMemoryManager(framePool, policy);
+ RecordDescriptor recordDescriptor = new RecordDescriptor(new ISerializerDeserializer[] { null });
+ fta = new FrameTupleAccessor(recordDescriptor);
+ random = new Random(System.currentTimeMillis());
+ frameList = new ArrayList<>();
+ }
+
+ @Test
+ public void testNormalIncomingFrames() throws HyracksDataException {
+ HashMap<Integer, Integer> tupleSet = prepareTuples();
+ for (IFrame frame : frameList) {
+ assertTrue(framesMemoryManager.insertFrame(frame.getBuffer()) >= 0);
+ }
+ assertEquals(NUM_MIN_FRAME, framesMemoryManager.getNumFrames());
+ assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
+ }
+
+ @Test
+ public void testRandomTuplesAreAllStoredInBuffer() throws HyracksDataException {
+ Map<Integer, Integer> tupleSet = prepareRandomTuples();
+ for (IFrame frame : frameList) {
+ if (framesMemoryManager.insertFrame(frame.getBuffer()) < 0) {
+ fta.reset(frame.getBuffer());
+ for (int i = 0; i < fta.getTupleCount(); ++i) {
+ int id = parseTuple(fta.getBuffer(),
+ fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 0) + fta.getFieldSlotsLength());
+ tupleSet.remove(id);
+ // System.out.println(
+ // "can't appended id:" + id + ",frameSize:" + frame.getInitialFrameSize());
+ }
+ }
+ }
+ assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
+ framesMemoryManager.reset();
+ }
+
+ @Test
+ public void testResetShouldWork() throws HyracksDataException {
+ testNormalIncomingFrames();
+ framesMemoryManager.reset();
+ testRandomTuplesAreAllStoredInBuffer();
+ framesMemoryManager.reset();
+ testRandomTuplesAreAllStoredInBuffer();
+ }
+
+ @Test
+ public void testCloseShouldAlsoWork() throws HyracksDataException {
+ testRandomTuplesAreAllStoredInBuffer();
+ framesMemoryManager.close();
+ testRandomTuplesAreAllStoredInBuffer();
+ framesMemoryManager.close();
+ testRandomTuplesAreAllStoredInBuffer();
+ }
+
+ private HashMap<Integer, Integer> prepareRandomTuples() throws HyracksDataException {
+ frameList.clear();
+ HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
+ int[] fieldSlot = { 0 };
+ int id = 0;
+ int size = 0;
+ while (size < BUDGET) {
+ int tupleLength = random.nextInt(BUDGET / 3) + 4;
+ IFrame frame = new FixedSizeFrame(Common.commonFrameManager
+ .allocateFrame(FrameHelper.calcAlignedFrameSizeToStore(1, tupleLength, MIN_FRAME_SIZE)));
+ IFrameTupleAppender appender = new FrameTupleAppender();
+ appender.reset(frame, true);
+ // System.out.println("id:" + id + ",frameSize:" + frame.getInitialFrameSize() / MIN_FRAME_SIZE);
+ ByteBuffer buffer = ByteBuffer.allocate(tupleLength);
+ buffer.putInt(0, id);
+ assertTrue(appender.append(fieldSlot, buffer.array(), 0, buffer.capacity()));
+ set.put(id++, tupleLength);
+ size += frame.getFrameSize();
+ frameList.add(frame);
+ }
+ return set;
+ }
+
+ private HashMap<Integer, Integer> prepareTuples() throws HyracksDataException {
+ frameList.clear();
+ HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
+ for (int i = 0; i < NUM_MIN_FRAME; ++i) {
+ IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(MIN_FRAME_SIZE));
+ IFrameTupleAppender appender = new FrameTupleAppender();
+ appender.reset(frame, true);
+
+ int[] fieldSlot = { 0 };
+ ByteBuffer buffer = ByteBuffer.allocate(MIN_FRAME_SIZE / 2);
+ buffer.putInt(0, i);
+ appender.append(fieldSlot, buffer.array(), 0, buffer.capacity());
+ set.put(i, buffer.capacity());
+ frameList.add(frame);
+ }
+ return set;
+ }
+
+ private void assertEveryTupleInFTAIsInFrameMemoryManager(Map<Integer, Integer> tupleSet,
+ VariableFrameMemoryManager framesMemoryManager) {
+ BufferInfo info = new BufferInfo(null, -1, -1);
+ for (int i = 0; i < framesMemoryManager.getNumFrames(); ++i) {
+ framesMemoryManager.getFrame(i, info);
+ fta.reset(info.getBuffer(), info.getStartOffset(), info.getLength());
+ for (int t = 0; t < fta.getTupleCount(); t++) {
+ int id = parseTuple(fta.getBuffer(), fta.getTupleStartOffset(t) + fta.getFieldSlotsLength() + fta
+ .getFieldStartOffset(t, 0));
+ // System.out.println("frameid:" + i + ",tuple:" + t + ",has id:" + id + ",length:" +
+ // (fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta.getFieldSlotsLength()));
+ assertTrue(tupleSet.remove(id) == fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta
+ .getFieldSlotsLength());
+ }
+ }
+ assertTrue(tupleSet.isEmpty());
+ }
+
+ private int parseTuple(ByteBuffer buffer, int fieldStartOffset) {
+ return buffer.getInt(fieldStartOffset);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java
new file mode 100644
index 0000000..036cd41
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.buffermanager;
+
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hyracks.api.comm.FixedSizeFrame;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.util.IntSerDeUtils;
+import org.apache.hyracks.dataflow.std.structures.TuplePointer;
+import org.junit.Before;
+import org.junit.Test;
+
+public class VariableTupleMemoryManagerTest extends AbstractTupleMemoryManagerTest {
+ VariableDeletableTupleMemoryManager tupleMemoryManager;
+ final int EXTRA_BYTES_FOR_DELETABLE_FRAME = 4;
+
+ @Before
+ public void setup() {
+ VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, Common.BUDGET);
+ tupleMemoryManager = new VariableDeletableTupleMemoryManager(framePool, recordDescriptor);
+ }
+
+ @Test
+ public void testInsertTupleToMemoryManager() throws HyracksDataException {
+ int iTuplePerFrame = 3;
+ Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame, EXTRA_BYTES_FOR_DELETABLE_FRAME, 0);
+ Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
+ assertEachTupleInFTAIsInBuffer(mapPrepare, mapInserted);
+ }
+
+ @Test
+ public void testReset() throws HyracksDataException {
+ testInsertVariableSizeTupleToMemoryManager();
+ tupleMemoryManager.reset();
+ testInsertTupleToMemoryManager();
+ tupleMemoryManager.reset();
+ testInsertVariableSizeTupleToMemoryManager();
+ }
+
+ @Test
+ public void testDeleteTupleInMemoryManager() throws HyracksDataException {
+ int iTuplePerFrame = 3;
+ Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame, EXTRA_BYTES_FOR_DELETABLE_FRAME, 0);
+ Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
+ deleteRandomSelectedTuples(mapPrepare, mapInserted, 1);
+ assertEachTupleInFTAIsInBuffer(mapPrepare, mapInserted);
+ }
+
+ @Test
+ public void testReOrganizeSpace() throws HyracksDataException {
+ int iTuplePerFrame = 3;
+ Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame, EXTRA_BYTES_FOR_DELETABLE_FRAME, 0);
+ Map<Integer, Integer> copyMap = new HashMap<>(mapPrepare);
+ Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
+ ByteBuffer buffer = deleteRandomSelectedTuples(mapPrepare, mapInserted, mapPrepare.size() / 2);
+ inFTA.reset(buffer);
+ //The deletable frame buffer will keep the deleted slot untouched, which will take more space.
+ // the reason is to not reuse the same TuplePointer outside.
+ Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferMayNotAllSuccess();
+ assertTrue(mapInserted2.size() > 0);
+ }
+
+ @Test
+ public void testReOrganizeVariableSizeTuple() throws HyracksDataException {
+ Map<Integer, Integer> map = prepareVariableSizeTuples();
+ Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
+ Map<Integer, Integer> copyMap = new HashMap<>(map);
+
+ ByteBuffer buffer = deleteRandomSelectedTuples(map, mapInserted, map.size() / 2);
+ inFTA.reset(buffer);
+
+ Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferCouldFailForLargerTuples(copyMap);
+ Map<TuplePointer, Integer> mergedMap = new HashMap<>(mapInserted);
+ mergedMap.putAll(mapInserted2);
+
+ assertEachTupleInFTAIsInBuffer(copyMap, mergedMap);
+ }
+
+ @Test
+ public void testInsertVariableSizeTupleToMemoryManager() throws HyracksDataException {
+ Map<Integer, Integer> map = prepareVariableSizeTuples();
+ Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
+ assertEachTupleInFTAIsInBuffer(map, mapInserted);
+ }
+
+ @Override
+ ITuplePointerAccessor getTupleAccessor() {
+ return tupleMemoryManager.createTupleAccessor();
+ }
+
+ private Map<TuplePointer, Integer> insertInFTAToBufferShouldAllSuccess() throws HyracksDataException {
+ Map<TuplePointer, Integer> tuplePointerIntegerMap = new HashMap<>();
+ for (int i = 0; i < inFTA.getTupleCount(); i++) {
+ TuplePointer tuplePointer = new TuplePointer();
+ assertTrue(tupleMemoryManager.insertTuple(inFTA, i, tuplePointer));
+ tuplePointerIntegerMap.put(tuplePointer,
+ IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+ }
+ return tuplePointerIntegerMap;
+ }
+
+ private Map<TuplePointer, Integer> insertInFTAToBufferMayNotAllSuccess() throws HyracksDataException {
+ Map<TuplePointer, Integer> tuplePointerIntegerMap = new HashMap<>();
+ for (int i = 0; i < inFTA.getTupleCount(); i++) {
+ TuplePointer tuplePointer = new TuplePointer();
+ if (!tupleMemoryManager.insertTuple(inFTA, i, tuplePointer)) {
+ break;
+ }
+ tuplePointerIntegerMap.put(tuplePointer,
+ IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+ }
+ return tuplePointerIntegerMap;
+ }
+
+ private Map<TuplePointer, Integer> insertInFTAToBufferCouldFailForLargerTuples(Map<Integer, Integer> map)
+ throws HyracksDataException {
+ Map<TuplePointer, Integer> tuplePointerIdMap = new HashMap<>();
+ int i = 0;
+ for (; i < inFTA.getTupleCount(); i++) {
+ TuplePointer tuplePointer = new TuplePointer();
+ if (!tupleMemoryManager.insertTuple(inFTA, i, tuplePointer)) {
+ break;
+ }
+ tuplePointerIdMap.put(tuplePointer,
+ IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+ }
+ for (; i < inFTA.getTupleCount(); i++) {
+ map.remove(IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+ }
+ return tuplePointerIdMap;
+ }
+
+ private ByteBuffer deleteRandomSelectedTuples(Map<Integer, Integer> map, Map<TuplePointer, Integer> mapInserted,
+ int minNumOfRecordTobeDeleted)
+ throws HyracksDataException {
+ ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
+ FixedSizeFrame frame = new FixedSizeFrame(buffer);
+ FrameTupleAppender appender = new FrameTupleAppender();
+ appender.reset(frame, true);
+
+ assert (minNumOfRecordTobeDeleted < mapInserted.size());
+ int countDeleted = minNumOfRecordTobeDeleted + random.nextInt(mapInserted.size() - minNumOfRecordTobeDeleted);
+
+ ITuplePointerAccessor accessor = tupleMemoryManager.createTupleAccessor();
+ for (int i = 0; i < countDeleted; i++) {
+ Iterator<Map.Entry<TuplePointer, Integer>> iter = mapInserted.entrySet().iterator();
+ assert (iter.hasNext());
+ Map.Entry<TuplePointer, Integer> pair = iter.next();
+ accessor.reset(pair.getKey());
+ appender.append(accessor.getBuffer().array(), accessor.getTupleStartOffset(), accessor.getTupleLength());
+ map.remove(pair.getValue());
+ tupleMemoryManager.deleteTuple(pair.getKey());
+ iter.remove();
+ }
+ return buffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java
deleted file mode 100644
index a6f10a6..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-
-public class Common {
- static int MIN_FRAME_SIZE = 256;
- static int NUM_MIN_FRAME = 15;
- static int BUDGET = NUM_MIN_FRAME * MIN_FRAME_SIZE;
-
- static FrameManager commonFrameManager = new FrameManager(MIN_FRAME_SIZE);
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java
deleted file mode 100644
index 58dd7e8..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class FrameFreeSlotBestFitUsingTreeMapTest {
-
- static int size = 10;
-
- FrameFreeSlotSmallestFit policy;
-
- @Before
- public void intial() {
- policy = new FrameFreeSlotSmallestFit();
- }
-
- @Test
- public void testAll() {
-
- for (int i = 0; i < size; i++) {
- policy.pushNewFrame(i, i);
- assertEquals(i, policy.popBestFit(i));
- }
- assertEquals(-1, policy.popBestFit(0));
-
- for (int i = 0; i < size; i++) {
- policy.pushNewFrame(i, i);
- }
- for (int i = 0; i < size; i++) {
- assertEquals(i, policy.popBestFit(i));
- }
-
- }
-
- @Test
- public void testReset(){
- testAll();
- policy.reset();
- testAll();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java
deleted file mode 100644
index 73eda47..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static junit.framework.Assert.assertEquals;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class FrameFreeSlotBiggestFirstTest {
-
- static int size = 10;
-
- FrameFreeSlotBiggestFirst policy;
-
- @Before
- public void intial() {
- policy = new FrameFreeSlotBiggestFirst(size);
- }
-
- @Test
- public void testAll() {
-
- for (int i = 0; i < size; i++) {
- policy.pushNewFrame(i, i);
- assertEquals(i, policy.popBestFit(i));
- }
- assertEquals(-1, policy.popBestFit(0));
-
- for (int i = 0; i < size; i++) {
- policy.pushNewFrame(i, i);
- }
- for (int i = 0; i < size; i++) {
- assertEquals(size - i - 1, policy.popBestFit(0));
- }
-
- for (int i = 0; i < size; i++) {
- policy.pushNewFrame(i, i);
- }
- for (int i = 0; i < size / 2; i++) {
- assertEquals(size - i - 1, policy.popBestFit(size / 2));
- }
- assertEquals(-1, policy.popBestFit(size / 2));
- for (int i = 0; i < size / 2; i++) {
- assertEquals(size / 2 - i - 1, policy.popBestFit(0));
- }
-
- }
-
- @Test
- public void testReset() {
- testAll();
- policy.reset();
- testAll();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java
deleted file mode 100644
index 16338fd..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class FrameFreeSlotLastFitTest {
-
- FrameFreeSlotLastFit zeroPolicy;
- FrameFreeSlotLastFit unifiedPolicy;
- FrameFreeSlotLastFit ascPolicy;
- FrameFreeSlotLastFit dscPolicy;
-
- static final int size = 10;
- static final int medium = 5;
-
- @Before
- public void setUp() throws Exception {
- zeroPolicy = new FrameFreeSlotLastFit(0);
- unifiedPolicy = new FrameFreeSlotLastFit(size);
- ascPolicy = new FrameFreeSlotLastFit(size);
- dscPolicy = new FrameFreeSlotLastFit(size);
- }
-
- @Test
- public void testPushAndPop() throws Exception {
- for (int i = 0; i < size; i++) {
- unifiedPolicy.pushNewFrame(i, medium);
- }
- for (int i = 0; i < size; i++) {
- assertTrue(unifiedPolicy.popBestFit(medium) == size - i - 1);
- }
- assertTrue(unifiedPolicy.popBestFit(0) == -1);
-
- for (int i = 0; i < size / 2; i++) {
- ascPolicy.pushNewFrame(i, i);
- assertEquals(ascPolicy.popBestFit(medium), -1);
- dscPolicy.pushNewFrame(i, size - i - 1);
- assertEquals(dscPolicy.popBestFit(medium), i);
- }
-
- for (int i = size / 2; i < size; i++) {
- ascPolicy.pushNewFrame(i, i);
- assertEquals(ascPolicy.popBestFit(medium), i);
- dscPolicy.pushNewFrame(i, size - i - 1);
- assertEquals(dscPolicy.popBestFit(medium), -1);
- }
-
- ascPolicy.reset();
- for (int i = 0; i < size; i++) {
- ascPolicy.pushNewFrame(size - i, size - i);
- }
-
- for (int i = 0; i < size; i++) {
- assertEquals(size - i, ascPolicy.popBestFit(size - i));
- }
- }
-
- @Test
- public void testReset() throws Exception {
- testPushAndPop();
-
- zeroPolicy.reset();
- unifiedPolicy.reset();
- ascPolicy.reset();
- dscPolicy.reset();
- testPushAndPop();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java
deleted file mode 100644
index e14c685..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.BUDGET;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.MIN_FRAME_SIZE;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.NUM_MIN_FRAME;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.commonFrameManager;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class VariableFramePoolTest {
-
- VariableFramePool pool;
- @Before
- public void setUp() throws Exception {
-
- pool = new VariableFramePool(commonFrameManager, BUDGET);
- }
-
- @Test
- public void testGetMinFrameSize() throws Exception {
- assertEquals(MIN_FRAME_SIZE, commonFrameManager.getInitialFrameSize());
- assertEquals(MIN_FRAME_SIZE, pool.getMinFrameSize());
- }
-
- @Test
- public void testGetMemoryBudgetBytes() throws Exception {
- assertEquals(BUDGET, pool.getMemoryBudgetBytes());
- }
-
- @Test
- public void testAllocateUniformFrameShouldSuccess() throws Exception {
- testAllocateAllSpacesWithMinFrames();
- testAllocateShouldFailAfterAllSpaceGetUsed();
- pool.reset();
- testAllocateAllSpacesWithMinFrames();
- pool.close();
- }
-
- @Test
- public void testResetShouldReuseExistingFrames() throws HyracksDataException {
- Set<?> set1 = testAllocateAllSpacesWithMinFrames();
- pool.reset();
- Set<?> set2 = testAllocateAllSpacesWithMinFrames();
- assertEquals(set1, set2);
- pool.close();
- }
-
- @Test
- public void testCloseShouldNotReuseExistingFrames() throws HyracksDataException {
- Set<?> set1 = testAllocateAllSpacesWithMinFrames();
- pool.close();
- Set<?> set2 = testAllocateAllSpacesWithMinFrames();
- assertFalse(set1.equals(set2));
- pool.close();
- }
-
- @Test
- public void testShouldReturnLargerFramesIfFitOneIsUsed() throws HyracksDataException {
- Set<?> set = testAllocateVariableFrames();
- pool.reset();
- testShouldFindTheMatchFrames(set);
- pool.reset();
-
- // allocate seq: 1, 1, 2, 3, 4
- ByteBuffer placeBuffer = pool.allocateFrame(MIN_FRAME_SIZE);
- assertTrue(set.contains(new ByteBufferPtr(placeBuffer)));
- for (int i = 1; i <= 4 ; i++) {
- ByteBuffer buffer = pool.allocateFrame(i * MIN_FRAME_SIZE);
- assertNotNull(buffer);
- assertTrue(set.contains(new ByteBufferPtr(buffer)));
- }
- assertNull(pool.allocateFrame(MIN_FRAME_SIZE));
- pool.close();
- }
-
- @Test
- public void testShouldMergeIfNoLargerFrames() throws HyracksDataException {
- Set<?> set = testAllocateAllSpacesWithMinFrames();
- pool.reset();
- int chunks = 5;
- for (int i = 0; i < NUM_MIN_FRAME; i+= chunks) {
- ByteBuffer buffer = pool.allocateFrame(chunks * MIN_FRAME_SIZE);
- assertNotNull(buffer);
- assertTrue(!set.contains(new ByteBufferPtr(buffer)));
- }
- }
-
- @Test
- public void testUseMiddleSizeFrameAndNeedToMergeSmallAndBigger() throws HyracksDataException {
- Set<?> set = testAllocateVariableFrames();
- pool.reset();
- // allocate seq: 3, 6, 1;
- ByteBuffer buffer = pool.allocateFrame(3 * MIN_FRAME_SIZE);
- assertTrue(set.contains(new ByteBufferPtr(buffer)));
- buffer = pool.allocateFrame(6 * MIN_FRAME_SIZE);
- assertFalse(set.contains(new ByteBufferPtr(buffer)));
- buffer = pool.allocateFrame(1 * MIN_FRAME_SIZE);
- assertTrue(set.contains(new ByteBufferPtr(buffer)));
- assertEquals(5 * MIN_FRAME_SIZE, buffer.capacity());
- pool.reset();
- }
-
- private void testAllocateShouldFailAfterAllSpaceGetUsed() throws HyracksDataException {
- for (int i = 0; i < NUM_MIN_FRAME; i++) {
- assertNull(pool.allocateFrame(MIN_FRAME_SIZE));
- }
- }
-
- private HashSet<ByteBufferPtr> testAllocateAllSpacesWithMinFrames() throws HyracksDataException {
- HashSet<ByteBufferPtr> set = new HashSet<>();
- for (int i = 0; i < NUM_MIN_FRAME; i++) {
- ByteBuffer buffer = pool.allocateFrame(MIN_FRAME_SIZE);
- assertNotNull(buffer);
- assertTrue(!set.contains(new ByteBufferPtr(buffer)));
- set.add(new ByteBufferPtr(buffer));
- }
- return set;
- }
-
- /**
- * Pool will become 1,2,3,4,5
- *
- * @throws HyracksDataException
- */
- private Set<ByteBufferPtr> testAllocateVariableFrames() throws HyracksDataException {
- int budget = BUDGET;
- int allocate = 0;
- int i = 1;
- Set<ByteBufferPtr> set = new HashSet<>();
- while (budget - allocate >= i * MIN_FRAME_SIZE) {
- ByteBuffer buffer = pool.allocateFrame(i * MIN_FRAME_SIZE);
- assertNotNull(buffer);
- set.add(new ByteBufferPtr(buffer));
- allocate += i++ * MIN_FRAME_SIZE;
- }
- return set;
- }
-
- private void testShouldFindTheMatchFrames(Set<?> set) throws HyracksDataException {
- pool.reset();
- List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
-
- for (int i = 0; i < list.size(); i++) {
- ByteBuffer buffer = pool.allocateFrame(list.get(i) * MIN_FRAME_SIZE);
- assertNotNull(buffer);
- assertTrue(set.contains(new ByteBufferPtr(buffer)));
- assertEquals(list.get(i) * MIN_FRAME_SIZE, buffer.capacity());
- }
- pool.reset();
- for (int i = list.size() - 1; i >= 0; i--) {
- ByteBuffer buffer = pool.allocateFrame(list.get(i) * MIN_FRAME_SIZE);
- assertNotNull(buffer);
- assertTrue(set.contains(new ByteBufferPtr(buffer)));
- assertEquals(list.get(i) * MIN_FRAME_SIZE, buffer.capacity());
- }
-
- Collections.shuffle(list);
- pool.reset();
- for (int i = 0; i < list.size(); i++) {
- ByteBuffer buffer = pool.allocateFrame(list.get(i) * MIN_FRAME_SIZE);
- assertNotNull(buffer);
- assertTrue(set.contains(new ByteBufferPtr(buffer)));
- assertEquals(list.get(i) * MIN_FRAME_SIZE, buffer.capacity());
- }
-
- }
-
- public static class ByteBufferPtr {
- ByteBuffer bytebuffer;
-
- public ByteBufferPtr(ByteBuffer buffer) {
- bytebuffer = buffer;
- }
-
- @Override
- public int hashCode() {
- return bytebuffer.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- return this.bytebuffer == ((ByteBufferPtr) obj).bytebuffer;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java
deleted file mode 100644
index 18f4f9b..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.BUDGET;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.MIN_FRAME_SIZE;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.NUM_MIN_FRAME;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hyracks.api.comm.FixedSizeFrame;
-import org.apache.hyracks.api.comm.FrameHelper;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAppender;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-
-public class VariableFramesMemoryManagerTest {
- VariableFrameMemoryManager framesMemoryManager;
- FrameTupleAccessor fta;
- Random random;
- List<IFrame> frameList;
-
- @Before
- public void setUp() throws Exception {
- VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, BUDGET);
- FrameFreeSlotLastFit policy = new FrameFreeSlotLastFit(NUM_MIN_FRAME);
- framesMemoryManager = new VariableFrameMemoryManager(framePool, policy);
- RecordDescriptor recordDescriptor = new RecordDescriptor(new ISerializerDeserializer[] { null });
- fta = new FrameTupleAccessor(recordDescriptor);
- random = new Random(System.currentTimeMillis());
- frameList = new ArrayList<>();
- }
-
- @Test
- public void testNormalIncomingFrames() throws HyracksDataException {
- HashMap<Integer, Integer> tupleSet = prepareTuples();
- for (IFrame frame : frameList) {
- assertTrue(framesMemoryManager.insertFrame(frame.getBuffer()) >=0);
- }
- assertEquals(NUM_MIN_FRAME, framesMemoryManager.getNumFrames());
- assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
- }
-
- @Test
- public void testRandomTuplesAreAllStoredInBuffer() throws HyracksDataException {
- Map<Integer, Integer> tupleSet = prepareRandomTuples();
- for (IFrame frame : frameList) {
- if (framesMemoryManager.insertFrame(frame.getBuffer()) < 0) {
- fta.reset(frame.getBuffer());
- for (int i = 0; i < fta.getTupleCount(); ++i) {
- int id = parseTuple(fta.getBuffer(),
- fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 0) + fta.getFieldSlotsLength());
- tupleSet.remove(id);
- // System.out.println(
- // "can't appended id:" + id + ",frameSize:" + frame.getInitialFrameSize());
- }
- }
- }
- assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
- framesMemoryManager.reset();
- }
-
- @Test
- public void testResetShouldWork() throws HyracksDataException {
- testNormalIncomingFrames();
- framesMemoryManager.reset();
- testRandomTuplesAreAllStoredInBuffer();
- framesMemoryManager.reset();
- testRandomTuplesAreAllStoredInBuffer();
- }
-
- @Test
- public void testCloseShouldAlsoWork() throws HyracksDataException {
- testRandomTuplesAreAllStoredInBuffer();
- framesMemoryManager.close();
- testRandomTuplesAreAllStoredInBuffer();
- framesMemoryManager.close();
- testRandomTuplesAreAllStoredInBuffer();
- }
-
- private HashMap<Integer, Integer> prepareRandomTuples() throws HyracksDataException {
- frameList.clear();
- HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
- int[] fieldSlot = { 0 };
- int id = 0;
- int size = 0;
- while (size < BUDGET) {
- int tupleLength = random.nextInt(BUDGET / 3) + 4;
- IFrame frame = new FixedSizeFrame(Common.commonFrameManager
- .allocateFrame(FrameHelper.calcAlignedFrameSizeToStore(1, tupleLength, MIN_FRAME_SIZE)));
- IFrameTupleAppender appender = new FrameTupleAppender();
- appender.reset(frame, true);
- // System.out.println("id:" + id + ",frameSize:" + frame.getInitialFrameSize() / MIN_FRAME_SIZE);
- ByteBuffer buffer = ByteBuffer.allocate(tupleLength);
- buffer.putInt(0, id);
- assertTrue(appender.append(fieldSlot, buffer.array(), 0, buffer.capacity()));
- set.put(id++, tupleLength);
- size += frame.getFrameSize();
- frameList.add(frame);
- }
- return set;
- }
-
- private HashMap<Integer, Integer> prepareTuples() throws HyracksDataException {
- frameList.clear();
- HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
- for (int i = 0; i < NUM_MIN_FRAME; ++i) {
- IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(MIN_FRAME_SIZE));
- IFrameTupleAppender appender = new FrameTupleAppender();
- appender.reset(frame, true);
-
- int[] fieldSlot = { 0 };
- ByteBuffer buffer = ByteBuffer.allocate(MIN_FRAME_SIZE / 2);
- buffer.putInt(0, i);
- appender.append(fieldSlot, buffer.array(), 0, buffer.capacity());
- set.put(i, buffer.capacity());
- frameList.add(frame);
- }
- return set;
- }
-
- private void assertEveryTupleInFTAIsInFrameMemoryManager(Map<Integer, Integer> tupleSet,
- VariableFrameMemoryManager framesMemoryManager) {
- for (int i = 0; i < framesMemoryManager.getNumFrames(); ++i) {
- fta.reset(framesMemoryManager.getFrame(i), framesMemoryManager.getFrameStartOffset(i),
- framesMemoryManager.getFrameSize(i));
- for (int t = 0; t < fta.getTupleCount(); t++) {
- int id = parseTuple(fta.getBuffer(), fta.getTupleStartOffset(t) + fta.getFieldSlotsLength() + fta
- .getFieldStartOffset(t, 0));
- // System.out.println("frameid:" + i + ",tuple:" + t + ",has id:" + id + ",length:" +
- // (fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta.getFieldSlotsLength()));
- assertTrue(tupleSet.remove(id) == fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta
- .getFieldSlotsLength());
- }
- }
- assertTrue(tupleSet.isEmpty());
- }
-
- private int parseTuple(ByteBuffer buffer, int fieldStartOffset) {
- return buffer.getInt(fieldStartOffset);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java
deleted file mode 100644
index d14e15b..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Random;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hyracks.api.comm.FixedSizeFrame;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.util.IntSerDeUtils;
-import org.apache.hyracks.dataflow.std.sort.Utility;
-import org.apache.hyracks.dataflow.std.structures.TuplePointer;
-
-public class VariableTupleMemoryManagerTest {
- ISerializerDeserializer[] fieldsSerDer = new ISerializerDeserializer[] {
- IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
- RecordDescriptor recordDescriptor = new RecordDescriptor(fieldsSerDer);
- ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptor.getFieldCount());
- VariableTupleMemoryManager tupleMemoryManager;
- FrameTupleAccessor inFTA = new FrameTupleAccessor(recordDescriptor);
- Random random = new Random(System.currentTimeMillis());
-
- @Before
- public void setup() {
- VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, Common.BUDGET);
- tupleMemoryManager = new VariableTupleMemoryManager(framePool, recordDescriptor);
- }
-
- @Test
- public void testInsertTupleToMemoryManager() throws HyracksDataException {
- int iTuplePerFrame = 3;
- Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame);
- Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
- assertEachTupleInFTAIsInBuffer(mapPrepare, mapInserted);
- }
-
- @Test
- public void testReset() throws HyracksDataException {
- testInsertVariableSizeTupleToMemoryManager();
- tupleMemoryManager.reset();
- testInsertTupleToMemoryManager();
- tupleMemoryManager.reset();
- testInsertVariableSizeTupleToMemoryManager();
- }
-
- @Test
- public void testDeleteTupleInMemoryManager() throws HyracksDataException {
- int iTuplePerFrame = 3;
- Map<Integer, Integer> map = prepareFixedSizeTuples(iTuplePerFrame);
- Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
- deleteRandomSelectedTuples(map, mapInserted, 1);
- assertEachTupleInFTAIsInBuffer(map, mapInserted);
- }
-
- @Test
- public void testReOrganizeSpace() throws HyracksDataException {
- int iTuplePerFrame = 3;
- Map<Integer, Integer> map = prepareFixedSizeTuples(iTuplePerFrame);
- Map<Integer, Integer> copyMap = new HashMap<>(map);
- Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
- ByteBuffer buffer = deleteRandomSelectedTuples(map, mapInserted, map.size() / 2);
- inFTA.reset(buffer);
- Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferShouldAllSuccess();
- Map<TuplePointer, Integer> mergedMap = new HashMap<>(mapInserted);
- mergedMap.putAll(mapInserted2);
- assertEachTupleInFTAIsInBuffer(copyMap, mergedMap);
- }
-
- @Test
- public void testReOrganizeVariableSizeTuple() throws HyracksDataException {
- Map<Integer, Integer> map = prepareVariableSizeTuples();
- Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
- Map<Integer, Integer> copyMap = new HashMap<>(map);
-
- ByteBuffer buffer = deleteRandomSelectedTuples(map, mapInserted, map.size() / 2);
- inFTA.reset(buffer);
-
- Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferCouldFailForLargerTuples(copyMap);
- Map<TuplePointer, Integer> mergedMap = new HashMap<>(mapInserted);
- mergedMap.putAll(mapInserted2);
-
- assertEachTupleInFTAIsInBuffer(copyMap, mergedMap);
- }
-
- @Test
- public void testInsertVariableSizeTupleToMemoryManager() throws HyracksDataException {
- Map<Integer, Integer> map = prepareVariableSizeTuples();
- Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
- assertEachTupleInFTAIsInBuffer(map, mapInserted);
- }
-
- private void assertEachTupleInFTAIsInBuffer(Map<Integer, Integer> map, Map<TuplePointer, Integer> mapInserted) {
- ITupleBufferAccessor accessor = tupleMemoryManager.getTupleAccessor();
- for (Map.Entry<TuplePointer, Integer> entry : mapInserted.entrySet()) {
- accessor.reset(entry.getKey());
- int dataLength = map.get(entry.getValue());
- assertEquals((int) entry.getValue(),
- IntSerDeUtils.getInt(accessor.getTupleBuffer().array(), accessor.getAbsFieldStartOffset(0)));
- assertEquals(dataLength, accessor.getTupleLength());
- }
- assertEquals(map.size(), mapInserted.size());
- }
-
- private Map<Integer, Integer> prepareFixedSizeTuples(int tuplePerFrame) throws HyracksDataException {
- Map<Integer, Integer> dataSet = new HashMap<>();
- ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
- FixedSizeFrame frame = new FixedSizeFrame(buffer);
- FrameTupleAppender appender = new FrameTupleAppender();
- appender.reset(frame, true);
-
- int sizePerTuple = (Common.MIN_FRAME_SIZE - 1 - 4 - tuplePerFrame * 4 - 4) / tuplePerFrame;
- int sizeChar = sizePerTuple - fieldsSerDer.length * 4 - 4 - 4;
- assert (sizeChar > 0);
- for (int i = 0; i < Common.NUM_MIN_FRAME * tuplePerFrame; i++) {
- tupleBuilder.reset();
- tupleBuilder.addField(fieldsSerDer[0], i);
- tupleBuilder.addField(fieldsSerDer[1], Utility.repeatString('a', sizeChar));
- assertTrue(appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
- tupleBuilder.getSize()));
- dataSet.put(i, tupleBuilder.getSize() + tupleBuilder.getFieldEndOffsets().length * 4);
- }
- inFTA.reset(buffer);
- return dataSet;
- }
-
- private Map<Integer, Integer> prepareVariableSizeTuples() throws HyracksDataException {
- Map<Integer, Integer> dataSet = new HashMap<>();
- ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
- FixedSizeFrame frame = new FixedSizeFrame(buffer);
- FrameTupleAppender appender = new FrameTupleAppender();
- appender.reset(frame, true);
-
- for (int i = 0; true; i++) {
- tupleBuilder.reset();
- tupleBuilder.addField(fieldsSerDer[0], i);
- tupleBuilder.addField(fieldsSerDer[1], Utility.repeatString('a', i));
- if (!appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
- tupleBuilder.getSize())) {
- break;
- }
- dataSet.put(i, tupleBuilder.getSize() + tupleBuilder.getFieldEndOffsets().length * 4);
- }
- inFTA.reset(buffer);
- return dataSet;
- }
-
- private Map<TuplePointer, Integer> insertInFTAToBufferShouldAllSuccess() throws HyracksDataException {
- Map<TuplePointer, Integer> tuplePointerIntegerMap = new HashMap<>();
- for (int i = 0; i < inFTA.getTupleCount(); i++) {
- TuplePointer tuplePointer = new TuplePointer();
- assertTrue(tupleMemoryManager.insertTuple(inFTA, i, tuplePointer));
- tuplePointerIntegerMap.put(tuplePointer,
- IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
- }
- return tuplePointerIntegerMap;
- }
-
- private Map<TuplePointer, Integer> insertInFTAToBufferCouldFailForLargerTuples(Map<Integer, Integer> map)
- throws HyracksDataException {
- Map<TuplePointer, Integer> tuplePointerIdMap = new HashMap<>();
- int i = 0;
- for (; i < inFTA.getTupleCount(); i++) {
- TuplePointer tuplePointer = new TuplePointer();
- if (!tupleMemoryManager.insertTuple(inFTA, i, tuplePointer)) {
- break;
- }
- tuplePointerIdMap.put(tuplePointer,
- IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
- }
- for (; i < inFTA.getTupleCount(); i++) {
- map.remove(IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
- }
- return tuplePointerIdMap;
- }
-
- private ByteBuffer deleteRandomSelectedTuples(Map<Integer, Integer> map, Map<TuplePointer, Integer> mapInserted,
- int minNumOfRecordTobeDeleted)
- throws HyracksDataException {
- ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
- FixedSizeFrame frame = new FixedSizeFrame(buffer);
- FrameTupleAppender appender = new FrameTupleAppender();
- appender.reset(frame, true);
-
- assert (minNumOfRecordTobeDeleted < mapInserted.size());
- int countDeleted = minNumOfRecordTobeDeleted + random.nextInt(mapInserted.size() - minNumOfRecordTobeDeleted);
-
- ITupleBufferAccessor accessor = tupleMemoryManager.getTupleAccessor();
- for (int i = 0; i < countDeleted; i++) {
- Iterator<Map.Entry<TuplePointer, Integer>> iter = mapInserted.entrySet().iterator();
- assert (iter.hasNext());
- Map.Entry<TuplePointer, Integer> pair = iter.next();
- accessor.reset(pair.getKey());
- appender.append(accessor.getTupleBuffer().array(), accessor.getTupleStartOffset(),
- accessor.getTupleLength());
- map.remove(pair.getValue());
- tupleMemoryManager.deleteTuple(pair.getKey());
- iter.remove();
- }
- return buffer;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java
new file mode 100644
index 0000000..5893f35
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.structures;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.nc.resources.memory.FrameManager;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SerializableHashTableTest {
+
+ SerializableHashTable nsTable;
+ final int NUM_PART = 101;
+ TuplePointer pointer = new TuplePointer(0, 0);
+ final int num = 1000;
+
+ @Before
+ public void setup() throws HyracksDataException {
+ nsTable = new SerializableHashTable(NUM_PART, new FrameManager(256));
+ }
+
+ @Test
+ public void testBatchDeletePartition() throws Exception {
+ testInsert();
+ for (int i = 0; i < NUM_PART; i++) {
+ nsTable.delete(i);
+ assertFalse(nsTable.getTuplePointer(i, 0, pointer));
+ assertEquals(0, nsTable.getTupleCount(i));
+
+ for (int j = i; j < num; j += NUM_PART) {
+ pointer.reset(j, j);
+ nsTable.insert(i, pointer);
+ }
+
+ assertGetValue();
+ }
+ }
+
+ @Test
+ public void testInsert() throws Exception {
+ for (int i = 0; i < num; i++) {
+ pointer.reset(i, i);
+ nsTable.insert(i % NUM_PART, pointer);
+ }
+ assertGetValue();
+ }
+
+ private void assertGetValue() {
+ int loop = 0;
+ for (int i = 0; i < num; i++) {
+ assertTrue(nsTable.getTuplePointer(i % NUM_PART, loop, pointer));
+ assertTrue(pointer.frameIndex == i);
+ if (i % NUM_PART == NUM_PART - 1) {
+ loop++;
+ }
+ }
+ for (int i = 0; i < NUM_PART; i++) {
+ assertTrue(nsTable.getTupleCount(i) == 10 || nsTable.getTupleCount(i) == 9);
+ }
+
+ }
+
+ @Test
+ public void testGetCount() throws Exception {
+ assertAllPartitionsCountIsZero();
+ }
+
+ private void assertAllPartitionsCountIsZero() {
+ for (int i = 0; i < NUM_PART; i++) {
+ assertEquals(0, nsTable.getTupleCount(i));
+ }
+ }
+}
\ No newline at end of file