You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by "Wail Alkowaileet (Code Review)" <do...@asterixdb.incubator.apache.org> on 2019/02/01 08:10:19 UTC

Change in asterixdb[master]: [ASTERIXDB-2491][TXN] Fix(hack) to 16-bit offset overflow

Wail Alkowaileet has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/3140

Change subject: [ASTERIXDB-2491][TXN] Fix(hack) to 16-bit offset overflow
......................................................................

[ASTERIXDB-2491][TXN] Fix(hack) to 16-bit offset overflow

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Add unsigned short deserializer
- Try to detect overflowed offsets and try to ensure
  offsets are monotonically increasing.
- As a last resort, it sets the last field's (payload's)
  end offset as the size of the value.
- Add recovery test (passed 100KB record)

Change-Id: I2ab49af8ce7c4753404dbb38ac58e28f75cae8fe
---
M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql
A asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql
A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh
A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh
A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh
A asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh
M asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml
M hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java
M hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
17 files changed, 342 insertions(+), 19 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/40/3140/1

diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java
index 7a1079d..c8da50a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/LogRecord.java
@@ -415,8 +415,8 @@
         if (srcBuffer.position() + size > srcBuffer.limit()) {
             throw new BufferUnderflowException();
         }
+        destTuple.resetByTupleOffset(srcBuffer.array(), srcBuffer.position(), size);
         destTuple.setFieldCount(fieldCnt);
-        destTuple.resetByTupleOffset(srcBuffer.array(), srcBuffer.position());
         srcBuffer.position(srcBuffer.position() + size);
         return destTuple;
     }
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql
new file mode 100644
index 0000000..7d441cd
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.1.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+create_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql
new file mode 100644
index 0000000..02f3a72
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.2.ddl.aql
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : large_object_100K
+ * Description     : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB)
+ * Expected Result : Success
+ * Date            : December 25 2018
+ */
+
+drop dataverse recovery if exists;
+create dataverse recovery;
+use dataverse recovery;
+
+create type RecoveryType as {
+   uid: uuid,
+   numbers: [int]
+};
+
+create dataset RecoveryDataset (RecoveryType)
+primary key uid autogenerated;
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql
new file mode 100644
index 0000000..1e24bcf
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.3.update.aql
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : large_object_100K
+ * Description     : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB)
+ * Expected Result : Success
+ * Date            : December 25 2018
+ */
+
+use dataverse recovery;
+
+//Create a 100KB record (8-bytes * 12800 = 102400 bytes)
+insert into dataset RecoveryDataset (
+    {"numbers": range(1, 12800)}
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql
new file mode 100644
index 0000000..2e19c39
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.4.txnqbc.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : large_object_100K
+ * Description     : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB)
+ * Expected Result : Success
+ * Date            : December 25 2018
+ */
+
+use dataverse recovery;
+
+for $x in dataset RecoveryDataset
+return count($x.numbers);
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql
new file mode 100644
index 0000000..4583455
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.5.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+kill_cc_and_nc.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql
new file mode 100644
index 0000000..7087cd3
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.6.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_start.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql
new file mode 100644
index 0000000..2e19c39
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.7.txnqar.aql
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name  : large_object_100K
+ * Description     : ASTERIXDB-2491 (Recovery fails for objects with size larger than 32KB)
+ * Expected Result : Success
+ * Date            : December 25 2018
+ */
+
+use dataverse recovery;
+
+for $x in dataset RecoveryDataset
+return count($x.numbers);
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql
new file mode 100644
index 0000000..40df6fb
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/queries/recover_after_abort/large_object_100K/large_object_100K.8.script.aql
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+stop_and_delete.sh
\ No newline at end of file
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh
new file mode 100755
index 0000000..e358618
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/create_and_start.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$NCSERVICE_HOME/opt/local/bin/start-sample-cluster.sh 1>/dev/null 2>&1;
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh
new file mode 100755
index 0000000..b6326cc
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/kill_cc_and_nc.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+ps -ef | awk '/java.*org\.apache\.hyracks\.control\.[cn]c\.[CN]CDriver/ {print $2}' | xargs -n 1 kill -9
+ps -ef | awk '/java.*org\.apache\.hyracks\.control\.nc\.service\.NCService/ {print $2}' | xargs -n 1 kill -9
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh
new file mode 100755
index 0000000..818d17d
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_delete.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$NCSERVICE_HOME/opt/local/bin/stop-sample-cluster.sh;
+rm -rf $NCSERVICE_HOME/opt/local/data;
+
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh
new file mode 100755
index 0000000..9a0c506
--- /dev/null
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/scripts/recover_after_abort/large_object_100K/stop_and_start.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+$NCSERVICE_HOME/opt/local/bin/stop-sample-cluster.sh;
+$NCSERVICE_HOME/opt/local/bin/start-sample-cluster.sh;
diff --git a/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml b/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml
index 38179b2..b00a14a 100644
--- a/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml
+++ b/asterixdb/asterix-server/src/test/resources/transactionts/testsuite.xml
@@ -158,9 +158,15 @@
     </test-case>
 
     <test-case FilePath="recover_after_abort">
-      <compilation-unit name="primary_plus_multiple_secondary_indices"><!-- The only exception here is during the kill command which is in a different JVM, hence not caught -->
+      <compilation-unit name="primary_plus_multiple_secondary_indices">The only exception here is during the kill command which is in a different JVM, hence not caught
         <output-dir compare="Text">primary_plus_multiple_secondary_indices</output-dir>
-        <!-- <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error> -->
+        <expected-error>org.apache.hyracks.algebricks.common.exceptions.AlgebricksException</expected-error>
+      </compilation-unit>
+    </test-case>
+
+    <test-case FilePath="recover_after_abort">
+      <compilation-unit name="large_object_100K">
+        <output-dir compare="Text">large_object_100K</output-dir>
       </compilation-unit>
     </test-case>
   </test-group>
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java
index 73f3779..a684792 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/primitive/ShortPointable.java
@@ -64,6 +64,10 @@
         return (short) (((bytes[start] & 0xff) << 8) + (bytes[start + 1] & 0xff));
     }
 
+    public static int getUnsignedShort(byte[] bytes, int start) {
+        return getShort(bytes, start) & 0xFFFF;
+    }
+
     public static void setShort(byte[] bytes, int start, short value) {
         bytes[start] = (byte) ((value >>> 8) & 0xFF);
         bytes[start + 1] = (byte) ((value >>> 0) & 0xFF);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java
index e82b037..90d35dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleReference.java
@@ -23,20 +23,30 @@
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
 import org.apache.hyracks.storage.am.common.util.BitOperationUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 public class SimpleTupleReference implements ITreeIndexTupleReference {
-
+    private static final Logger LOGGER = LogManager.getLogger();
     protected byte[] buf;
     protected int fieldStartIndex;
     protected int fieldCount;
     protected int tupleStartOff;
     protected int nullFlagsBytes;
     protected int fieldSlotsBytes;
+    private int[] fixedEndOffsets;
+    private int size;
 
     @Override
     public void resetByTupleOffset(byte[] buf, int tupleStartOff) {
         this.buf = buf;
         this.tupleStartOff = tupleStartOff;
+    }
+
+    public void resetByTupleOffset(byte[] buf, int tupleStartOff, int size) {
+        this.buf = buf;
+        this.tupleStartOff = tupleStartOff;
+        this.size = size;
     }
 
     @Override
@@ -50,6 +60,7 @@
         nullFlagsBytes = getNullFlagsBytes();
         fieldSlotsBytes = getFieldSlotsBytes();
         fieldStartIndex = 0;
+        fixEndOffsets();
     }
 
     @Override
@@ -71,10 +82,9 @@
     @Override
     public int getFieldLength(int fIdx) {
         if (fIdx == 0) {
-            return ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes);
+            return fixedEndOffsets[0];
         } else {
-            return ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + fIdx * 2)
-                    - ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + ((fIdx - 1) * 2));
+            return fixedEndOffsets[fIdx] - fixedEndOffsets[fIdx - 1];
         }
     }
 
@@ -83,8 +93,7 @@
         if (fIdx == 0) {
             return tupleStartOff + nullFlagsBytes + fieldSlotsBytes;
         } else {
-            return tupleStartOff + nullFlagsBytes + fieldSlotsBytes
-                    + ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + ((fIdx - 1) * 2));
+            return tupleStartOff + nullFlagsBytes + fieldSlotsBytes + fixedEndOffsets[fIdx - 1];
         }
     }
 
@@ -98,7 +107,39 @@
 
     @Override
     public int getTupleSize() {
-        return nullFlagsBytes + fieldSlotsBytes
-                + ShortPointable.getShort(buf, tupleStartOff + nullFlagsBytes + (fieldCount - 1) * 2);
+        return nullFlagsBytes + fieldSlotsBytes + fixedEndOffsets[fieldCount - 1];
+    }
+
+    private int getCorruptedFieldEndOffset(int fIdx) {
+        return ShortPointable.getUnsignedShort(buf, tupleStartOff + nullFlagsBytes + fIdx * 2);
+    }
+
+    /*
+     * This is a hack to fix the 16-bit overflow issue [ASTERIXDB-2491] 
+     * It won't detect double overflow or if the first endOffset overflowed.
+     * Hence, we need a more comprehensive fix.
+     */
+    private void fixEndOffsets() {
+        if (fixedEndOffsets == null || fixedEndOffsets.length < fieldCount) {
+            fixedEndOffsets = new int[fieldCount];
+        }
+
+        fixedEndOffsets[0] = getCorruptedFieldEndOffset(0);
+        boolean isOverflowed = false;
+        for (int i = 1; i < fieldCount; i++) {
+            fixedEndOffsets[i] = getCorruptedFieldEndOffset(i);
+            if (isOverflowed || fixedEndOffsets[i - 1] > fixedEndOffsets[i]) {
+                fixedEndOffsets[i] += fixedEndOffsets[i - 1];
+                isOverflowed = true;
+            }
+        }
+
+        //it assumes that the payload (the largest field) is at the end
+        if (fixedEndOffsets[fieldCount - 1] != size) {
+            if (LOGGER.isDebugEnabled()) {
+                LOGGER.debug("Double overflow detected in log record tuple");
+            }
+            fixedEndOffsets[fieldCount - 1] = size;
+        }
     }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
index 410a0e3..e87832c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
@@ -21,6 +21,7 @@
 
 import java.nio.ByteBuffer;
 
+import org.apache.hyracks.data.std.primitive.ShortPointable;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
 import org.apache.hyracks.storage.am.common.util.BitOperationUtils;
@@ -33,12 +34,6 @@
     public static final SimpleTupleWriter INSTANCE = new SimpleTupleWriter();
 
     private SimpleTupleWriter() {
-    }
-
-    // Write short in little endian to target byte array at given offset.
-    private static void writeShortL(short s, byte[] buf, int targetOff) {
-        buf[targetOff] = (byte) (s >> 8);
-        buf[targetOff + 1] = (byte) (s >> 0);
     }
 
     @Override
@@ -83,7 +78,7 @@
             System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner, tuple.getFieldLength(i));
             fieldEndOff += tuple.getFieldLength(i);
             runner += tuple.getFieldLength(i);
-            writeShortL((short) fieldEndOff, targetBuf, targetOff + nullFlagsBytes + i * 2);
+            ShortPointable.setShort(targetBuf, targetOff + nullFlagsBytes + i * 2, (short) fieldEndOff);
         }
         return runner - targetOff;
     }
@@ -103,7 +98,7 @@
             System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner, tuple.getFieldLength(i));
             fieldEndOff += tuple.getFieldLength(i);
             runner += tuple.getFieldLength(i);
-            writeShortL((short) fieldEndOff, targetBuf, targetOff + nullFlagsBytes + fieldCounter * 2);
+            ShortPointable.setShort(targetBuf, targetOff + nullFlagsBytes + fieldCounter * 2, (short) fieldEndOff);
             fieldCounter++;
         }
 

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/3140
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I2ab49af8ce7c4753404dbb38ac58e28f75cae8fe
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Wail Alkowaileet <wa...@gmail.com>