You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by sc...@apache.org on 2016/02/17 19:43:42 UTC
svn commit: r1730902 [2/3] - in
/uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl:
BinaryCasSerDes4.java BinaryCasSerDes6.java
Modified: uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java?rev=1730902&r1=1730901&r2=1730902&view=diff
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java (original)
+++ uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java Wed Feb 17 18:43:42 2016
@@ -19,34 +19,7 @@
package org.apache.uima.cas.impl;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.NBR_SLOT_KIND_ZIP_STREAMS;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_ArrayLength;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Boolean;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_BooleanRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Byte;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_ByteRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Control;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_DoubleRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Double_Exponent;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Double_Mantissa_Sign;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Float;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Float_Exponent;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Float_Mantissa_Sign;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_FsIndexes;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_HeapRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Int;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_LongRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Long_High;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Long_Low;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_MainHeap;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Short;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_ShortRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrChars;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrLength;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrOffset;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrSeg;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_TypeCode;
+import static org.apache.uima.cas.impl.SlotKinds.SlotKind.Slot_Int;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -63,10 +36,10 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
+import java.util.BitSet;
+import java.util.Collections;
import java.util.List;
-import java.util.Map;
+import java.util.function.Consumer;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.Inflater;
@@ -74,11 +47,29 @@ import java.util.zip.InflaterInputStream
import org.apache.uima.cas.AbstractCas;
import org.apache.uima.cas.CASRuntimeException;
-import org.apache.uima.cas.Feature;
import org.apache.uima.cas.Marker;
+import org.apache.uima.cas.impl.CASImpl.FsChange;
import org.apache.uima.cas.impl.FSsTobeAddedback.FSsTobeAddedbackSingle;
+import org.apache.uima.cas.impl.SlotKinds.SlotKind;
+import org.apache.uima.internal.util.Int2ObjHashMap;
+import org.apache.uima.internal.util.IntListIterator;
import org.apache.uima.internal.util.IntVector;
+import org.apache.uima.internal.util.Obj2IntIdentityHashMap;
import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.cas.BooleanArray;
+import org.apache.uima.jcas.cas.ByteArray;
+import org.apache.uima.jcas.cas.CommonArray;
+import org.apache.uima.jcas.cas.DoubleArray;
+import org.apache.uima.jcas.cas.FSArray;
+import org.apache.uima.jcas.cas.FloatArray;
+import org.apache.uima.jcas.cas.IntegerArray;
+import org.apache.uima.jcas.cas.LongArray;
+import org.apache.uima.jcas.cas.ShortArray;
+import org.apache.uima.jcas.cas.Sofa;
+import org.apache.uima.jcas.cas.StringArray;
+import org.apache.uima.jcas.cas.TOP;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.Misc;
import org.apache.uima.util.impl.DataIO;
import org.apache.uima.util.impl.OptimizeStrings;
import org.apache.uima.util.impl.SerializationMeasures;
@@ -126,9 +117,19 @@ import org.apache.uima.util.impl.Seriali
* Read all bytes,
* create separate ByteArrayInputStreams for each segment, sharing byte bfr
* create appropriate unzip data input streams for these
+ *
+ * Properties of Form 4:
+ * 1) Indexes ** not used ** for determining what gets serialized, instead
+ * the "heap" is walked and all FSs found there are serialized
+ *
+ * 2) The number used for references to FSs is a sequentially incrementing one, starting at 1
+ *
+ *
*/
-public class BinaryCasSerDes4 {
-
+public class BinaryCasSerDes4 implements SlotKindsConstants {
+ // 0 - original, 1 = fixes from V3 in v2, 2 = v3 --------------------------v
+ private static final boolean TRACE_DOUBLE = false;
+ private static final boolean TRACE_INT = false;
public static final int TYPECODE_COMPR = 8;
public static final boolean CHANGE_FS_REFS_TO_SEQUENTIAL = true;
// may add more later - to specify differing trade-offs between speed and compression
@@ -138,7 +139,6 @@ public class BinaryCasSerDes4 {
public static final boolean IGNORED = true;
public static final boolean IN_MAIN_HEAP = true;
- private static final long DBL_1 = Double.doubleToLongBits(1D);
/**
* The kinds of slots that can exist
@@ -203,101 +203,109 @@ public class BinaryCasSerDes4 {
}
}
- /**
- * Define all the slot kinds.
- */
- public enum SlotKind {
- Slot_ArrayLength(! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
- Slot_HeapRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_Int( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_Byte( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
- Slot_Short( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_TypeCode( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
-
- Slot_StrOffset( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
- Slot_StrLength( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
- Slot_Long_High( IS_DIFF_ENCODE, IGNORED, 0, !IN_MAIN_HEAP),
- Slot_Long_Low ( IS_DIFF_ENCODE, IGNORED, 0, !IN_MAIN_HEAP),
-
- // the next are not actual slot kinds, but instead
- // are codes used to control encoding of Floats and Doubles.
- Slot_Float_Mantissa_Sign( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
- // exponent is 8 bits, and shifted in the expectation
- // that many values may be between 1 and 0 (e.g., normalized values)
- // -- so sign moving is needed
- Slot_Float_Exponent( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
-
- Slot_Double_Mantissa_Sign(! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
- Slot_Double_Exponent( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
- Slot_FsIndexes( IS_DIFF_ENCODE, IGNORED, 4, !IN_MAIN_HEAP),
-
- Slot_StrChars( IGNORED, IGNORED, 2, !IN_MAIN_HEAP),
-
- Slot_Control( IGNORED, IGNORED, 0, !IN_MAIN_HEAP),
- Slot_StrSeg( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 0, ! IN_MAIN_HEAP),
-
- // the next slots are not serialized
- Slot_StrRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_BooleanRef( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
- Slot_ByteRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_ShortRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_LongRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_DoubleRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
- Slot_Float( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
- Slot_Boolean( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
- // next used to capture original heap size
- Slot_MainHeap( IGNORED, IGNORED, 4, !IN_MAIN_HEAP),
-
- ;
-
- public final int i;
- public final boolean isDiffEncode;
- public final boolean canBeNegative;
- public final boolean inMainHeap;
- public final int elementSize;
-
- public static final int NBR_SLOT_KIND_ZIP_STREAMS;
- static {NBR_SLOT_KIND_ZIP_STREAMS = Slot_StrRef.i;}
-
- SlotKind(boolean isDiffEncode,
- boolean canBeNegative,
- int elementSize,
- boolean inMainHeap) {
- this.i = this.ordinal();
- this.isDiffEncode = isDiffEncode;
- this.canBeNegative = isDiffEncode ? true : canBeNegative;
- this.elementSize = elementSize;
- this.inMainHeap = inMainHeap;
- }
- }
+// /**
+// * Define all the slot kinds.
+// */
+// public enum SlotKind {
+// Slot_ArrayLength(! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+// Slot_HeapRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_Int( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_Byte( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+// Slot_Short( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_TypeCode( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+//
+// Slot_StrOffset( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
+// Slot_StrLength( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
+// Slot_Long_High( IS_DIFF_ENCODE, IGNORED, 0, !IN_MAIN_HEAP),
+// Slot_Long_Low ( IS_DIFF_ENCODE, IGNORED, 0, !IN_MAIN_HEAP),
+//
+// // the next are not actual slot kinds, but instead
+// // are codes used to control encoding of Floats and Doubles.
+// Slot_Float_Mantissa_Sign( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+// // exponent is 8 bits, and shifted in the expectation
+// // that many values may be between 1 and 0 (e.g., normalized values)
+// // -- so sign moving is needed
+// Slot_Float_Exponent( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+//
+// Slot_Double_Mantissa_Sign(! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+// Slot_Double_Exponent( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+// Slot_FsIndexes( IS_DIFF_ENCODE, IGNORED, 4, !IN_MAIN_HEAP),
+//
+// Slot_StrChars( IGNORED, IGNORED, 2, !IN_MAIN_HEAP),
+//
+// Slot_Control( IGNORED, IGNORED, 0, !IN_MAIN_HEAP),
+// Slot_StrSeg( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 0, ! IN_MAIN_HEAP),
+//
+// // the next slots are not serialized
+// Slot_StrRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_BooleanRef( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+// Slot_ByteRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_ShortRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_LongRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_DoubleRef( IS_DIFF_ENCODE, IGNORED, 4, IN_MAIN_HEAP),
+// Slot_Float( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+// Slot_Boolean( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+// // next used to capture original heap size
+// Slot_MainHeap( IGNORED, IGNORED, 4, !IN_MAIN_HEAP),
+//
+// //TODO fix this
+// Slot_JavaObjectRef( IGNORED, CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP);
+// ;
+//
+// public final int i;
+// public final boolean isDiffEncode;
+// public final boolean canBeNegative;
+// public final boolean inMainHeap;
+// public final int elementSize;
+//
+// public static final int NBR_SLOT_KIND_ZIP_STREAMS;
+// static {NBR_SLOT_KIND_ZIP_STREAMS = Slot_StrRef.i;}
+//
+// SlotKind(boolean isDiffEncode,
+// boolean canBeNegative,
+// int elementSize,
+// boolean inMainHeap) {
+// this.i = this.ordinal();
+// this.isDiffEncode = isDiffEncode;
+// this.canBeNegative = isDiffEncode ? true : canBeNegative;
+// this.elementSize = elementSize;
+// this.inMainHeap = inMainHeap;
+// }
+// }
/**
* Things set up for one instance of this class, and
* reuse-able
*/
- final private TypeInfo [] typeInfoArray; // lazy initialization of elements
- final private TypeSystemImpl ts; // for debugging
+ final private TypeSystemImpl ts;
final private boolean doMeasurements;
+ final TypeImpl fsArrayType;
+
+
+ /**
+ * Things shared between serialization and deserialization
+ */
+
// speedups
- final private static int arrayLength_i = Slot_ArrayLength.i;
- final private static int heapRef_i = Slot_HeapRef.i;
- final private static int int_i = Slot_Int.i;
- final private static int byte_i = Slot_Byte.ordinal();
- final private static int short_i = Slot_Short.i;
- final private static int typeCode_i = Slot_TypeCode.i;
- final private static int strOffset_i = Slot_StrOffset.i;
- final private static int strLength_i = Slot_StrLength.i;
- final private static int long_High_i = Slot_Long_High.i;
- final private static int long_Low_i = Slot_Long_Low.i;
- final private static int float_Mantissa_Sign_i = Slot_Float_Mantissa_Sign.i;
- final private static int float_Exponent_i = Slot_Float_Exponent.i;
- final private static int double_Mantissa_Sign_i = Slot_Double_Mantissa_Sign.i;
- final private static int double_Exponent_i = Slot_Double_Exponent.i;
- final private static int fsIndexes_i = Slot_FsIndexes.i;
- final private static int strChars_i = Slot_StrChars.i;
- final private static int control_i = Slot_Control.i;
- final private static int strSeg_i = Slot_StrSeg.i;
+// final private static int arrayLength_i = Slot_ArrayLength.i;
+// final private static int heapRef_i = Slot_HeapRef.i;
+// final private static int int_i = Slot_Int.i;
+// final private static int byte_i = Slot_Byte.ordinal();
+// final private static int short_i = Slot_Short.i;
+// final private static int typeCode_i = Slot_TypeCode.i;
+// final private static int strOffset_i = Slot_StrOffset.i;
+// final private static int strLength_i = Slot_StrLength.i;
+// final private static int long_High_i = Slot_Long_High.i;
+// final private static int long_Low_i = Slot_Long_Low.i;
+// final private static int float_Mantissa_Sign_i = Slot_Float_Mantissa_Sign.i;
+// final private static int float_Exponent_i = Slot_Float_Exponent.i;
+// final private static int double_Mantissa_Sign_i = Slot_Double_Mantissa_Sign.i;
+// final private static int double_Exponent_i = Slot_Double_Exponent.i;
+// final private static int fsIndexes_i = Slot_FsIndexes.i;
+// final private static int strChars_i = Slot_StrChars.i;
+// final private static int control_i = Slot_Control.i;
+// final private static int strSeg_i = Slot_StrSeg.i;
/**
*
@@ -307,9 +315,8 @@ public class BinaryCasSerDes4 {
public BinaryCasSerDes4(TypeSystemImpl ts, boolean doMeasurements) {
this.ts = ts;
this.doMeasurements = doMeasurements;
-
- typeInfoArray = new TypeInfo[(ts.getTypeArraySize())];
- }
+ this.fsArrayType = ts.fsArrayType;
+ }
/**
*
@@ -326,8 +333,7 @@ public class BinaryCasSerDes4 {
SerializationMeasures sm = (doMeasurements) ? new SerializationMeasures() : null;
CASImpl casImpl = (CASImpl) ((cas instanceof JCas) ? ((JCas)cas).getCas(): cas);
if (null != trackingMark && !trackingMark.isValid() ) {
- throw new CASRuntimeException(
- CASRuntimeException.INVALID_MARKER, new String[] { "Invalid Marker." });
+ throw new CASRuntimeException(CASRuntimeException.INVALID_MARKER, "Invalid Marker.");
}
Serializer serializer = new Serializer(
@@ -351,15 +357,10 @@ public class BinaryCasSerDes4 {
return serialize(cas, out, null);
}
- public void deserialize(CASImpl cas, InputStream deserIn, boolean isDelta) throws IOException {
- DataInput in;
- if (deserIn instanceof DataInputStream) {
- in = (DataInputStream)deserIn;
- } else {
- in = new DataInputStream(deserIn);
- }
+ public void deserialize(CASImpl cas, InputStream deserIn, boolean isDelta, int version1) throws IOException {
+ DataInput in = CommonSerDes.maybeWrapToDataInputStream(deserIn);
Deserializer deserializer = new Deserializer(cas, in, isDelta);
- deserializer.deserialize();
+ deserializer.deserialize(version1);
}
/**
@@ -370,65 +371,94 @@ public class BinaryCasSerDes4 {
private class Serializer {
final private DataOutputStream serializedOut; // where to write out the serialized result
- final private CASImpl cas; // cas being serialized
+ final private CASImpl baseCas; // cas being serialized
+ final private BinaryCasSerDes bcsd;
final private MarkerImpl mark; // the mark to serialize from
final private SerializationMeasures sm; // null or serialization measurements
final private ByteArrayOutputStream[] baosZipSources = new ByteArrayOutputStream[NBR_SLOT_KIND_ZIP_STREAMS]; // lazily created, indexed by SlotKind.i
final private DataOutputStream[] dosZipSources = new DataOutputStream[NBR_SLOT_KIND_ZIP_STREAMS]; // lazily created, indexed by SlotKind.i
- final private int[] heap; // main heap
+// final private int[] heap; // main heap
private int heapStart;
- final private int heapEnd;
- final private StringHeap stringHeapObj;
- final private LongHeap longHeapObj;
- final private ShortHeap shortHeapObj;
- final private ByteHeap byteHeapObj;
+ private int heapEnd;
+// final private LongHeap longHeapObj;
+// final private ShortHeap shortHeapObj;
+// final private ByteHeap byteHeapObj;
final private boolean isDelta; // if true, there is a marker indicating the start spot(s)
final private boolean doMeasurement; // if true, doing measurements
- final private ComprItemRefs fsStartIndexes = (CHANGE_FS_REFS_TO_SEQUENTIAL) ? new ComprItemRefs() : null;
- final private int[] typeCodeHisto = new int[ts.getTypeArraySize()];
- final private Integer[] serializedTypeCode2Code = new Integer[ts.getTypeArraySize()]; // needs to be Integer to get comparator choice
- final private int[] estimatedZipSize = new int[NBR_SLOT_KIND_ZIP_STREAMS]; // one entry for each output stream kind
+// final private ComprItemRefs fsStartIndexes = (CHANGE_FS_REFS_TO_SEQUENTIAL) ? new ComprItemRefs() : null;
+// final private int[] typeCodeHisto = new int[ts.getTypeArraySize()];
+// final private Integer[] serializedTypeCode2Code = new Integer[ts.getTypeArraySize()]; // needs to be Integer to get comparator choice
+// final private int[] estimatedZipSize = new int[NBR_SLOT_KIND_ZIP_STREAMS]; // one entry for each output stream kind
final private OptimizeStrings os;
final private CompressLevel compressLevel;
final private CompressStrat compressStrategy;
- // typeInfo is local to this serialization instance to permit multiple threads
- private TypeInfo typeInfo; // type info for the current type being serialized
- private int iPrevHeap; // 0 or heap addr of previous instance of current type
+ // private int iPrevHeap; // 0 or heap addr of previous instance of current type
+ /**
+ * For differencing when reading and writing.
+ * Also used for arrays to difference the 0th element.
+ */
+ final private TOP prevFsByType[];
+ private TOP prevFs;
+
private boolean only1CommonString; // true if only one common string
+// final private CommonCompressedSerialization ccs;
+ final private CommonSerDesSequential csds;
+
// speedups
// any use of these means caller handles measurement
// some of these are never used, because the current impl
// is using the _i form to get measurements done
- final private DataOutputStream arrayLength_dos;
- final private DataOutputStream heapRef_dos;
- final private DataOutputStream int_dos;
+// final private DataOutputStream arrayLength_dos;
+// final private DataOutputStream heapRef_dos;
+// final private DataOutputStream int_dos;
final private DataOutputStream byte_dos;
- final private DataOutputStream short_dos;
+// final private DataOutputStream short_dos;
final private DataOutputStream typeCode_dos;
final private DataOutputStream strOffset_dos;
final private DataOutputStream strLength_dos;
- final private DataOutputStream long_High_dos;
- final private DataOutputStream long_Low_dos;
+// final private DataOutputStream long_High_dos;
+// final private DataOutputStream long_Low_dos;
final private DataOutputStream float_Mantissa_Sign_dos;
final private DataOutputStream float_Exponent_dos;
final private DataOutputStream double_Mantissa_Sign_dos;
final private DataOutputStream double_Exponent_dos;
final private DataOutputStream fsIndexes_dos;
- final private DataOutputStream strChars_dos;
+// final private DataOutputStream strChars_dos;
final private DataOutputStream control_dos;
final private DataOutputStream strSeg_dos;
+
+ /**
+ * convert between FSs and "sequential" numbers
+ * Note: This may be identity map, but may not in the case for V3 where some FSs are GC'd
+ */
+ private final Obj2IntIdentityHashMap<TOP> fs2seq = new Obj2IntIdentityHashMap<TOP>(TOP.class, TOP.singleton);
+ private final Int2ObjHashMap<TOP> seq2fs = new Int2ObjHashMap<>(TOP.class);
+ /**
+ *
+ * @param cas -
+ * @param serializedOut -
+ * @param mark -
+ * @param sm -
+ * @param compressLevel -
+ * @param compressStrategy -
+ */
private Serializer(CASImpl cas, DataOutputStream serializedOut, MarkerImpl mark,
SerializationMeasures sm,
CompressLevel compressLevel,
CompressStrat compressStrategy) {
- this.cas = cas;
+ this.baseCas = cas.getBaseCAS();
+ this.bcsd = cas.getBinaryCasSerDes();
+ this.csds = new CommonSerDesSequential(this.baseCas);
+// this.ccs = new CommonCompressedSerialization(
+// new CommonSerDesTypeMap(cas.getTypeSystemImpl(), cas.getTypeSystemImpl()), // no type mapping
+// mark);
this.serializedOut = serializedOut;
this.mark = mark;
this.sm = sm;
@@ -437,253 +467,173 @@ public class BinaryCasSerDes4 {
isDelta = (mark != null);
doMeasurement = (sm != null);
- heap = cas.getHeap().heap;
- heapEnd = cas.getHeap().getCellsUsed();
- heapStart = isDelta ? mark.getNextFSId() : 0;
-
- stringHeapObj = cas.getStringHeap();
- longHeapObj = cas.getLongHeap();
- shortHeapObj = cas.getShortHeap();
- byteHeapObj = cas.getByteHeap();
+// heap = cas.getHeap().heap;
+// heapEnd = cas.getHeap().getCellsUsed();
+
+//
+// stringHeapObj = cas.getStringHeap();
+// longHeapObj = cas.getLongHeap();
+// shortHeapObj = cas.getShortHeap();
+// byteHeapObj = cas.getByteHeap();
os = new OptimizeStrings(doMeasurement);
- setupOutputStreams();
- arrayLength_dos = dosZipSources[arrayLength_i];
- heapRef_dos = dosZipSources[heapRef_i];
- int_dos = dosZipSources[int_i];
+ BinaryCasSerDes6.setupOutputStreams(baseCas, baosZipSources, dosZipSources);
+
+// arrayLength_dos = dosZipSources[arrayLength_i];
+// heapRef_dos = dosZipSources[heapRef_i];
+// int_dos = dosZipSources[int_i];
byte_dos = dosZipSources[byte_i];
- short_dos = dosZipSources[short_i];
+// short_dos = dosZipSources[short_i];
typeCode_dos = dosZipSources[typeCode_i];
strOffset_dos = dosZipSources[strOffset_i];
strLength_dos = dosZipSources[strLength_i];
- long_High_dos = dosZipSources[long_High_i];
- long_Low_dos = dosZipSources[long_Low_i];
+// long_High_dos = dosZipSources[long_High_i];
+// long_Low_dos = dosZipSources[long_Low_i];
float_Mantissa_Sign_dos = dosZipSources[float_Mantissa_Sign_i];
float_Exponent_dos = dosZipSources[float_Exponent_i];
double_Mantissa_Sign_dos = dosZipSources[double_Mantissa_Sign_i];
double_Exponent_dos = dosZipSources[double_Exponent_i];
fsIndexes_dos = dosZipSources[fsIndexes_i];
- strChars_dos = dosZipSources[strChars_i];
+// strChars_dos = dosZipSources[strChars_i];
control_dos = dosZipSources[control_i];
strSeg_dos = dosZipSources[strSeg_i];
- }
-
- private void setupOutputStreams() {
- // estimate model:
- // 33% of space in strings, 33% in heap, 33% other
- // compr ratio for heap is 98%
- int compr = (heapEnd - heapStart) * 8 / 3 / 50;
- int compr1000 = Math.max(512, compr/1000);
- // 2nd arg is the number of bytes in the byte output stream, initially
- estimatedZipSize[typeCode_i] = Math.max(512, compr/4); // /4 for ~4 slots per fs
-// estimatedZipSize[boolean_i] =compr1000;
- estimatedZipSize[byte_i] = compr1000;
- estimatedZipSize[short_i] = compr1000;
- estimatedZipSize[int_i] = Math.max(1024, compr1000);
- estimatedZipSize[arrayLength_i] = compr1000;
- estimatedZipSize[float_Mantissa_Sign_i] = compr1000;
- estimatedZipSize[float_Exponent_i] = compr1000;
- estimatedZipSize[double_Mantissa_Sign_i] = compr1000;
- estimatedZipSize[double_Exponent_i] = compr1000;
- estimatedZipSize[long_High_i] = compr1000;
- estimatedZipSize[long_Low_i] = compr1000;
- estimatedZipSize[heapRef_i] = Math.max(1024, compr1000);
- estimatedZipSize[strOffset_i] = Math.max(512, compr/4);
- estimatedZipSize[strLength_i] = Math.max(512, compr/4);
- estimatedZipSize[fsIndexes_i] = Math.max(512, compr/8); // /4 for ~4 slots/fs, / 2 for # indexed
- estimatedZipSize[strChars_i] = Math.max(512, compr/4); // strings compress better
- estimatedZipSize[control_i] = 128;
- for (int i = 0; i < baosZipSources.length; i++) {
- setupOutputStream(i);
- }
-
+ this.prevFsByType = new TOP[ts.getTypeArraySize()];
}
+ /**
+ * Form 4 serialization is tied to the layout of V2 Feature Structures in heaps.
+ * It does not walk the indexes to serialize just those FSs that are reachable.
+ *
+ * For V3, it scans the CASImpl.id2fs information and serializes those (except those
+ * which have been GC'd). The seq numbers of the target incrementing sequentially will
+ * be different from the source id's if some FSs were GC'd.
+ *
+ * To determine for delta what new strings and new
+ *
+ * @throws IOException
+ */
private void serialize() throws IOException {
- if (doMeasurement) {
- System.out.println(printCasInfo(cas));
- sm.origAuxBytes = cas.getByteHeap().getSize();
- sm.origAuxShorts = cas.getShortHeap().getSize() * 2;
- sm.origAuxLongs = cas.getLongHeap().getSize() * 8;
- sm.totalTime = System.currentTimeMillis();
- }
-
- /******************
- * Process Header
- * Standardized
- ******************/
- // encode: bits 7 6 5 4 3 2 1 0
- // 0 0 1 = no delta, no compression
- // 0 1 - = delta, no compression
- // 1 d - = compression, w/wo delta
- int version = 4 | ((isDelta) ? 2 : 0);
- CASSerializer.outputVersion(version, serializedOut);
-
- serializedOut.writeInt(0); // reserved for future version info
- if (doMeasurement) {
- sm.header = 12;
+ // if (doMeasurement) {
+// System.out.println(printCasInfo(baseCas));
+// sm.origAuxBytes = baseCas.getByteHeap().getSize();
+// sm.origAuxShorts = baseCas.getShortHeap().getSize() * 2;
+// sm.origAuxLongs = baseCas.getLongHeap().getSize() * 8;
+// sm.totalTime = System.currentTimeMillis();
+// }
+
+ /************************
+ * Write standard header
+ ************************/
+ CommonSerDes.createHeader()
+ .v3()
+ .seqVer(2) // 0 - original, 1 - UIMA-4743, 2 - v3
+ .form4()
+ .delta(isDelta)
+ .write(serializedOut);
+
+ /*******************************************************************************
+ * Setup tables that map to v2 "addresses" - needed for backwards compatibility
+ * fs2addr - feature structure to address
+ * addr2fs - address to feature structure
+ * sortedFSs - sorted by addr (sorted by id)
+ *******************************************************************************/
+ csds.setup();
+
+ fs2seq.clear();
+ seq2fs.clear();
+ int seq = 1; // origin 1
+ for (TOP fs : csds.sortedFSs) {
+ fs2seq.put(fs, seq);
+ seq2fs.put(seq++, fs);
+ }
+
+ final List<TOP> newSortedFSs; // a sublist for delta cas of just the new ones
+ // may be null if there are no FSs to serialize
+ TOP firstFS = isDelta ? baseCas.getFsFromId(mark.getNextFSId()) : (csds.sortedFSs.size() == 0) ? null : csds.sortedFSs.get(0);
+ if (firstFS != null) {
+ if (isDelta) {
+ int i = Collections.binarySearch(csds.sortedFSs, firstFS);
+ assert(i >= 0);
+ newSortedFSs = csds.sortedFSs.subList(i, csds.sortedFSs.size());
+ } else {
+ newSortedFSs = csds.sortedFSs;
+ }
+ } else {
+ newSortedFSs = Collections.emptyList();
}
-
+
+
/**************************
* Strings
+ * For delta, to determine "new" strings that should be serialized,
+ * use the same method as used in Binary (plain) serialization.
**************************/
- int stringHeapStart = isDelta ? mark.nextStringHeapAddr : 1;
- int stringHeapEnd = stringHeapObj.getSize();
-
- for (int i = stringHeapStart; i < stringHeapEnd; i++) {
- os.add(stringHeapObj.getStringForCode(i));
+ for (TOP fs : newSortedFSs) {
+ extractStrings(fs);
}
-
- // also add in all modified strings
- // with current design, all modified strings are guaranteed
- // to be above the mark, so this code is commented out
-// addModStr:
-// if (isDelta) {
-// serializerForModifiedFSs.sortModifications();
-// final int[] mods = serializerForModifiedFSs.modifiedMainHeapAddrs;
-// final int modLen = serializerForModifiedFSs.modMainHeapAddrsLength;
-// int nextMod = 0;
-// if (modLen > 0) {
-//
-// for (int iHeap = 1; iHeap < heapStart; iHeap += incrToNextFs(heap, iHeap, typeInfo)) {
-// typeInfo = getTypeInfo(heap[iHeap]);
-// FsStringRefs fsStringRefs = new FsStringRefs(typeInfo, heap, iHeap);
-// for (int nextRef = fsStringRefs.next(); nextRef >= 0; nextRef = fsStringRefs.next()) {
-// while (nextRef > mods[nextMod]) {
-// nextMod ++;
-// if (nextMod == modLen) {
-// break addModStr;
-// }
-// }
-// if (nextRef == mods[nextMod]) {
-// os.add(stringHeapObj.getStringForCode(heap[nextRef]));
-// nextMod ++;
-// if (nextMod == modLen) {
-// break addModStr;
-// }
-// }
-// }
-// }
-// }
-// }
- os.optimize();
- String [] commonStrings = os.getCommonStrings();
- writeVnumber(strChars_i, commonStrings.length);
- for (int i = 0; i < commonStrings.length; i++) {
- int startPos = dosZipSources[strChars_i].size();
- DataIO.writeUTFv(commonStrings[i], dosZipSources[strChars_i]);
- // approximate histogram
- if (doMeasurement) {
- // len is utf-8 encoding
- float len = dosZipSources[strChars_i].size() - startPos;
- // if len == chars, then all got coded as 1 byte
- // if len > chars, some were utf-8 coded as 2 bytes
- float excess = (len / commonStrings[i].length()) - 1; // excess over length 1
- int encAs2 = (int)(excess * commonStrings[i].length());
-
- // simulate histo for all the chars, as 1 or 2 byte UTF8 encoding
- sm.statDetails[strChars_i].countTotal += commonStrings[i].length(); // total chars accum
- sm.statDetails[strChars_i].c[0] = commonStrings[i].length() - encAs2;
- sm.statDetails[strChars_i].c[1] = encAs2;
- sm.statDetails[strChars_i].lengthTotal += len; // total as UTF-8 encode
+ if (isDelta) {
+ FsChange[] fssModified = baseCas.getModifiedFSList();
+
+ // also add in all modified strings
+ for (FsChange fsChange : fssModified) {
+ extractStringsFromModifications(fsChange);
}
}
-
- only1CommonString = commonStrings.length == 1;
- if (doMeasurement) {
- long commonStringsLength = 0;
- sm.stringsNbrCommon = commonStrings.length;
- int r = 0;
- for (int i = 0; i < commonStrings.length; i++) {
- r += DataIO.lengthUTFv(commonStrings[i]);
- commonStringsLength += commonStrings[i].length();
- }
- sm.stringsCommonChars = r;
-
- sm.stringsSavedExact = os.getSavedCharsExact() * 2;
- sm.stringsSavedSubstr = os.getSavedCharsSubstr() * 2;
- sm.statDetails[strChars_i].original = os.getSavedCharsExact() * 2
- + os.getSavedCharsSubstr() * 2
- + commonStringsLength * 2;
- sm.statDetails[strLength_i].original = (stringHeapEnd - stringHeapStart) * 4;
- sm.statDetails[strOffset_i].original = (stringHeapEnd - stringHeapStart) * 4;
- }
+ os.optimize();
+ writeStringInfo(os);
+
/***************************
* Prepare to walk main heap
***************************/
- writeVnumber(control_dos, heapEnd - heapStart);
- if (doMeasurement) {
- sm.statDetails[Slot_MainHeap.i].original = (1 + heapEnd - heapStart) * 4;
+ heapEnd = csds.heapEnd;
+
+
+ if (isDelta) {
+ // edge case - delta serializing with no new fs
+ heapStart = (null == firstFS) ? heapEnd : csds.fs2addr.get(firstFS);
+ } else {
+ heapStart = 0; // not 1, in order to match v2 semantics
+ // is switched to 1 later
}
- resetIprevious();
+// if (isDelta) {
+// // debug
+// for (TOP fs : csds.sortedFSs) {
+// System.out.format("debug heapAddr: %,d type: %s%n", csds.fs2addr.get(fs), fs._typeImpl.getShortName());
+// if (csds.fs2addr.get(fs) == 439) {
+// System.out.println("debug");
+// }
+// }
+// System.out.format("debug End of debug scan, heapStart: %,d heapEnd: %,d%n%n", heapStart, heapEnd);
+// }
+
+ writeVnumber(control_dos, heapEnd - heapStart); // used for delta heap size to grow the CAS and ending condition on deser loop
+
+ Arrays.fill(prevFsByType, null);
+
+// if (heapStart == 0) {
+// heapStart = 1; // slot 0 not serialized, it's null / 0
+// }
- if (heapStart == 0) {
- heapStart = 1; // slot 0 not serialized, it's null / 0
- }
- if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
// scan thru all fs and save their offsets in the heap
// to allow conversion from addr to sequential fs numbers
- initFsStartIndexes(fsStartIndexes, heap, heapStart, heapEnd, typeCodeHisto);
-
- for (int i = ts.getTypeArraySize() - 1; i >= 0; i--) {
- serializedTypeCode2Code[i] = i;
- }
-
- // set typeCode2serializeCode so that the 0th element is the typeCode with the highest frequency, etc.
- Arrays.sort(serializedTypeCode2Code, 0, serializedTypeCode2Code.length, new Comparator<Integer>() {
- public int compare(Integer o1, Integer o2) {
- return (typeCodeHisto[o1] > typeCodeHisto[o2]) ? -1 :
- (typeCodeHisto[o1] < typeCodeHisto[o2]) ? 1 : 0;
- }
- });
-
-// for (int i = 0; i < serializedTypeCode2Code.length; i++) {
-// int tCode = serializedTypeCode2Code[i];
-// int c = typeCodeHisto[tCode];
-// if (c > 0) {
-// System.out.format("%2d %,9d instance of Type %s%n", i, c, typeInfoArray[tCode]);
-// }
-// }
+// initFsStartIndexes(fsStartIndexes, heap, heapStart, heapEnd, typeCodeHisto);
- }
-
-
/***************************
- * walk main heap
+ * walk all fs's
+ * For delta, just those above the line
***************************/
-
- for (int iHeap = heapStart; iHeap < heapEnd; iHeap += incrToNextFs(heap, iHeap, typeInfo)) {
- int tCode = heap[iHeap]; // get type code
- typeInfo = getTypeInfo(tCode);
- iPrevHeap = typeInfo.iPrevHeap;
-
- writeVnumber(typeCode_dos, tCode);
-
- if (typeInfo.isHeapStoredArray) {
- serializeHeapStoredArray(iHeap);
- } else if (typeInfo.isArray) {
- serializeNonHeapStoredArray(iHeap);
- } else {
- for (int i = 1; i < typeInfo.slotKinds.length + 1; i++) {
- serializeByKind(iHeap, i);
- }
- }
-
- typeInfo.iPrevHeap = iHeap; // make this one the "prev" one for subsequent testing
- if (doMeasurement) {
- sm.statDetails[typeCode_i].incr(DataIO.lengthVnumber(tCode));
- sm.mainHeapFSs ++;
- }
- } // end of heap walk
+ for (TOP fs : newSortedFSs) {
+ writeFs(fs);
+ }
serializeIndexedFeatureStructures();
@@ -698,9 +648,84 @@ public class BinaryCasSerDes4 {
}
}
+ /**
+ * Write the compressed string table(s)
+ * @throws IOException
+ */
+ private void writeStringInfo(OptimizeStrings os) throws IOException {
+ String [] commonStrings = os.getCommonStrings();
+ writeVnumber(strChars_i, commonStrings.length);
+ DataOutputStream out = dosZipSources[strChars_i];
+ for (int i = 0; i < commonStrings.length; i++) {
+ int startPos = doMeasurements ? out.size() : 0;
+ DataIO.writeUTFv(commonStrings[i], out);
+ // approximate histogram
+ if (doMeasurements) {
+ // len is utf-8 encoding
+ float len = out.size() - startPos;
+ // if len == chars, then all got coded as 1 byte
+ // if len > chars, some were utf-8 coded as 2 bytes
+ float excess = (len / commonStrings[i].length()) - 1; // excess over length 1
+ int encAs2 = (int)(excess * commonStrings[i].length());
+
+ // simulate histo for all the chars, as 1 or 2 byte UTF8 encoding
+ sm.statDetails[strChars_i].countTotal += commonStrings[i].length(); // total chars accum
+ sm.statDetails[strChars_i].c[0] = commonStrings[i].length() - encAs2;
+ sm.statDetails[strChars_i].c[1] = encAs2;
+ sm.statDetails[strChars_i].lengthTotal += len; // total as UTF-8 encode
+ }
+ }
+
+ only1CommonString = commonStrings.length == 1;
+
+ if (doMeasurements) {
+// long commonStringsLength = 0;
+// sm.stringsNbrCommon = commonStrings.length;
+// int r = 0;
+// for (int i = 0; i < commonStrings.length; i++) {
+// r += DataIO.lengthUTFv(commonStrings[i]);
+// commonStringsLength += commonStrings[i].length();
+// }
+// sm.stringsCommonChars = r;
+ //
+// sm.stringsSavedExact = os.getSavedCharsExact() * 2;
+// sm.stringsSavedSubstr = os.getSavedCharsSubstr() * 2;
+// sm.statDetails[strChars_i].original = os.getSavedCharsExact() * 2
+// + os.getSavedCharsSubstr() * 2
+// + commonStringsLength * 2;
+// final int stringHeapStart = isSerializingDelta ? mark.nextFSId : 1;
+// final int stringHeapEnd = stringHeapObj.getSize();
+// sm.statDetails[strLength_i].original = (stringHeapEnd - stringHeapStart) * 4;
+// sm.statDetails[strOffset_i].original = (stringHeapEnd - stringHeapStart) * 4;
+ }
+
+ }
+
+ private void writeFs(TOP fs) throws IOException {
+ TypeImpl type = fs._typeImpl;
+ int typeCode = type.getCode();
+ writeVnumber(typeCode_dos, typeCode);
+
+ prevFs = prevFsByType[typeCode];
+
+ if (type.isArray()) {
+ serializeArray(fs);
+ } else {
+ for (FeatureImpl feat : type.getFeatureImpls()) {
+ serializeByKind(fs, feat);
+ }
+ }
+
+ prevFsByType[typeCode] = fs;
+// if (doMeasurement) {
+// sm.statDetails[typeCode_i].incr(DataIO.lengthVnumber(tCode));
+// sm.mainHeapFSs ++;
+// }
+ }
private void serializeIndexedFeatureStructures() throws IOException {
- int[] fsIndexes = isDelta ? cas.getDeltaIndexedFSs(mark) : cas.getIndexedFSs();
+ // fsIndexes already have the modelled address conversion
+ int[] fsIndexes = isDelta ? bcsd.getDeltaIndexedFSs(mark, csds.fs2addr) : bcsd.getIndexedFSs(csds.fs2addr);
if (doMeasurement) {
sm.statDetails[fsIndexes_i].original = fsIndexes.length * 4 + 1;
}
@@ -746,13 +771,10 @@ public class BinaryCasSerDes4 {
System.arraycopy(fsIndexes, ix, ia, 0, nbrEntries);
Arrays.sort(ia);
- int prev = 0;
-
+ int prev = 0;
for (int i = 0; i < ia.length; i++) {
int v = ia[i];
- if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
- v = fsStartIndexes.getItemIndex(v);
- }
+ v = fs2seq(csds.addr2fs.get(v)); // v2 wrote the sequential number
writeVnumber(fsIndexes_dos, v - prev);
if (doMeasurement) {
sm.statDetails[fsIndexes_i].incr(DataIO.lengthVnumber(v - prev));
@@ -763,124 +785,158 @@ public class BinaryCasSerDes4 {
return end;
}
- private void serializeHeapStoredArray(int iHeap) throws IOException {
- final int length = serializeArrayLength(iHeap);
- // output values
+ private void serializeArray(TOP fs) throws IOException {
+ final int length = serializeArrayLength(fs);
// special case 0 and 1st value
if (length == 0) {
return;
}
- SlotKind arrayElementKind = typeInfo.slotKinds[1];
- final int endi = iHeap + length + 2;
+ final TypeImpl type = fs._typeImpl;
+
+ // output values
+
+ SlotKind arrayElementKind = type.getComponentSlotKind();
switch (arrayElementKind) {
- case Slot_HeapRef: case Slot_Int: case Slot_Short:
- {
- int prev = (iPrevHeap == 0) ? 0 :
- (heap[iPrevHeap + 1] == 0) ? 0 :
- heap[iPrevHeap + 2];
- for (int i = iHeap + 2; i < endi; i++) {
- prev = writeIntOrHeapRef(arrayElementKind.i, i, prev);
- }
+ case Slot_HeapRef: {
+ int prev = getPrevArray0HeapRef();
+ for (TOP item : ((FSArray)fs)._getTheArray()) {
+ int v = fs2seq(item);
+ writeDiff(arrayElementKind.ordinal(), v, prev);
+ prev = v;
+ }
+ break;
+ }
+ case Slot_Int: {
+ int prev = getPrevArray0Int();
+ for (int item : ((IntegerArray)fs)._getTheArray()) {
+ writeDiff(arrayElementKind.ordinal(), item, prev);
+ prev = item;
+ }
+ break;
+ }
+
+ case Slot_ShortRef: {
+ int prev = 0;
+ for (int item : ((ShortArray)fs)._getTheArray()) {
+ writeDiff(short_i, item, prev);
+ prev = item;
}
break;
- case Slot_Float:
- for (int i = iHeap + 2; i < endi; i++) {
- writeFloat(heap[i]);
+ }
+ case Slot_Float:
+ for (float item : ((FloatArray)fs)._getTheArray()) {
+ writeFloat(CASImpl.float2int(item));
}
break;
+
case Slot_StrRef:
- for (int i = iHeap + 2; i < endi; i++) {
- writeString(stringHeapObj.getStringForCode(heap[i]));
+ for (String item : ((StringArray)fs)._getTheArray()) {
+ writeString(item);
}
break;
+
+ case Slot_BooleanRef:
+ for (boolean item : ((BooleanArray)fs)._getTheArray()) {
+ byte_dos.write(item ? 1 : 0);
+ }
+ break;
+
+ case Slot_ByteRef:
+ byte_dos.write(((ByteArray)fs)._getTheArray());
+ break;
- default: throw new RuntimeException("internal error");
+ case Slot_LongRef: {
+ long prev = 0;
+ for (long item : ((LongArray)fs)._getTheArray()) {
+ writeLong(item, prev);
+ prev = item;
+ }
+ break;
+ }
+
+ case Slot_DoubleRef:
+ for (double item : ((DoubleArray)fs)._getTheArray()) {
+ writeDouble(CASImpl.double2long(item));
+ }
+ break;
+
+ default: Misc.internalError();
} // end of switch
}
- private int writeIntOrHeapRef(int kind, int index, int prev) throws IOException {
- final int v = heap[index];
- writeDiff(kind, v, prev);
- return v;
+ private int getPrevArray0HeapRef() {
+ if (isNoPrevArrayValue((CommonArray)prevFs)) return 0;
+ return fs2seq(((FSArray)prevFs).get(0));
}
- private long writeLongFromHeapIndex(int index, long prev) throws IOException {
- final long v = longHeapObj.getHeapValue(heap[index]);
- writeLong(v, prev);
- return v;
+ private int getPrevArray0Int() {
+ if (isNoPrevArrayValue((CommonArray)prevFs)) return 0;
+ return ((IntegerArray)prevFs).get(0);
}
-
- private void serializeNonHeapStoredArray(int iHeap) throws IOException {
- final int length = serializeArrayLength(iHeap);
- if (length == 0) {
- return;
- }
- SlotKind refKind = typeInfo.getSlotKind(2);
- switch (refKind) {
- case Slot_BooleanRef: case Slot_ByteRef:
- writeFromByteArray(refKind, heap[iHeap + 2], length);
- if (doMeasurement) {
- sm.statDetails[byte_i].incr(1);
- sm.origAuxByteArrayRefs += 4;
- }
- break;
- case Slot_ShortRef:
- writeFromShortArray(heap[iHeap + 2], length);
- if (doMeasurement) {
- sm.origAuxShortArrayRefs += 4;
- }
- break;
- case Slot_LongRef: case Slot_DoubleRef:
- writeFromLongArray(refKind, heap[iHeap + 2], length);
- if (doMeasurement) {
- sm.origAuxLongArrayRefs += 4;
- }
- break;
- default:
- throw new RuntimeException();
- }
+
+ private boolean isNoPrevArrayValue(CommonArray prevFs) {
+ return prevFs == null || prevFs.size() == 0;
}
-
- private void serializeByKind(int iHeap, int offset) throws IOException {
- SlotKind kind = typeInfo.getSlotKind(offset);
+
+ private void serializeByKind(TOP fs, FeatureImpl feat) throws IOException {
+ SlotKind kind = feat.getSlotKind();
switch (kind) {
- //Slot_Int, Slot_Float, Slot_Boolean, Slot_Byte, Slot_Short
- case Slot_Int: case Slot_Short: case Slot_HeapRef:
- serializeDiffWithPrevTypeSlot(kind, iHeap, offset);
+ case Slot_Int: {
+ final int prev = (prevFs == null) ? 0 : prevFs.getIntValueNc(feat);
+ final int v = fs.getIntValue(feat);
+// if (TRACE_INT) System.out.format("writeInt value: %,d prev: %,d%n", v, prev);
+ writeDiff(kind.ordinal(), v, prev);
+ break;
+ }
+
+ case Slot_Short:
+ writeDiff(kind.ordinal(), fs.getShortValueNc(feat), (prevFs == null) ? 0 : prevFs.getShortValueNc(feat));
+ break;
+
+ case Slot_HeapRef:
+ final TOP ref = fs.getFeatureValueNc(feat);
+ writeDiff(kind.ordinal(), fs2seq(ref),
+ (prevFs == null) ? 0 : fs2seq(prevFs.getFeatureValueNc(feat)));
break;
+
case Slot_Float:
- writeFloat(heap[iHeap + offset]);
+ writeFloat(CASImpl.float2int(fs.getFloatValueNc(feat)));
+ break;
+
+ case Slot_Boolean:
+ byte_dos.write(fs.getBooleanValueNc(feat) ? 1 : 0);
break;
- case Slot_Boolean: case Slot_Byte:
- byte_dos.write(heap[iHeap + offset]);
+
+ case Slot_Byte:
+ byte_dos.write(fs.getByteValueNc(feat));
break;
+
case Slot_StrRef:
- writeString(stringHeapObj.getStringForCode(heap[iHeap + offset]));
+ writeString(fs.getStringValueNc(feat));
break;
- case Slot_LongRef:
- writeLongFromHeapIndex(iHeap + offset,
- (iPrevHeap == 0) ?
- 0L :
- longHeapObj.getHeapValue(heap[iPrevHeap + offset]));
+
+ case Slot_LongRef:
+ writeLong(fs.getLongValueNc(feat), (prevFs == null) ? 0L : prevFs.getLongValueNc(feat));
break;
+
case Slot_DoubleRef:
- writeDouble(longHeapObj.getHeapValue(heap[iHeap + offset]));
+ writeDouble(CASImpl.double2long(fs.getDoubleValueNc(feat)));
break;
- default:
- throw new RuntimeException("internal error");
+
+ default: Misc.internalError();
} // end of switch
}
- private int serializeArrayLength(int iHeap) throws IOException {
- final int length = heap[iHeap + 1];
+ private int serializeArrayLength(TOP fs) throws IOException {
+ int length = ((CommonArray)fs).size();
writeVnumber(arrayLength_i, length);
return length;
}
- private void serializeDiffWithPrevTypeSlot(SlotKind kind, int iHeap, int offset) throws IOException {
- int prev = (iPrevHeap == 0) ? 0 : heap[iPrevHeap + offset];
- writeDiff(kind.i, heap[iHeap + offset], prev);
- }
+// private void serializeDiffWithPrevTypeSlot(SlotKind kind, TOP fs, FeatureImpl feat) throws IOException {
+// int prev = (prevFs == null) ? 0 : fs.setIntLikeValue(slotKind, fi, v);getheap[iPrevHeap + offset];
+// writeDiff(kind.ordinal(), heap[iHeap + offset], prev);
+// }
/**
* Method:
@@ -945,13 +1001,6 @@ public class BinaryCasSerDes4 {
// }
// return dos;
// }
-
- public DataOutputStream setupOutputStream(int i) {
- // set up output stream
- int size = estimatedZipSize[i];
- baosZipSources[i] = new ByteArrayOutputStream(size);
- return dosZipSources[i] = new DataOutputStream(baosZipSources[i]);
- }
private void writeLong(long v, long prev) throws IOException {
writeDiff(long_High_i, (int)(v >>> 32), (int)(prev >>> 32));
@@ -1097,6 +1146,9 @@ public class BinaryCasSerDes4 {
final long mants = (revMants << 1) + ((raw < 0) ? 1 : 0);
writeVnumber(double_Exponent_dos, exponent);
writeVnumber(double_Mantissa_Sign_dos, mants);
+ if (TRACE_DOUBLE) {
+ System.out.format("write Double: raw = %,d, exponent = %,d, mantissa + lowbit sign: %,d%n", raw, exponent, mants);
+ }
if (doMeasurement) {
sm.statDetails[double_Exponent_i].incr(DataIO.lengthVnumber(exponent));
sm.statDetails[double_Mantissa_Sign_i].incr(DataIO.lengthVnumber(mants));
@@ -1137,13 +1189,6 @@ public class BinaryCasSerDes4 {
return;
}
- if (CHANGE_FS_REFS_TO_SEQUENTIAL && (kind == heapRef_i)) {
- v = fsStartIndexes.getItemIndex(v);
- if (prev != 0) {
- prev = fsStartIndexes.getItemIndex(prev);
- }
- }
-
final int absV = Math.abs(v);
if (((v > 0) && (prev > 0)) ||
((v < 0) && (prev < 0))) {
@@ -1171,37 +1216,60 @@ public class BinaryCasSerDes4 {
sm.statDetails[kind].valueLeDiff ++;
}
}
-
- private void writeFromByteArray(SlotKind kind, int startPos, int length) throws IOException {
- byte_dos.write(byteHeapObj.heap, startPos, length);
- }
-
- private void writeFromLongArray(SlotKind kind, int startPos, int length) throws IOException {
- final long[] h = longHeapObj.heap;
- final int endPos = startPos + length;
- long prev = 0;
- for (int i = startPos; i < endPos; i++) {
- final long e = h[i];
- if (kind == Slot_DoubleRef) {
- writeDouble(e);
- } else {
- writeLong(e, prev);
- prev = e;
- }
+
+ /**
+ * add strings to the optimizestrings object
+ *
+ * If delta, only process for fs's that are new;
+ * modified string values picked up when scanning FsChange items
+ * @param fs feature structure
+ */
+ private void extractStrings(TOP fs) {
+ if (isDelta && !mark.isNew(fs)) {
+ return;
}
+ TypeImpl type = fs._typeImpl;
+
+ if (type.isArray()) {
+ if (type.getComponentSlotKind() == SlotKind.Slot_StrRef) {
+ for (String s : ((StringArray)fs)._getTheArray()) {
+ os.add(s);
+ }
+ }
+ } else { // end of is-array
+ for (FeatureImpl feat : type.getFeatureImpls()) {
+ if (feat.getSlotKind() == SlotKind.Slot_StrRef) {
+ os.add(fs.getStringValueNc(feat));
+ }
+ } // end of iter over all features
+ } // end of if-is-not-array
}
- private void writeFromShortArray(int startPos, int length) throws IOException {
- final short[] h = shortHeapObj.heap;
- final int endPos = startPos + length;
- int prev = 0;
- for (int i = startPos; i < endPos; i++) {
- final short e = h[i];
- writeDiff(short_i, e, prev);
- prev = e;
- }
+ /**
+ * For delta, for each fsChange element, extract any strings
+ * @param fsChange
+ */
+ private void extractStringsFromModifications(FsChange fsChange) {
+ final TOP fs = fsChange.fs;
+ final TypeImpl type = fs._typeImpl;
+ if (fsChange.arrayUpdates != null) {
+ if (type.getComponentSlotKind() == SlotKind.Slot_StrRef) {
+ String[] sa = ((StringArray)fs)._getTheArray();
+ fsChange.arrayUpdates.forAllInts(index -> {
+ os.add(sa[index]);
+ });
+ } // end of is string array
+ } else { // end of is array
+ BitSet fm = fsChange.featuresModified;
+ for (int offset = fm.nextSetBit(0); offset >= 0; offset = fm.nextSetBit(offset + 1)) {
+ FeatureImpl feat = type.getFeatureImpls().get(offset);
+ if (feat.getSlotKind() == SlotKind.Slot_StrRef) {
+ os.add(fs.getStringValueNc(feat));
+ }
+ } // end of iter over features
+ } // end of is-not-array
}
-
+
/******************************************************************************
* Modified Values
* Output:
@@ -1209,47 +1277,24 @@ public class BinaryCasSerDes4 {
* write the heap addr of the FS
*
* For all modified values within the FS:
- * if it is an aux array element, write the index in the aux array and the new value
+ * if it is an aux array element, write the index in the individual array instance and the new value
* otherwise, write the slot offset and the new value
******************************************************************************/
public class SerializeModifiedFSs {
- final int[] modifiedMainHeapAddrs = cas.getModifiedFSHeapAddrs().toArray();
- final int[] modifiedFSs = cas.getModifiedFSList().toArray();
- final int[] modifiedByteHeapAddrs = cas.getModifiedByteHeapAddrs().toArray();
- final int[] modifiedShortHeapAddrs = cas.getModifiedShortHeapAddrs().toArray();
- final int[] modifiedLongHeapAddrs = cas.getModifiedLongHeapAddrs().toArray();
-
- {sortModifications();}
-
- final int modMainHeapAddrsLength = eliminateDuplicatesInMods(modifiedMainHeapAddrs);
- final int modFSsLength = eliminateDuplicatesInMods(modifiedFSs);
- final int modByteHeapAddrsLength = eliminateDuplicatesInMods(modifiedByteHeapAddrs);
- final int modShortHeapAddrsLength = eliminateDuplicatesInMods(modifiedShortHeapAddrs);
- final int modLongHeapAddrsLength = eliminateDuplicatesInMods(modifiedLongHeapAddrs);
-
- // ima - index into modified arrays
- // ixx, iPrevxxx - index in heap being changed
- // value comes via the main heap or aux heaps
-
- int imaModMainHeap = 0;
- int imaModByteRef = 0;
- int imaModShortRef = 0;
- int imaModLongRef = 0;
-
// previous value - for things diff encoded
int vPrevModInt = 0;
int vPrevModHeapRef = 0;
short vPrevModShort = 0;
long vPrevModLong = 0;
-
- int iHeap;
- TypeInfo typeInfo;
-
+
private void serializeModifiedFSs() throws IOException {
- iPrevHeap = 0;
+
+ int iPrevAddr = 0;
+
+ FsChange[] fsChanges = baseCas.getModifiedFSList();
// write out number of modified Feature Structures
- writeVnumber(control_dos, modFSsLength);
+ writeVnumber(control_dos, fsChanges.length);
// iterate over all modified feature structures
/**
* Theorems about these data
@@ -1259,196 +1304,199 @@ public class BinaryCasSerDes4 {
* 3) Assumption: String array element modifications are main heap slot changes
* and recorded as such
*/
-
- for (int i = 0; i < modFSsLength; i++) {
- iHeap = modifiedFSs[i];
- final int tCode = heap[iHeap];
- typeInfo = getTypeInfo(tCode);
-
+
+ for (FsChange fsChange : fsChanges) {
+
+ TOP fs = fsChange.fs;
+ TypeImpl ti = fs._typeImpl;
+ final int addr = csds.fs2addr.get(fs);
// write out the address of the modified FS
- writeVnumber(fsIndexes_dos, iHeap - iPrevHeap);
- // delay updating iPrevHeap until end of "for" loop
+ writeVnumber(fsIndexes_dos, addr - iPrevAddr);
+ // delay updating iPrevAddr until end of "for" loop
/**************************************************
* handle aux byte, short, long array modifications
**************************************************/
- if (typeInfo.isArray && (!typeInfo.isHeapStoredArray)) {
- writeAuxHeapMods();
+ if (ti.isArray() && !ti.isHeapStoredArray()) {
+ writeAuxHeapMods(fsChange);
} else {
- writeMainHeapMods();
+ writeMainHeapMods(fsChange);
} // end of processing 1 modified FS
- iPrevHeap = iHeap;
+ iPrevAddr = addr;
} // end of for loop over all modified FSs
} // end of method
-
- // sort and remove duplicates
- private void sortModifications() {
- Arrays.sort(modifiedMainHeapAddrs);
- Arrays.sort(modifiedFSs);
- Arrays.sort(modifiedByteHeapAddrs);
- Arrays.sort(modifiedShortHeapAddrs);
- Arrays.sort(modifiedLongHeapAddrs);
- }
-
- private int eliminateDuplicatesInMods(final int[] sorted) {
- int length = sorted.length;
- if (length < 2) {
- return length;
- }
-
- int prev = sorted[0];
- int to = 1;
- for(int from = 1; from < length; from++) {
- int s = sorted[from];
- if (s == prev) {
- continue;
- }
- prev = s;
- sorted[to] = s;
- to++;
- }
- return to; // to is length
- }
-
- private int countModifiedSlotsInFs(int fsLength) {
- return countModifiedSlots(iHeap, fsLength, modifiedMainHeapAddrs, imaModMainHeap, modMainHeapAddrsLength);
- }
-
- private int countModifiedSlotsInAuxHeap(int[] modifiedAddrs, int indexInModAddrs, int length) {
- return countModifiedSlots(heap[iHeap + 2], heap[iHeap + 1], modifiedAddrs, indexInModAddrs, length);
- }
-
- private int countModifiedSlots(int firstAddr, int length, int[] modifiedAddrs, int indexInModAddrs, int modAddrsLength) {
- if (0 == length) {
- throw new RuntimeException(); // can't happen
- }
- final int nextAddr = firstAddr + length;
- int nextModAddr = modifiedAddrs[indexInModAddrs];
- if ((firstAddr > nextModAddr) ||
- (nextModAddr >= nextAddr)) {
- throw new RuntimeException(); // never happen - must have one slot at least modified in this fs
- }
- int i = 1;
- for (;; i++) {
- if ((indexInModAddrs + i) == modAddrsLength) {
- break;
- }
- nextModAddr = modifiedAddrs[indexInModAddrs + i];
- if (nextModAddr >= nextAddr) {
- break;
- }
- }
- return i;
- }
-
- private void writeMainHeapMods() throws IOException {
- final int fsLength = incrToNextFs(heap, iHeap, typeInfo);
- final int numberOfModsInFs = countModifiedSlotsInFs(fsLength);
- writeVnumber(fsIndexes_dos, numberOfModsInFs);
- int iPrevOffsetInFs = 0;
-
- for (int i = 0; i < numberOfModsInFs; i++) {
- final int nextMainHeapIndex = modifiedMainHeapAddrs[imaModMainHeap++];
- final int offsetInFs = nextMainHeapIndex - iHeap;
-
- writeVnumber(fsIndexes_dos, offsetInFs - iPrevOffsetInFs);
- iPrevOffsetInFs = offsetInFs;
+
+ private void writeMainHeapMods(FsChange fsChange) throws IOException {
+ int nbrOfMods = (fsChange.arrayUpdates == null)
+ ? fsChange.featuresModified.cardinality()
+ : fsChange.arrayUpdates.size();
+ writeVnumber(fsIndexes_dos, nbrOfMods);
+
+ final TOP fs = fsChange.fs;
+
+ if (fsChange.arrayUpdates == null) {
+ List<FeatureImpl> features = fs._typeImpl.getFeatureImpls();
+ int iPrevOffsetInFs = 0;
+ final BitSet bs = fsChange.featuresModified;
+ for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i+1)) {
+ FeatureImpl feat = features.get(i);
+ // next +1 to conform to v2 encoding of feat offsets
+ writeVnumber(fsIndexes_dos, i + 1 - iPrevOffsetInFs);
+ iPrevOffsetInFs = i + 1;
+
+ final SlotKind kind = feat.getSlotKind();
+ final int kindi = kind.ordinal();
+
+ switch (kind) {
+ case Slot_Boolean:
+ byte_dos.write(fs.getBooleanValueNc(feat) ? 1 : 0);
+ break;
+
+ case Slot_Byte:
+ byte_dos.write(fs.getByteValueNc(feat));
+ break;
+
+ case Slot_Short: {
+ final short v = fs.getShortValueNc(feat);
+ writeDiff(kindi, v, vPrevModShort);
+ vPrevModShort = v;
+ break;
+ }
+
+ case Slot_Int: {
+ final int v = fs.getIntValueNc(feat);
+ writeDiff(kindi, v, vPrevModInt);
+ vPrevModInt = v;
+ break;
+ }
+
+ case Slot_Float:
+ writeFloat(CASImpl.float2int(fs.getFloatValueNc(feat)));
+ break;
+
+ case Slot_LongRef: {
+ long v = fs.getLongValueNc(feat);
+ writeLong(v, vPrevModLong);
+ vPrevModLong = v;
+ break;
+ }
+
+ case Slot_DoubleRef:
+ writeDouble(CASImpl.double2long(fs.getDoubleValueNc(feat)));
+ break;
+
+ case Slot_HeapRef: {
+ int v = fs2seq(fs.getFeatureValueNc(feat));
+ writeDiff(kindi, v, vPrevModHeapRef);
+ vPrevModHeapRef = v;
+ break;
+ }
+
+ case Slot_StrRef:
+ writeString(fs.getStringValueNc(feat));
+ break;
+
+ default: Misc.internalError();
+
+ } // end of switch
+ } // end of looping for all modified slots in this FS
+ } else { // end of processing of features
+ // heap stored arrays
+ TypeImpl type = fs._typeImpl;
+ SlotKind kind = type.getComponentSlotKind();
+ int kindi = kind.ordinal();
- final SlotKind kind = typeInfo.getSlotKind(typeInfo.isArray ? 2 : offsetInFs);
+ IntListIterator it = fsChange.arrayUpdates.iterator();
+ while (it.hasNext()) {
+ int i = it.next();
+ // write the offset of the of the modified entry
+ // from the beginning of the fs addr
+ // i is already the 0 based offset, make it a 2 based one
+ // to account for the type code and length in v2 layout
+ writeVnumber(fsIndexes_dos, i + 2);
+
+ switch(kind) {
+ case Slot_Int: {
+ final int v = ((IntegerArray)fs).get(i);
+ writeDiff(kindi, v, vPrevModInt);
+ vPrevModInt = v;
+ break;
+ }
+
+ case Slot_Float:
+ writeFloat(CASImpl.float2int(((FloatArray)fs).get(i)));
+ break;
+
+ case Slot_StrRef:
+ writeString(((StringArray)fs).get(i));
+ break;
+
+ case Slot_HeapRef:
+ int v = fs2seq(((FSArray)fs).get(i));
+ writeDiff(kindi, v, vPrevModHeapRef);
+ vPrevModHeapRef = v;
+ break;
+
+ default: Misc.internalError();
+ } // end of switch
+ } // end of iteration over all changed slots in one array
+ } // end of if statement for processing arrays
+ } // end of method
+
+ private void writeAuxHeapMods(FsChange fsChange) throws IOException {
+ final TOP fs = fsChange.fs;
+ final TypeImpl type = fs._typeImpl;
+
+ int iPrevOffset = 0;
+
+ final SlotKind kind = type.getComponentSlotKind();
+
+ writeVnumber(fsIndexes_dos, fsChange.arrayUpdates.size());
+
+ IntListIterator it = fsChange.arrayUpdates.iterator();
+ while (it.hasNext()) {
+ int i = it.next();
+
+ writeVnumber(fsIndexes_dos, i - iPrevOffset);
+ iPrevOffset = i;
- switch (kind) {
- case Slot_HeapRef:
- vPrevModHeapRef = writeIntOrHeapRef(heapRef_i, nextMainHeapIndex, vPrevModHeapRef);
- break;
- case Slot_Int:
- vPrevModInt = writeIntOrHeapRef(int_i, nextMainHeapIndex, vPrevModInt);
- break;
- case Slot_Short:
- vPrevModShort = (short)writeIntOrHeapRef(int_i, nextMainHeapIndex, vPrevModShort);
- break;
- case Slot_LongRef:
- vPrevModLong = writeLongFromHeapIndex(nextMainHeapIndex, vPrevModLong);
- break;
- case Slot_Byte: case Slot_Boolean:
- byte_dos.write(heap[nextMainHeapIndex]);
- break;
- case Slot_Float:
- writeFloat(heap[nextMainHeapIndex]);
- break;
- case Slot_StrRef:
- writeString(stringHeapObj.getStringForCode(heap[nextMainHeapIndex]));
- break;
- case Slot_DoubleRef:
- writeDouble(longHeapObj.getHeapValue(heap[nextMainHeapIndex]));
+ switch(kind) {
+ case Slot_BooleanRef: byte_dos.write(((BooleanArray)fs).get(i) ? 1 : 0); break;
+ case Slot_ByteRef: byte_dos.write(((ByteArray )fs).get(i) ); break;
+ case Slot_ShortRef: {
+ short v = ((ShortArray)fs).get(i);
+ writeDiff(int_i, v, vPrevModShort);
+ vPrevModShort = v;
break;
- default:
- throw new RuntimeException();
}
-
- } // end of looping for all modified slots in this FS
- }
-
- private void writeAuxHeapMods() throws IOException {
- final int auxHeapIndex = heap[iHeap + 2];
- int iPrevOffsetInAuxArray = 0;
-
- final SlotKind kind = typeInfo.getSlotKind(2); // get kind of element
- final boolean isAuxByte = ((kind == Slot_BooleanRef) || (kind == Slot_ByteRef));
- final boolean isAuxShort = (kind == Slot_ShortRef);
- final boolean isAuxLong = ((kind == Slot_LongRef) || (kind == Slot_DoubleRef));
-
- if (!(isAuxByte | isAuxShort | isAuxLong)) {
- throw new RuntimeException(); // never happen
- }
-
- final int[] modXxxHeapAddrs = isAuxByte ? modifiedByteHeapAddrs :
- isAuxShort ? modifiedShortHeapAddrs :
- modifiedLongHeapAddrs;
- final int modXxxHeapAddrsLength = isAuxByte ? modByteHeapAddrsLength :
- isAuxShort ? modShortHeapAddrsLength :
- modLongHeapAddrsLength;
- int imaModXxxRef = isAuxByte ? imaModByteRef :
- isAuxShort ? imaModShortRef :
- imaModLongRef;
-
- final int numberOfModsInAuxHeap = countModifiedSlotsInAuxHeap(modXxxHeapAddrs, imaModXxxRef, modXxxHeapAddrsLength);
- writeVnumber(fsIndexes_dos, numberOfModsInAuxHeap);
-
- for (int i = 0; i < numberOfModsInAuxHeap; i++) {
- final int nextModAuxIndex = modXxxHeapAddrs[imaModXxxRef++];
- final int offsetInAuxArray = nextModAuxIndex - auxHeapIndex;
- writeVnumber(fsIndexes_dos, offsetInAuxArray - iPrevOffsetInAuxArray);
- iPrevOffsetInAuxArray = offsetInAuxArray;
-
- if (isAuxByte) {
- writeUnsignedByte(byte_dos, byteHeapObj.getHeapValue(nextModAuxIndex));
- } else if (isAuxShort) {
- final short v = shortHeapObj.getHeapValue(nextModAuxIndex);
- writeDiff(int_i, v, vPrevModShort);
- vPrevModShort = v;
- } else {
- long v = longHeapObj.getHeapValue(nextModAuxIndex);
- if (kind == Slot_LongRef) {
- writeLong(v, vPrevModLong);
- vPrevModLong = v;
- } else {
- writeDouble(v);
- }
+ case Slot_LongRef: {
+ long v = ((LongArray)fs).get(i);
+ writeLong(v, vPrevModLong);
+ vPrevModLong = v;
+ break;
}
- if (isAuxByte) {
- imaModByteRef++;
- } else if (isAuxShort) {
- imaModShortRef++;
- } else {
- imaModLongRef++;
+ case Slot_DoubleRef: {
+ double v = ((DoubleArray)fs).get(i);
+ writeDouble(CASImpl.double2long(v));
+ break;
}
- }
- }
+ default: Misc.internalError();
+ } // end of switch
+ } // end of iteration over items changed in the array
+ } // end of method
} // end of class definition for SerializeModifiedFSs
+ private int fs2seq(TOP fs) {
+ return (fs == null) ? 0 : fs2seq.get(fs);
+ }
+
+// private TOP seq2fs(int s) {
+// return (s == 0) ? null : seq2fs.get(s);
+// }
+
} // end of class definition for Serializer
/**
@@ -1458,39 +1506,51 @@ public class BinaryCasSerDes4 {
*/
private class Deserializer {
- final private CASImpl cas; // cas being deserialized into
+ final private CASImpl baseCas; // cas being deserialized into
+ final private CASImpl ivCas; // initial view cas - where by default new fs are created
+ final private BinaryCasSerDes bcsd;
+ final private CommonSerDesSequential csds;
final private DataInput deserIn;
final private DataInputStream[] dataInputs = new DataInputStream[NBR_SLOT_KIND_ZIP_STREAMS];
private Inflater[] inflaters = new Inflater[NBR_SLOT_KIND_ZIP_STREAMS];
- private int[] heap; // main heap
+ /** the FS being deserialized */
+ private TOP currentFs;
+
+ /**
+ * the deferrals needed when deserializing a subtype of AnnotationBase before the sofa is known
+ * Also for Sofa creation where some fields are final
+ * */
+ final private List<Runnable> singleFsDefer = new ArrayList<>();
+
+ /** used for deferred creation */
+ private int sofaNum;
+ private String sofaName;
+ private String sofaMimeType;
+ private Sofa sofaRef;
+
+// private int[] heap; // main heap
private int heapStart;
private int heapEnd;
- private IntVector fixupsNeeded; // for deserialization, the "fixups" for relative heap refs needed
- private StringHeap stringHeapObj;
- private LongHeap longHeapObj;
- private ShortHeap shortHeapObj;
- private ByteHeap byteHeapObj;
+ /** the "fixups" for relative heap refs */
+ final private List<Runnable> fixupsNeeded = new ArrayList<>();
- private int stringTableOffset;
+ final private StringHeap stringHeapObj = new StringHeap();
+// private LongHeap longHeapObj;
+// private ShortHeap shortHeapObj;
+// private ByteHeap byteHeapObj;
+//
+// private int stringTableOffset;
- /**
- * Cache sharable common values in aux heaps
- * Values must be in aux heap, but not part of arrays there
- * so that rules out boolean, byte, and shorts
- */
- private int longZeroIndex = -1; // also used for double 0 indix
- private int double1Index = -1;
-
final private boolean isDelta; // if true, a delta is being deserialized
- final private ComprItemRefs fsStartIndexes = (CHANGE_FS_REFS_TO_SEQUENTIAL) ? new ComprItemRefs() : null;
private String[] readCommonString;
- private TypeInfo typeInfo; // type info for the current type being serialized
+// private TypeInfo typeInfo; // type info for the current type being serialized
+
+// private int iPrevHeap; // 0 or heap addr of previous instance of current type
- private int iPrevHeap; // 0 or heap addr of previous instance of current type
private boolean only1CommonString;
// speedups
@@ -1515,6 +1575,45 @@ public class BinaryCasSerDes4 {
final private DataInputStream strSeg_dis;
/**
+ * For differencing when reading.
+ * Also used for arrays to difference the 0th element.
+ *
+ * Can't use real fs for heap refs - may be forward refs not yet fixedup
+ *
+ * Hold prev instance of FS which have FSRef slots
+ *
+ * for each target typecode, only set if the type
+ * - has 1 or more non-array fsref
+ * - is a (subtype of) FSArray
+ * set for both 0 and non-0 values !! Different from form6
+ * first index: key is type code
+ * 2nd index: key is slot-offset number (0-based)
+ *
+ * Also used for array refs, for the 1st entry in the array
+ * - feature slot 0 is used for this when reading (not when writing - could be made more uniform)
+ */
+ final private int[] [] prevFsRefsByType = new int[ts.getTypeArraySize()][];
+ private int[] prevFsRefs;
+
+ /**
+ * Used for differencing, except for HeapRef values which
+ * use above
+ */
+ final private TOP[] prevFsByType = new TOP[ts.getTypeArraySize()];
+ private TOP prevFs;
+ /**
+ * If true, adjust type codes in serialized forms > 37 up by 2
+ */
+ private boolean isBeforeV3;
+
+ /**
+ * convert between FSs and "sequential" numbers
+ * Note: This may be identity map, but may not in the case for V3 where some FSs are GC'd
+ */
+ private final Obj2IntIdentityHashMap<TOP> fs2seq = new Obj2IntIdentityHashMap<TOP>(TOP.class, TOP.singleton);
+ private final Int2ObjHashMap<TOP> seq2fs = new Int2ObjHashMap<>(TOP.class);
+
+ /**
* Called after header was read and determined that
* this was a compressed binary
* @param cas CAS
@@ -1522,16 +1621,13 @@ public class BinaryCasSerDes4 {
* @throws IOException passthru
*/
Deserializer(CASImpl cas, DataInput deserIn, boolean isDelta) throws IOException {
- this.cas = cas;
+ this.baseCas = cas.getBaseCAS();
+ this.ivCas = baseCas.getInitialView();
+ this.bcsd = cas.getBinaryCasSerDes();
+ this.csds = new CommonSerDesSequential(this.baseCas);
this.deserIn = deserIn;
this.isDelta = isDelta;
- stringHeapObj = cas.getStringHeap();
- longHeapObj = cas.getLongHeap();
- shortHeapObj = cas.getShortHeap();
- byteHeapObj = cas.getByteHeap();
-
-// deserIn.readInt(); // reserved to record additional version info // already read before calling
final int nbrEntries = deserIn.readInt(); // number of compressed streams
IntVector idxAndLen = new IntVector(nbrEntries * 3);
@@ -1565,15 +1661,19 @@ public class BinaryCasSerDes4 {
control_dis = dataInputs[control_i];
strSeg_dis = dataInputs[strSeg_i];
-
}
- private void deserialize() throws IOException {
+ private void deserialize(int version1) throws IOException {
+ isBeforeV3 = (version1 & 0xff00) == 0;
+
+ fs2seq.clear();
+ seq2fs.clear();
/************************************************
* Setup all the input streams with inflaters
************************************************/
// long startTime1 = System.currentTimeMillis();
+ stringHeapObj.reset();
/************************************************
* Read in the common string(s)
@@ -1587,65 +1687,135 @@ public class BinaryCasSerDes4 {
/***************************
* Prepare to walk main heap
***************************/
+ csds.setup();
+ int seq = 1;
+ for (TOP fs : csds.sortedFSs) {
+ fs2seq.put(fs, seq);
+ seq2fs.put(seq++, fs);
+ }
+
int deltaHeapSize = readVnumber(control_dis);
- final Heap heapObj = cas.getHeap();
- heapStart = isDelta ? heapObj.getNextId() : 0;
- stringTableOffset = isDelta ? (stringHeapObj.getSize() - 1) : 0;
+ heapStart = isDelta ? csds.heapEnd : 0;
+
+// stringTableOffset = isDelta ? (stringHeapObj.getSize() - 1) : 0;
- if (isDelta) {
- heapObj.grow(deltaHeapSize);
- } else {
- heapObj.reinitSizeOnly(deltaHeapSize);
- }
+// if (isDelta) {
+// heapObj.grow(deltaHeapSize);
+// } else {
+// heapObj.reinitSizeOnly(deltaHeapSize);
+// }
heapEnd = heapStart + deltaHeapSize;
- heap = heapObj.heap;
+// heap = heapObj.heap;
- resetIprevious();
+ for (int[] ia : prevFsRefsByType) {
+ if (ia != null) Arrays.fill(ia, 0);
+ }
if (heapStart == 0) {
heapStart = 1; // slot 0 not serialized, it's null / 0
}
- if (CHANGE_FS_REFS_TO_SEQUENTIAL && (heapStart > 1)) {
- initFsStartIndexes(fsStartIndexes, heap, 1, heapStart, null);
- }
- fixupsNeeded = new IntVector(Math.max(16, heap.length / 10));
-
- /***************************
- * walk main heap
- ***************************/
+
+
+// if (CHANGE_FS_REFS_TO_SEQUENTIAL && (heapStart > 1)) {
+// initFsStartIndexes(fsStartIndexes, heap, 1, heapStart, null);
+// }
+// fixupsNeeded = new IntVector(Math.max(16, heap.length / 10));
- for (int iHeap = heapStart; iHeap < heapEnd; iHeap += incrToNextFs(heap, iHeap, typeInfo)) {
- if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
- fsStartIndexes.addItemAddr(iHeap);
- }
- int tCode = heap[iHeap] = readVnumber(typeCode_dis); // get type code
- typeInfo = getTypeInfo(tCode);
- iPrevHeap = typeInfo.iPrevHeap;
-
- if (typeInfo.isHeapStoredArray) {
- readHeapStoredArray(iHeap);
- } else if (typeInfo.isArray) {
- readNonHeapStoredArray(iHeap);
+ /*******************************
+ * walk main heap - deserialize
+ *******************************/
+ TypeImpl type;
+ int arraySize = 0;
+ Arrays.fill(prevFsByType, null);
+
+ for (int iHeap = heapStart; iHeap < heapEnd; iHeap += type.getFsSpaceReq(arraySize)) {
+
+ final int typeCode = readVnumber(typeCode_dis);
+ final int adjTypeCode = typeCode + ((isBeforeV3 && typeCode >= TypeSystemImpl.javaObjectTypeCode) ? 2 : 0);
+ type = ts.getTypeForCode(adjTypeCode);
+
+ prevFs = prevFsByType[adjTypeCode]; // could be null;
+ prevFsRefs = getPrevFsRef(type); // null or int[], only for things having fsrefs (array or not)
+
+ if (type.isArray()) {
+ currentFs = readArray(iHeap, type);
+ arraySize = ((CommonArray)currentFs).size();
} else {
- for (int i = 1; i < typeInfo.slotKinds.length + 1; i++) {
- readByKind(iHeap, i);
+ if (!ts.annotBaseType.subsumes(type) && // defer subtypes of AnnotationBase
+ !(ts.sofaType == type)) { // defer sofa types
+ currentFs = ivCas.createFS(type);
+
+ } else {
+ currentFs = null;
+ singleFsDefer.clear();
+ sofaRef = null;
+ sofaNum = -1;
+ sofaName = null;
+ sofaMimeType = null;
+ }
+ for (FeatureImpl feat : type.getFeatureImpls()) {
+ readByKind(feat, type);
+ }
+// for (int i = 1; i < typeInfo.slotKinds.length + 1; i++) {
+// readByKind(iHeap, i);
+// }
+ }
+
+ if (currentFs == null) {
+
+ /**
+ * Create single deferred FS
+ * Either: Sofa (has final fields) or
+ * Subtype of AnnotationBase - needs to be in the right view
+ *
+ * For the latter, handle document annotation specially
+ */
+
+ if (ts.sofaType == type) {
+ currentFs = baseCas.createSofa(sofaNum, sofaName, sofaMimeType);
+ } else {
+ CASImpl view = (CASImpl) baseCas.getView(sofaRef);
+ if (type.getCode() == TypeSystemImpl.docTypeCode) {
+ currentFs = view.getDocumentAnnotation(); // creates the document annotation if it doesn't exist
+ // we could remove this from the indexes until deserialization is over, but then, other calls to getDocumentAnnotation
+ // would end up creating additional instances
+ } else {
+ currentFs = view.createFS(type);
+ }
+ }
+ if (type.getCode() == TypeSystemImpl.docTypeCode) {
+ boolean wasRemoved = baseCas.checkForInvalidFeatureSetting(currentFs, baseCas.getAddbackSingle());
+ for (Runnable r : singleFsDefer) {
+ r.run();
+ }
+ baseCas.maybeAddbackSingle(wasRemoved, currentFs);
+ } else {
+ for (Runnable r : singleFsDefer) {
+ r.run();
+ }
}
}
- typeInfo.iPrevHeap = iHeap; // make this one the "prev" one for subsequent testing
+ assert(currentFs != null);
+// System.out.format("Adding %,d to csds%n", iHeap);
+// if (isDelta) {
+// System.out.format("debug adding iHeap: %,d afterAdd: %,d%n", iHeap, iHeap + nextHeapAddrAfterMark);
+// }
+ csds.addFS(currentFs, iHeap);
+ int s2 = 1 + fs2seq.size();
+ fs2seq.put(currentFs, s2); // 1 origin to match v2
+ seq2fs.put(s2, currentFs);
+
+ prevFsByType[adjTypeCode] = currentFs;
}
- if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
- fsStartIndexes.finishSetup();
- final int end = fixupsNeeded.size();
- for (int i = 0; i < end; i++) {
- final int heapAddrToFix = fixupsNeeded.get(i);
- heap[heapAddrToFix] = fsStartIndexes.getItemAddr(heap[heapAddrToFix]);
- }
+ for (Runnable r : fixupsNeeded) {
+ r.run();
}
+
readIndexedFeatureStructures();
@@ -1657,123 +1827,202 @@ public class BinaryCasSerDes4 {
// System.out.format("Deserialize took %,d ms%n", System.currentTimeMillis() - startTime1);
}
- private void readNonHeapStoredArray(int iHeap) throws IOException {
- final int length = readArrayLength(iHeap);
+ private TOP readArray(int iHeap, TypeImpl type) throws IOException {
+ final int length = readArrayLength();
+ TOP fs = ivCas.createArray(type, length);
if (length == 0) {
- return;
+ return fs;
}
- SlotKind refKind = typeInfo.getSlotKind(2);
+
+ SlotKind refKind = type.getComponentSlotKind();
switch (refKind) {
- case Slot_BooleanRef: case Slot_ByteRef:
- heap[iHeap + 2] = readIntoByteArray(length);
- break;
- case Slot_ShortRef:
- heap[iHeap + 2] = readIntoShortArray(length);
- break;
- case Slot_LongRef: case Slot_DoubleRef:
- heap[iHeap + 2] = readIntoLongArray(refKind, length);
- break;
- default:
- throw new RuntimeException();
+
+ case Slot_BooleanRef: {
+ boolean [] ba = ((BooleanArray)fs)._getTheArray();
+ for (int i = 0; i < length; i++) {
+ ba[i] = byte_dis.readByte() == 1;
+ }
+ break;
}
- }
-
- private int readArrayLength(int iHeap) throws IOException {
- return heap[iHeap + 1] = readVnumber(arrayLength_dis);
- }
+
+ case Slot_ByteRef:
+ readIntoByteArray(((ByteArray)fs)._getTheArray());
+ break;
- private void readHeapStoredArray(int iHeap) throws IOException {
- final int length = readArrayLength(iHeap);
- // output values
- // special case 0 and 1st value
- if (length == 0) {
- return;
- }
- SlotKind arrayElementKind = typeInfo.slotKinds[1];
- final int endi = iHeap + length + 2;
- switch (arrayElementKind) {
- case Slot_HeapRef: case Slot_Int: case Slot_Short:
- {
- int prev = (iPrevHeap == 0) ? 0 :
- (heap[iPrevHeap + 1] == 0) ? 0 :
- heap[iPrevHeap + 2];
- for (int i = iHeap + 2; i < endi; i++) {
- final int v = heap[i] = readDiff(arrayElementKind, prev);
- prev = v;
- if (arrayElementKind == Slot_HeapRef) {
- fixupsNeeded.add(i);
- }
+ case Slot_ShortRef:
+ readIntoShortArray(((ShortArray)fs)._getTheArray());
+ break;
+
+ case Slot_Int: {
+ final int[] ia = ((IntegerArray)fs)._getTheArray();
+ int prev = getPrevIntValue(refKind, null);
+ for (int i = 0; i < length; i++) {
+ int v = readDiff(Slot_Int, prev);
+ prev = v;
+ if (i == 0) {
+ savePrevHeapRef(type.getCode(), 1, 0, v);
}
+ ia[i] = v;
}
break;
- case Slot_Float:
- for (int i = iHeap + 2; i < endi; i++) {
- heap[i] = readFloat();
- }
+ }
+
+ case Slot_LongRef:
+ readIntoLongArray(((LongArray)fs)._getTheArray());
break;
- case Slot_StrRef:
- for (int i = iHeap + 2; i < endi; i++) {
- heap[i] = readString();
+
+ case Slot_Float: {
+ final float[] fa = ((FloatArray)fs)._getTheArray();
+ for (int i = 0; i < length; i++) {
+ final int floatRef = readFloat();
+ fa[i] = Float.intBitsToFloat(floatRef);
}
break;
+ }
+
+ case Slot_DoubleRef:
+ readIntoDoubleArray(((DoubleArray)fs)._getTheArray()); ;
+ break;
[... 1675 lines stripped ...]