You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by sc...@apache.org on 2016/02/17 19:43:42 UTC

svn commit: r1730902 [2/3] - in /uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl: BinaryCasSerDes4.java BinaryCasSerDes6.java

Modified: uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java?rev=1730902&r1=1730901&r2=1730902&view=diff
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java (original)
+++ uima/uimaj/branches/experiment-v3-jcas/uimaj-core/src/main/java/org/apache/uima/cas/impl/BinaryCasSerDes4.java Wed Feb 17 18:43:42 2016
@@ -19,34 +19,7 @@
 
 package org.apache.uima.cas.impl;
 
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.NBR_SLOT_KIND_ZIP_STREAMS;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_ArrayLength;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Boolean;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_BooleanRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Byte;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_ByteRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Control;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_DoubleRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Double_Exponent;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Double_Mantissa_Sign;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Float;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Float_Exponent;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Float_Mantissa_Sign;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_FsIndexes;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_HeapRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Int;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_LongRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Long_High;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Long_Low;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_MainHeap;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_Short;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_ShortRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrChars;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrLength;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrOffset;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrRef;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_StrSeg;
-import static org.apache.uima.cas.impl.BinaryCasSerDes4.SlotKind.Slot_TypeCode;
+import static org.apache.uima.cas.impl.SlotKinds.SlotKind.Slot_Int;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
@@ -63,10 +36,10 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
+import java.util.BitSet;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
+import java.util.function.Consumer;
 import java.util.zip.Deflater;
 import java.util.zip.DeflaterOutputStream;
 import java.util.zip.Inflater;
@@ -74,11 +47,29 @@ import java.util.zip.InflaterInputStream
 
 import org.apache.uima.cas.AbstractCas;
 import org.apache.uima.cas.CASRuntimeException;
-import org.apache.uima.cas.Feature;
 import org.apache.uima.cas.Marker;
+import org.apache.uima.cas.impl.CASImpl.FsChange;
 import org.apache.uima.cas.impl.FSsTobeAddedback.FSsTobeAddedbackSingle;
+import org.apache.uima.cas.impl.SlotKinds.SlotKind;
+import org.apache.uima.internal.util.Int2ObjHashMap;
+import org.apache.uima.internal.util.IntListIterator;
 import org.apache.uima.internal.util.IntVector;
+import org.apache.uima.internal.util.Obj2IntIdentityHashMap;
 import org.apache.uima.jcas.JCas;
+import org.apache.uima.jcas.cas.BooleanArray;
+import org.apache.uima.jcas.cas.ByteArray;
+import org.apache.uima.jcas.cas.CommonArray;
+import org.apache.uima.jcas.cas.DoubleArray;
+import org.apache.uima.jcas.cas.FSArray;
+import org.apache.uima.jcas.cas.FloatArray;
+import org.apache.uima.jcas.cas.IntegerArray;
+import org.apache.uima.jcas.cas.LongArray;
+import org.apache.uima.jcas.cas.ShortArray;
+import org.apache.uima.jcas.cas.Sofa;
+import org.apache.uima.jcas.cas.StringArray;
+import org.apache.uima.jcas.cas.TOP;
+import org.apache.uima.resource.ResourceInitializationException;
+import org.apache.uima.util.Misc;
 import org.apache.uima.util.impl.DataIO;
 import org.apache.uima.util.impl.OptimizeStrings;
 import org.apache.uima.util.impl.SerializationMeasures;
@@ -126,9 +117,19 @@ import org.apache.uima.util.impl.Seriali
  *   Read all bytes, 
  *   create separate ByteArrayInputStreams for each segment, sharing byte bfr
  *   create appropriate unzip data input streams for these
+ *   
+ * Properties of Form 4:
+ *   1) Indexes ** not used ** for determining what gets serialized, instead
+ *      the "heap" is walked and all FSs found there are serialized
+ *      
+ *   2) The number used for references to FSs is a sequentially incrementing one, starting at 1
+ *   
+ *   
  */
-public class BinaryCasSerDes4 {
-  
+public class BinaryCasSerDes4 implements SlotKindsConstants {
+  //  0 - original, 1 = fixes from V3 in v2, 2 = v3 --------------------------v
+  private static final boolean TRACE_DOUBLE = false;
+  private static final boolean TRACE_INT = false;
   public static final int TYPECODE_COMPR = 8;
   public static final boolean CHANGE_FS_REFS_TO_SEQUENTIAL = true;
   // may add more later - to specify differing trade-offs between speed and compression
@@ -138,7 +139,6 @@ public class BinaryCasSerDes4 {
   public static final boolean IGNORED = true;
   public static final boolean IN_MAIN_HEAP = true;
   
-  private static final long DBL_1 = Double.doubleToLongBits(1D);
 
   /**
    * The kinds of slots that can exist 
@@ -203,101 +203,109 @@ public class BinaryCasSerDes4 {
     }
   }
   
-  /**
-   * Define all the slot kinds.
-   */
-  public enum SlotKind {
-    Slot_ArrayLength(! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
-    Slot_HeapRef(    IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_Int(        IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_Byte(       ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
-    Slot_Short(      IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_TypeCode(   ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
-
-    Slot_StrOffset(  ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
-    Slot_StrLength(  ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
-    Slot_Long_High(    IS_DIFF_ENCODE,           IGNORED, 0, !IN_MAIN_HEAP),
-    Slot_Long_Low (    IS_DIFF_ENCODE,           IGNORED, 0, !IN_MAIN_HEAP),
-
-    // the next are not actual slot kinds, but instead
-    // are codes used to control encoding of Floats and Doubles.
-    Slot_Float_Mantissa_Sign( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
-    // exponent is 8 bits, and shifted in the expectation
-    // that many values may be between 1 and 0 (e.g., normalized values)
-    //   -- so sign moving is needed
-    Slot_Float_Exponent(      ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
-    
-    Slot_Double_Mantissa_Sign(! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
-    Slot_Double_Exponent(     ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
-    Slot_FsIndexes(             IS_DIFF_ENCODE,         IGNORED, 4, !IN_MAIN_HEAP),
-    
-    Slot_StrChars(            IGNORED,          IGNORED, 2, !IN_MAIN_HEAP),
-    
-    Slot_Control(             IGNORED,          IGNORED, 0, !IN_MAIN_HEAP),
-    Slot_StrSeg(              ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 0, ! IN_MAIN_HEAP),
-    
-    // the next slots are not serialized
-    Slot_StrRef(     IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_BooleanRef( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
-    Slot_ByteRef(    IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_ShortRef(   IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_LongRef(    IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_DoubleRef(  IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
-    Slot_Float(      ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
-    Slot_Boolean(    ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
-    // next used to capture original heap size
-    Slot_MainHeap(   IGNORED,          IGNORED,           4, !IN_MAIN_HEAP),
-
-    ;
-
-    public final int i;
-    public final boolean isDiffEncode;
-    public final boolean canBeNegative;
-    public final boolean inMainHeap;
-    public final int elementSize;
-    
-    public static final int NBR_SLOT_KIND_ZIP_STREAMS;
-    static {NBR_SLOT_KIND_ZIP_STREAMS = Slot_StrRef.i;}
-    
-    SlotKind(boolean isDiffEncode, 
-             boolean canBeNegative, 
-             int elementSize,
-             boolean inMainHeap) {
-      this.i = this.ordinal();
-      this.isDiffEncode = isDiffEncode;
-      this.canBeNegative = isDiffEncode ? true : canBeNegative;
-      this.elementSize = elementSize; 
-      this.inMainHeap = inMainHeap;
-    }
-  }
+//  /**
+//   * Define all the slot kinds.
+//   */
+//  public enum SlotKind {
+//    Slot_ArrayLength(! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+//    Slot_HeapRef(    IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_Int(        IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_Byte(       ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+//    Slot_Short(      IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_TypeCode(   ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+//
+//    Slot_StrOffset(  ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
+//    Slot_StrLength(  ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, !IN_MAIN_HEAP),
+//    Slot_Long_High(    IS_DIFF_ENCODE,           IGNORED, 0, !IN_MAIN_HEAP),
+//    Slot_Long_Low (    IS_DIFF_ENCODE,           IGNORED, 0, !IN_MAIN_HEAP),
+//
+//    // the next are not actual slot kinds, but instead
+//    // are codes used to control encoding of Floats and Doubles.
+//    Slot_Float_Mantissa_Sign( ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+//    // exponent is 8 bits, and shifted in the expectation
+//    // that many values may be between 1 and 0 (e.g., normalized values)
+//    //   -- so sign moving is needed
+//    Slot_Float_Exponent(      ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+//    
+//    Slot_Double_Mantissa_Sign(! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+//    Slot_Double_Exponent(     ! IS_DIFF_ENCODE, CAN_BE_NEGATIVE, 0, !IN_MAIN_HEAP),
+//    Slot_FsIndexes(             IS_DIFF_ENCODE,         IGNORED, 4, !IN_MAIN_HEAP),
+//    
+//    Slot_StrChars(            IGNORED,          IGNORED, 2, !IN_MAIN_HEAP),
+//    
+//    Slot_Control(             IGNORED,          IGNORED, 0, !IN_MAIN_HEAP),
+//    Slot_StrSeg(              ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 0, ! IN_MAIN_HEAP),
+//    
+//    // the next slots are not serialized
+//    Slot_StrRef(     IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_BooleanRef( ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+//    Slot_ByteRef(    IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_ShortRef(   IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_LongRef(    IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_DoubleRef(  IS_DIFF_ENCODE,             IGNORED, 4, IN_MAIN_HEAP),
+//    Slot_Float(      ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+//    Slot_Boolean(    ! IS_DIFF_ENCODE, ! CAN_BE_NEGATIVE, 4, IN_MAIN_HEAP),
+//    // next used to capture original heap size
+//    Slot_MainHeap(   IGNORED,          IGNORED,           4, !IN_MAIN_HEAP),
+//
+//  //TODO fix this 
+//    Slot_JavaObjectRef(  IGNORED,      CAN_BE_NEGATIVE,   4, IN_MAIN_HEAP);
+//    ;
+//
+//    public final int i;
+//    public final boolean isDiffEncode;
+//    public final boolean canBeNegative;
+//    public final boolean inMainHeap;
+//    public final int elementSize;
+//    
+//    public static final int NBR_SLOT_KIND_ZIP_STREAMS;
+//    static {NBR_SLOT_KIND_ZIP_STREAMS = Slot_StrRef.i;}
+//    
+//    SlotKind(boolean isDiffEncode, 
+//             boolean canBeNegative, 
+//             int elementSize,
+//             boolean inMainHeap) {
+//      this.i = this.ordinal();
+//      this.isDiffEncode = isDiffEncode;
+//      this.canBeNegative = isDiffEncode ? true : canBeNegative;
+//      this.elementSize = elementSize; 
+//      this.inMainHeap = inMainHeap;
+//    }
+//  }
   
   /**
    * Things set up for one instance of this class, and
    * reuse-able
    */
-  final private TypeInfo [] typeInfoArray;  // lazy initialization of elements
-  final private TypeSystemImpl ts;  // for debugging
+  final private TypeSystemImpl ts;  
   final private boolean doMeasurements;
   
+  final TypeImpl fsArrayType;
+
+  
+  /** 
+   * Things shared between serialization and deserialization
+   */
+    
   // speedups
-  final private static int arrayLength_i = Slot_ArrayLength.i;
-  final private static int heapRef_i = Slot_HeapRef.i;
-  final private static int int_i = Slot_Int.i;
-  final private static int byte_i = Slot_Byte.ordinal();
-  final private static int short_i = Slot_Short.i;
-  final private static int typeCode_i = Slot_TypeCode.i;
-  final private static int strOffset_i = Slot_StrOffset.i;
-  final private static int strLength_i = Slot_StrLength.i;
-  final private static int long_High_i = Slot_Long_High.i;
-  final private static int long_Low_i = Slot_Long_Low.i;
-  final private static int float_Mantissa_Sign_i = Slot_Float_Mantissa_Sign.i;
-  final private static int float_Exponent_i = Slot_Float_Exponent.i;
-  final private static int double_Mantissa_Sign_i = Slot_Double_Mantissa_Sign.i;
-  final private static int double_Exponent_i = Slot_Double_Exponent.i;
-  final private static int fsIndexes_i = Slot_FsIndexes.i;
-  final private static int strChars_i = Slot_StrChars.i;
-  final private static int control_i = Slot_Control.i;
-  final private static int strSeg_i = Slot_StrSeg.i;
+//  final private static int arrayLength_i = Slot_ArrayLength.i;
+//  final private static int heapRef_i = Slot_HeapRef.i;
+//  final private static int int_i = Slot_Int.i;
+//  final private static int byte_i = Slot_Byte.ordinal();
+//  final private static int short_i = Slot_Short.i;
+//  final private static int typeCode_i = Slot_TypeCode.i;
+//  final private static int strOffset_i = Slot_StrOffset.i;
+//  final private static int strLength_i = Slot_StrLength.i;
+//  final private static int long_High_i = Slot_Long_High.i;
+//  final private static int long_Low_i = Slot_Long_Low.i;
+//  final private static int float_Mantissa_Sign_i = Slot_Float_Mantissa_Sign.i;
+//  final private static int float_Exponent_i = Slot_Float_Exponent.i;
+//  final private static int double_Mantissa_Sign_i = Slot_Double_Mantissa_Sign.i;
+//  final private static int double_Exponent_i = Slot_Double_Exponent.i;
+//  final private static int fsIndexes_i = Slot_FsIndexes.i;
+//  final private static int strChars_i = Slot_StrChars.i;
+//  final private static int control_i = Slot_Control.i;
+//  final private static int strSeg_i = Slot_StrSeg.i;
   
   /**
    * 
@@ -307,9 +315,8 @@ public class BinaryCasSerDes4 {
   public BinaryCasSerDes4(TypeSystemImpl ts, boolean doMeasurements) {
     this.ts = ts;
     this.doMeasurements = doMeasurements;
-
-    typeInfoArray = new TypeInfo[(ts.getTypeArraySize())];
-  }
+    this.fsArrayType = ts.fsArrayType;
+   }
 
   /**
    * 
@@ -326,8 +333,7 @@ public class BinaryCasSerDes4 {
     SerializationMeasures sm = (doMeasurements) ? new SerializationMeasures() : null;
     CASImpl casImpl = (CASImpl) ((cas instanceof JCas) ? ((JCas)cas).getCas(): cas);
     if (null != trackingMark && !trackingMark.isValid() ) {
-      throw new CASRuntimeException(
-                CASRuntimeException.INVALID_MARKER, new String[] { "Invalid Marker." });
+      throw new CASRuntimeException(CASRuntimeException.INVALID_MARKER, "Invalid Marker.");
     }
     
     Serializer serializer = new Serializer(
@@ -351,15 +357,10 @@ public class BinaryCasSerDes4 {
     return serialize(cas, out, null);
   }
 
-  public void deserialize(CASImpl cas, InputStream deserIn, boolean isDelta) throws IOException {
-    DataInput in;
-    if (deserIn instanceof DataInputStream) {
-      in = (DataInputStream)deserIn;
-    } else {
-      in = new DataInputStream(deserIn);
-    }
+  public void deserialize(CASImpl cas, InputStream deserIn, boolean isDelta, int version1) throws IOException {
+    DataInput in = CommonSerDes.maybeWrapToDataInputStream(deserIn);
     Deserializer deserializer = new Deserializer(cas, in, isDelta);    
-    deserializer.deserialize();
+    deserializer.deserialize(version1);
   }
 
   /**
@@ -370,65 +371,94 @@ public class BinaryCasSerDes4 {
   
   private class Serializer {
     final private DataOutputStream serializedOut;  // where to write out the serialized result
-    final private CASImpl cas;  // cas being serialized
+    final private CASImpl baseCas;  // cas being serialized
+    final private BinaryCasSerDes bcsd;  
     final private MarkerImpl mark;  // the mark to serialize from
     
     final private SerializationMeasures sm;  // null or serialization measurements
     final private ByteArrayOutputStream[] baosZipSources = new ByteArrayOutputStream[NBR_SLOT_KIND_ZIP_STREAMS];  // lazily created, indexed by SlotKind.i
     final private DataOutputStream[] dosZipSources = new DataOutputStream[NBR_SLOT_KIND_ZIP_STREAMS];      // lazily created, indexed by SlotKind.i
 
-    final private int[] heap;           // main heap
+//    final private int[] heap;           // main heap
     private int heapStart;
-    final private int heapEnd;
-    final private StringHeap stringHeapObj;
-    final private LongHeap longHeapObj;
-    final private ShortHeap shortHeapObj;
-    final private ByteHeap byteHeapObj;
+    private int heapEnd;
+//    final private LongHeap longHeapObj;
+//    final private ShortHeap shortHeapObj;
+//    final private ByteHeap byteHeapObj;
 
     final private boolean isDelta;        // if true, there is a marker indicating the start spot(s)
     final private boolean doMeasurement;  // if true, doing measurements
-    final private ComprItemRefs fsStartIndexes = (CHANGE_FS_REFS_TO_SEQUENTIAL) ? new ComprItemRefs() : null;
-    final private int[] typeCodeHisto = new int[ts.getTypeArraySize()]; 
-    final private Integer[] serializedTypeCode2Code = new Integer[ts.getTypeArraySize()]; // needs to be Integer to get comparator choice
-    final private int[] estimatedZipSize = new int[NBR_SLOT_KIND_ZIP_STREAMS]; // one entry for each output stream kind
+//    final private ComprItemRefs fsStartIndexes = (CHANGE_FS_REFS_TO_SEQUENTIAL) ? new ComprItemRefs() : null;
+//    final private int[] typeCodeHisto = new int[ts.getTypeArraySize()]; 
+//    final private Integer[] serializedTypeCode2Code = new Integer[ts.getTypeArraySize()]; // needs to be Integer to get comparator choice
+//    final private int[] estimatedZipSize = new int[NBR_SLOT_KIND_ZIP_STREAMS]; // one entry for each output stream kind
     final private OptimizeStrings os;
     final private CompressLevel compressLevel;
     final private CompressStrat compressStrategy;
     
-    // typeInfo is local to this serialization instance to permit multiple threads
-    private TypeInfo typeInfo; // type info for the current type being serialized
-    private int iPrevHeap;        // 0 or heap addr of previous instance of current type
+ //    private int iPrevHeap;        // 0 or heap addr of previous instance of current type
+    /**
+     * For differencing when reading and writing.
+     * Also used for arrays to difference the 0th element.
+     */
+    final private TOP prevFsByType[];
+    private TOP prevFs;
+
     private boolean only1CommonString;  // true if only one common string
     
+//    final private CommonCompressedSerialization ccs; 
+    final private CommonSerDesSequential csds;
+    
     // speedups
     
     // any use of these means caller handles measurement
     // some of these are never used, because the current impl
     //   is using the _i form to get measurements done
-    final private DataOutputStream arrayLength_dos;
-    final private DataOutputStream heapRef_dos;
-    final private DataOutputStream int_dos;
+//    final private DataOutputStream arrayLength_dos;
+//    final private DataOutputStream heapRef_dos;
+//    final private DataOutputStream int_dos;
     final private DataOutputStream byte_dos;
-    final private DataOutputStream short_dos;
+//    final private DataOutputStream short_dos;
     final private DataOutputStream typeCode_dos;
     final private DataOutputStream strOffset_dos;
     final private DataOutputStream strLength_dos;
-    final private DataOutputStream long_High_dos;
-    final private DataOutputStream long_Low_dos;
+//    final private DataOutputStream long_High_dos;
+//    final private DataOutputStream long_Low_dos;
     final private DataOutputStream float_Mantissa_Sign_dos;
     final private DataOutputStream float_Exponent_dos;
     final private DataOutputStream double_Mantissa_Sign_dos;
     final private DataOutputStream double_Exponent_dos;
     final private DataOutputStream fsIndexes_dos;
-    final private DataOutputStream strChars_dos;
+//    final private DataOutputStream strChars_dos;
     final private DataOutputStream control_dos;
     final private DataOutputStream strSeg_dos;
+    
+    /**
+     * convert between FSs and "sequential" numbers
+     * Note: This may be identity map, but may not in the case for V3 where some FSs are GC'd
+     */    
+    private final Obj2IntIdentityHashMap<TOP> fs2seq = new Obj2IntIdentityHashMap<TOP>(TOP.class, TOP.singleton);
+    private final Int2ObjHashMap<TOP> seq2fs = new Int2ObjHashMap<>(TOP.class);
+    /**
+     * 
+     * @param cas -
+     * @param serializedOut -
+     * @param mark -
+     * @param sm -
+     * @param compressLevel -
+     * @param compressStrategy -
+     */
 
     private Serializer(CASImpl cas, DataOutputStream serializedOut, MarkerImpl mark,
                        SerializationMeasures sm,
                        CompressLevel compressLevel,
                        CompressStrat compressStrategy) {
-      this.cas = cas;
+      this.baseCas = cas.getBaseCAS();
+      this.bcsd = cas.getBinaryCasSerDes();
+      this.csds = new CommonSerDesSequential(this.baseCas);
+//      this.ccs = new CommonCompressedSerialization(
+//          new CommonSerDesTypeMap(cas.getTypeSystemImpl(),  cas.getTypeSystemImpl()), // no type mapping
+//          mark);
       this.serializedOut = serializedOut;
       this.mark = mark;
       this.sm = sm;
@@ -437,253 +467,173 @@ public class BinaryCasSerDes4 {
       isDelta = (mark != null);
       doMeasurement = (sm != null);
       
-      heap = cas.getHeap().heap;
-      heapEnd = cas.getHeap().getCellsUsed();
-      heapStart = isDelta ? mark.getNextFSId() : 0;
-      
-      stringHeapObj = cas.getStringHeap();
-      longHeapObj   = cas.getLongHeap();
-      shortHeapObj  = cas.getShortHeap();
-      byteHeapObj   = cas.getByteHeap();
+//      heap = cas.getHeap().heap;
+//      heapEnd = cas.getHeap().getCellsUsed();
+     
+//      
+//      stringHeapObj = cas.getStringHeap();
+//      longHeapObj   = cas.getLongHeap();
+//      shortHeapObj  = cas.getShortHeap();
+//      byteHeapObj   = cas.getByteHeap();
      
       os = new OptimizeStrings(doMeasurement);
       
-      setupOutputStreams();   
-      arrayLength_dos = dosZipSources[arrayLength_i];
-      heapRef_dos = dosZipSources[heapRef_i];
-      int_dos = dosZipSources[int_i];
+      BinaryCasSerDes6.setupOutputStreams(baseCas, baosZipSources, dosZipSources);  
+      
+//      arrayLength_dos = dosZipSources[arrayLength_i];
+//      heapRef_dos = dosZipSources[heapRef_i];
+//      int_dos = dosZipSources[int_i];
       byte_dos = dosZipSources[byte_i];
-      short_dos = dosZipSources[short_i];
+//      short_dos = dosZipSources[short_i];
       typeCode_dos = dosZipSources[typeCode_i];
       strOffset_dos = dosZipSources[strOffset_i];
       strLength_dos = dosZipSources[strLength_i];
-      long_High_dos = dosZipSources[long_High_i];
-      long_Low_dos = dosZipSources[long_Low_i];
+//      long_High_dos = dosZipSources[long_High_i];
+//      long_Low_dos = dosZipSources[long_Low_i];
       float_Mantissa_Sign_dos = dosZipSources[float_Mantissa_Sign_i];
       float_Exponent_dos = dosZipSources[float_Exponent_i];
       double_Mantissa_Sign_dos = dosZipSources[double_Mantissa_Sign_i];
       double_Exponent_dos = dosZipSources[double_Exponent_i];
       fsIndexes_dos = dosZipSources[fsIndexes_i];
-      strChars_dos = dosZipSources[strChars_i];
+//      strChars_dos = dosZipSources[strChars_i];
       control_dos = dosZipSources[control_i];
       strSeg_dos = dosZipSources[strSeg_i];
-    }
-    
-    private void setupOutputStreams() {
-      // estimate model:
-      //   33% of space in strings, 33% in heap, 33% other
-      //   compr ratio for heap is 98%
-      int compr = (heapEnd - heapStart) * 8 / 3 / 50;
-      int compr1000 = Math.max(512, compr/1000);
-      // 2nd arg is the number of bytes in the byte output stream, initially
-      estimatedZipSize[typeCode_i] = Math.max(512, compr/4);  // /4 for ~4 slots per fs 
-//      estimatedZipSize[boolean_i] =compr1000;
-      estimatedZipSize[byte_i] = compr1000;
-      estimatedZipSize[short_i] = compr1000;
-      estimatedZipSize[int_i] = Math.max(1024, compr1000);
-      estimatedZipSize[arrayLength_i] = compr1000;
-      estimatedZipSize[float_Mantissa_Sign_i] = compr1000;
-      estimatedZipSize[float_Exponent_i] = compr1000;
-      estimatedZipSize[double_Mantissa_Sign_i] = compr1000;
-      estimatedZipSize[double_Exponent_i] = compr1000;
-      estimatedZipSize[long_High_i] = compr1000;
-      estimatedZipSize[long_Low_i] = compr1000;
-      estimatedZipSize[heapRef_i] = Math.max(1024, compr1000);
-      estimatedZipSize[strOffset_i] = Math.max(512, compr/4);
-      estimatedZipSize[strLength_i] = Math.max(512, compr/4);
-      estimatedZipSize[fsIndexes_i] = Math.max(512, compr/8);  // /4 for ~4 slots/fs, / 2 for # indexed
-      estimatedZipSize[strChars_i] = Math.max(512, compr/4); // strings compress better
-      estimatedZipSize[control_i] = 128;
       
-      for (int i = 0; i < baosZipSources.length; i++) {
-        setupOutputStream(i);
-      }
-
+      this.prevFsByType = new TOP[ts.getTypeArraySize()];
     }
     
+    /**
+     * Form 4 serialization is tied to the layout of V2 Feature Structures in heaps.
+     * It does not walk the indexes to serialize just those FSs that are reachable.
+     * 
+     * For V3, it scans the CASImpl.id2fs information and serializes those (except those 
+     * which have been GC'd).  The seq numbers of the target incrementing sequentially will
+     * be different from the source id's if some FSs were GC'd.
+     * 
+     * To determine for delta what new strings and new  
+     *
+     * @throws IOException
+     */
     private void serialize() throws IOException {   
 
-      if (doMeasurement) {
-        System.out.println(printCasInfo(cas));
-        sm.origAuxBytes = cas.getByteHeap().getSize();
-        sm.origAuxShorts = cas.getShortHeap().getSize() * 2;
-        sm.origAuxLongs = cas.getLongHeap().getSize() * 8;
-        sm.totalTime = System.currentTimeMillis();
-      }
-
-      /******************
-       * Process Header  
-       * Standardized    
-       ******************/
-      // encode: bits 7 6 5 4 3 2 1 0
-      //                        0 0 1 = no delta, no compression
-      //                        0 1 - = delta, no compression
-      //                        1 d - = compression, w/wo delta
-      int version = 4 | ((isDelta) ? 2 : 0);
-      CASSerializer.outputVersion(version, serializedOut);
-        
-      serializedOut.writeInt(0);  // reserved for future version info
-      if (doMeasurement) {
-        sm.header = 12;
+ //      if (doMeasurement) {
+//        System.out.println(printCasInfo(baseCas));
+//        sm.origAuxBytes = baseCas.getByteHeap().getSize();
+//        sm.origAuxShorts = baseCas.getShortHeap().getSize() * 2;
+//        sm.origAuxLongs = baseCas.getLongHeap().getSize() * 8;
+//        sm.totalTime = System.currentTimeMillis();
+//      }
+
+      /************************
+       * Write standard header
+       ************************/
+      CommonSerDes.createHeader()
+      .v3()
+      .seqVer(2)    // 0 - original, 1 - UIMA-4743, 2 - v3 
+      .form4()
+      .delta(isDelta)
+      .write(serializedOut);
+             
+      /*******************************************************************************
+       * Setup tables that map to v2 "addresses" - needed for backwards compatibility
+       *   fs2addr - feature structure to address
+       *   addr2fs - address to feature structure
+       *   sortedFSs - sorted by addr (sorted by id)
+       *******************************************************************************/
+      csds.setup();
+      
+      fs2seq.clear();
+      seq2fs.clear();
+      int seq = 1;  // origin 1
+      for (TOP fs : csds.sortedFSs) {
+        fs2seq.put(fs, seq);
+        seq2fs.put(seq++, fs);
+      }
+      
+      final List<TOP> newSortedFSs; // a sublist for delta cas of just the new ones
+      // may be null if there are no FSs to serialize
+      TOP firstFS = isDelta ? baseCas.getFsFromId(mark.getNextFSId()) : (csds.sortedFSs.size() == 0) ? null : csds.sortedFSs.get(0);
+      if (firstFS != null) {
+        if (isDelta) {
+          int i = Collections.binarySearch(csds.sortedFSs, firstFS);
+          assert(i >= 0);
+          newSortedFSs = csds.sortedFSs.subList(i, csds.sortedFSs.size());
+        } else {
+          newSortedFSs = csds.sortedFSs;
+        }
+      } else {
+        newSortedFSs = Collections.emptyList();
       }
-           
+       
+            
       /**************************
        * Strings
+       * For delta, to determine "new" strings that should be serialized, 
+       *   use the same method as used in Binary (plain) serialization.
        **************************/
-      int stringHeapStart = isDelta ? mark.nextStringHeapAddr : 1;
-      int stringHeapEnd = stringHeapObj.getSize();
- 
-      for (int i = stringHeapStart; i < stringHeapEnd; i++) {
-        os.add(stringHeapObj.getStringForCode(i));
+      for (TOP fs : newSortedFSs) {
+        extractStrings(fs);
       }
-      
-      // also add in all modified strings
-      // with current design, all modified strings are guaranteed
-      // to be above the mark, so this code is commented out
-//    addModStr:
-//      if (isDelta) {
-//        serializerForModifiedFSs.sortModifications();
-//        final int[] mods = serializerForModifiedFSs.modifiedMainHeapAddrs;
-//        final int modLen = serializerForModifiedFSs.modMainHeapAddrsLength;
-//        int nextMod = 0;
-//        if (modLen > 0) {
-// 
-//          for (int iHeap = 1; iHeap < heapStart; iHeap += incrToNextFs(heap, iHeap, typeInfo)) {
-//            typeInfo = getTypeInfo(heap[iHeap]);
-//            FsStringRefs fsStringRefs = new FsStringRefs(typeInfo, heap, iHeap);
-//            for (int nextRef = fsStringRefs.next(); nextRef >= 0; nextRef = fsStringRefs.next()) {
-//              while (nextRef > mods[nextMod]) {
-//                nextMod ++;
-//                if (nextMod == modLen) {
-//                  break addModStr;
-//                }
-//              }
-//              if (nextRef == mods[nextMod]) {
-//                os.add(stringHeapObj.getStringForCode(heap[nextRef]));
-//                nextMod ++;
-//                if (nextMod == modLen) {
-//                  break addModStr;
-//                }
-//              } 
-//            }
-//          }   
-//        }
-//      }
 
-      os.optimize();
-      String [] commonStrings = os.getCommonStrings();
-      writeVnumber(strChars_i, commonStrings.length);
-      for (int i = 0; i < commonStrings.length; i++) {
-        int startPos = dosZipSources[strChars_i].size();
-        DataIO.writeUTFv(commonStrings[i], dosZipSources[strChars_i]);
-        // approximate histogram
-        if (doMeasurement) {
-          // len is utf-8 encoding
-          float len = dosZipSources[strChars_i].size() - startPos;
-          // if len == chars, then all got coded as 1 byte
-          // if len > chars, some were utf-8 coded as 2 bytes
-          float excess = (len / commonStrings[i].length()) - 1;  // excess over length 1
-          int encAs2 = (int)(excess * commonStrings[i].length());
-          
-          // simulate histo for all the chars, as 1 or 2 byte UTF8 encoding 
-          sm.statDetails[strChars_i].countTotal += commonStrings[i].length(); // total chars accum
-          sm.statDetails[strChars_i].c[0] = commonStrings[i].length() - encAs2;
-          sm.statDetails[strChars_i].c[1] = encAs2;
-          sm.statDetails[strChars_i].lengthTotal += len;  // total as UTF-8 encode
+      if (isDelta) {
+        FsChange[] fssModified = baseCas.getModifiedFSList();
+      
+        // also add in all modified strings
+        for (FsChange fsChange : fssModified) {
+          extractStringsFromModifications(fsChange);
         }
       }
-      
-      only1CommonString = commonStrings.length == 1;
 
-      if (doMeasurement) {
-        long commonStringsLength = 0;
-        sm.stringsNbrCommon = commonStrings.length;
-        int r = 0;
-        for (int i = 0; i < commonStrings.length; i++) {
-          r += DataIO.lengthUTFv(commonStrings[i]);
-          commonStringsLength += commonStrings[i].length();
-        }
-        sm.stringsCommonChars = r;
-
-        sm.stringsSavedExact = os.getSavedCharsExact() * 2;
-        sm.stringsSavedSubstr = os.getSavedCharsSubstr() * 2;
-        sm.statDetails[strChars_i].original = os.getSavedCharsExact() * 2
-                                 + os.getSavedCharsSubstr() * 2
-                                 + commonStringsLength * 2;
-        sm.statDetails[strLength_i].original = (stringHeapEnd - stringHeapStart) * 4;
-        sm.statDetails[strOffset_i].original = (stringHeapEnd - stringHeapStart) * 4;
-      }
+      os.optimize();
       
+      writeStringInfo(os);
+            
       /***************************
        * Prepare to walk main heap
        ***************************/
-      writeVnumber(control_dos, heapEnd - heapStart);  
-      if (doMeasurement) {
-        sm.statDetails[Slot_MainHeap.i].original = (1 + heapEnd - heapStart) * 4;      
+      heapEnd = csds.heapEnd;
+      
+      
+      if (isDelta) {
+        // edge case - delta serializing with no new fs
+        heapStart = (null == firstFS) ? heapEnd : csds.fs2addr.get(firstFS);
+      } else {
+        heapStart = 0; // not 1, in order to match v2 semantics
+                       // is switched to 1 later
       }
       
-      resetIprevious();
+//      if (isDelta) {
+//        // debug
+//        for (TOP fs : csds.sortedFSs) {
+//          System.out.format("debug heapAddr: %,d type: %s%n", csds.fs2addr.get(fs), fs._typeImpl.getShortName());
+//          if (csds.fs2addr.get(fs) == 439) {
+//            System.out.println("debug");
+//          }
+//        }
+//        System.out.format("debug End of debug scan, heapStart: %,d heapEnd: %,d%n%n", heapStart, heapEnd);
+//      }
+      
+      writeVnumber(control_dos, heapEnd - heapStart);   // used for delta heap size to grow the CAS and ending condition on deser loop
+      
+      Arrays.fill(prevFsByType, null);
+
+//      if (heapStart == 0) {
+//        heapStart = 1;  // slot 0 not serialized, it's null / 0
+//      }
 
-      if (heapStart == 0) {
-        heapStart = 1;  // slot 0 not serialized, it's null / 0
-      }
 
-      if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
         // scan thru all fs and save their offsets in the heap
         // to allow conversion from addr to sequential fs numbers
-        initFsStartIndexes(fsStartIndexes, heap, heapStart, heapEnd, typeCodeHisto);
-        
-        for (int i = ts.getTypeArraySize() - 1; i >= 0; i--) {
-          serializedTypeCode2Code[i] = i;
-        }
-        
-        // set typeCode2serializeCode so that the 0th element is the typeCode with the highest frequency, etc.
-        Arrays.sort(serializedTypeCode2Code, 0, serializedTypeCode2Code.length, new Comparator<Integer>() {
-          public int compare(Integer o1, Integer o2) {
-            return (typeCodeHisto[o1] > typeCodeHisto[o2]) ? -1 :
-                   (typeCodeHisto[o1] < typeCodeHisto[o2]) ? 1 : 0;
-          }
-        });
-        
-//        for (int i = 0; i < serializedTypeCode2Code.length; i++) {
-//          int tCode = serializedTypeCode2Code[i];
-//          int c = typeCodeHisto[tCode];
-//          if (c > 0) {
-//            System.out.format("%2d %,9d instance of Type %s%n", i, c, typeInfoArray[tCode]);
-//          }
-//        }
+//        initFsStartIndexes(fsStartIndexes, heap, heapStart, heapEnd, typeCodeHisto);
         
-      }
-
-      
       
       /***************************
-       * walk main heap
+       * walk all fs's
+       * For delta, just those above the line
        ***************************/
-
-      for (int iHeap = heapStart; iHeap < heapEnd; iHeap += incrToNextFs(heap, iHeap, typeInfo)) {
-        int tCode = heap[iHeap];  // get type code      
-        typeInfo = getTypeInfo(tCode);
-        iPrevHeap = typeInfo.iPrevHeap;
-        
-        writeVnumber(typeCode_dos, tCode);
-
-        if (typeInfo.isHeapStoredArray) {
-          serializeHeapStoredArray(iHeap);
-        } else if (typeInfo.isArray) {
-          serializeNonHeapStoredArray(iHeap);
-        } else {
-          for (int i = 1; i < typeInfo.slotKinds.length + 1; i++) {
-            serializeByKind(iHeap, i);
-          }
-        }
-      
-        typeInfo.iPrevHeap = iHeap;  // make this one the "prev" one for subsequent testing
-        if (doMeasurement) {
-          sm.statDetails[typeCode_i].incr(DataIO.lengthVnumber(tCode));
-          sm.mainHeapFSs ++;
-        }
-      }  // end of heap walk
+      for (TOP fs : newSortedFSs) {
+        writeFs(fs);
+      }
       
       serializeIndexedFeatureStructures();
 
@@ -698,9 +648,84 @@ public class BinaryCasSerDes4 {
       }
     }
     
+    /** 
+     * Write the compressed string table(s)
+     * @throws IOException
+     */
+    private void writeStringInfo(OptimizeStrings os) throws IOException {
+      String [] commonStrings = os.getCommonStrings();
+      writeVnumber(strChars_i, commonStrings.length);
+      DataOutputStream out = dosZipSources[strChars_i];
+      for (int i = 0; i < commonStrings.length; i++) {
+        int startPos = doMeasurements ? out.size() : 0;
+        DataIO.writeUTFv(commonStrings[i], out);
+        // approximate histogram
+        if (doMeasurements) {
+          // len is utf-8 encoding
+          float len = out.size() - startPos;
+          // if len == chars, then all got coded as 1 byte
+          // if len > chars, some were utf-8 coded as 2 bytes
+          float excess = (len / commonStrings[i].length()) - 1;  // excess over length 1
+          int encAs2 = (int)(excess * commonStrings[i].length());
+          
+          // simulate histo for all the chars, as 1 or 2 byte UTF8 encoding 
+          sm.statDetails[strChars_i].countTotal += commonStrings[i].length(); // total chars accum
+          sm.statDetails[strChars_i].c[0] = commonStrings[i].length() - encAs2;
+          sm.statDetails[strChars_i].c[1] = encAs2;
+          sm.statDetails[strChars_i].lengthTotal += len;  // total as UTF-8 encode
+        }
+      }
+      
+      only1CommonString = commonStrings.length == 1;
+
+      if (doMeasurements) {
+//        long commonStringsLength = 0;
+//        sm.stringsNbrCommon = commonStrings.length;
+//        int r = 0;
+//        for (int i = 0; i < commonStrings.length; i++) {
+//          r += DataIO.lengthUTFv(commonStrings[i]);
+//          commonStringsLength += commonStrings[i].length();
+//        }
+//        sm.stringsCommonChars = r;
+  //
+//        sm.stringsSavedExact = os.getSavedCharsExact() * 2;
+//        sm.stringsSavedSubstr = os.getSavedCharsSubstr() * 2;
+//        sm.statDetails[strChars_i].original = os.getSavedCharsExact() * 2
+//                                 + os.getSavedCharsSubstr() * 2
+//                                 + commonStringsLength * 2;
+//        final int stringHeapStart = isSerializingDelta ? mark.nextFSId : 1;
+//        final int stringHeapEnd = stringHeapObj.getSize();
+//        sm.statDetails[strLength_i].original = (stringHeapEnd - stringHeapStart) * 4;
+//        sm.statDetails[strOffset_i].original = (stringHeapEnd - stringHeapStart) * 4;
+      }
+
+    }  
+
+    private void writeFs(TOP fs) throws IOException {
+      TypeImpl type = fs._typeImpl;
+      int typeCode = type.getCode();
+      writeVnumber(typeCode_dos, typeCode);
+      
+      prevFs = prevFsByType[typeCode];            
+
+      if (type.isArray()) {
+        serializeArray(fs);
+      } else {
+        for (FeatureImpl feat : type.getFeatureImpls()) {
+          serializeByKind(fs, feat);
+        }
+      }
+    
+      prevFsByType[typeCode] = fs;
+//      if (doMeasurement) {
+//        sm.statDetails[typeCode_i].incr(DataIO.lengthVnumber(tCode));
+//        sm.mainHeapFSs ++;
+//      }
+    }
     
     private void serializeIndexedFeatureStructures() throws IOException {
-      int[] fsIndexes = isDelta ? cas.getDeltaIndexedFSs(mark) : cas.getIndexedFSs();
+      // fsIndexes already have the modelled address conversion
+      int[] fsIndexes = isDelta ? bcsd.getDeltaIndexedFSs(mark, csds.fs2addr) : bcsd.getIndexedFSs(csds.fs2addr);
       if (doMeasurement) {
         sm.statDetails[fsIndexes_i].original = fsIndexes.length * 4 + 1;      
       }
@@ -746,13 +771,10 @@ public class BinaryCasSerDes4 {
       System.arraycopy(fsIndexes, ix, ia, 0, nbrEntries);
       Arrays.sort(ia);
      
-      int prev = 0;
-      
+      int prev = 0;           
       for (int i = 0; i < ia.length; i++) {
         int v = ia[i];
-        if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
-          v = fsStartIndexes.getItemIndex(v);
-        }
+        v = fs2seq(csds.addr2fs.get(v));  // v2 wrote the sequential number
         writeVnumber(fsIndexes_dos, v - prev);
         if (doMeasurement) {
           sm.statDetails[fsIndexes_i].incr(DataIO.lengthVnumber(v - prev));
@@ -763,124 +785,158 @@ public class BinaryCasSerDes4 {
       return end;
     } 
 
-    private void serializeHeapStoredArray(int iHeap) throws IOException {
-      final int length = serializeArrayLength(iHeap);
-      // output values
+    private void serializeArray(TOP fs) throws IOException {
+      final int length = serializeArrayLength(fs);
       // special case 0 and 1st value
       if (length == 0) {
         return;
       }
-      SlotKind arrayElementKind = typeInfo.slotKinds[1];
-      final int endi = iHeap + length + 2;
+      final TypeImpl type = fs._typeImpl;
+      
+      // output values
+
+      SlotKind arrayElementKind = type.getComponentSlotKind();
       switch (arrayElementKind) {
-      case Slot_HeapRef: case Slot_Int: case Slot_Short:
-        {
-          int prev = (iPrevHeap == 0) ? 0 :
-                     (heap[iPrevHeap + 1] == 0) ? 0 :
-                      heap[iPrevHeap + 2]; 
-          for (int i = iHeap + 2; i < endi; i++) {
-            prev = writeIntOrHeapRef(arrayElementKind.i, i, prev);
-          }
+      case Slot_HeapRef: {
+        int prev = getPrevArray0HeapRef();
+        for (TOP item : ((FSArray)fs)._getTheArray()) {
+          int v = fs2seq(item);
+          writeDiff(arrayElementKind.ordinal(), v, prev);
+          prev = v;
+        }
+        break;
+      }
+      case Slot_Int: {
+        int prev = getPrevArray0Int();
+        for (int item : ((IntegerArray)fs)._getTheArray()) {
+          writeDiff(arrayElementKind.ordinal(), item, prev);
+          prev = item;
+        }
+        break;
+      }
+      
+      case Slot_ShortRef: { 
+        int prev = 0;
+        for (int item : ((ShortArray)fs)._getTheArray()) {
+          writeDiff(short_i, item, prev);
+          prev = item;
         }
         break;
-      case Slot_Float: 
-        for (int i = iHeap + 2; i < endi; i++) {
-          writeFloat(heap[i]);
+      }
+      case Slot_Float:  
+        for (float item : ((FloatArray)fs)._getTheArray()) {
+          writeFloat(CASImpl.float2int(item));
         }
         break;
+        
       case Slot_StrRef:
-        for (int i = iHeap + 2; i < endi; i++) {
-          writeString(stringHeapObj.getStringForCode(heap[i]));
+        for (String item : ((StringArray)fs)._getTheArray()) {
+          writeString(item);
         }
         break;
+      
+      case Slot_BooleanRef:
+        for (boolean item : ((BooleanArray)fs)._getTheArray()) {
+          byte_dos.write(item ? 1 : 0);
+        }
+        break;
+        
+      case Slot_ByteRef:
+        byte_dos.write(((ByteArray)fs)._getTheArray());
+        break;
         
-      default: throw new RuntimeException("internal error");
+      case Slot_LongRef: {
+        long prev = 0;
+        for (long item : ((LongArray)fs)._getTheArray()) {
+          writeLong(item, prev);
+          prev = item;
+        }
+        break;
+      }
+
+      case Slot_DoubleRef: 
+        for (double item : ((DoubleArray)fs)._getTheArray()) {
+          writeDouble(CASImpl.double2long(item));
+        }
+        break;
+      
+       default: Misc.internalError();
       } // end of switch    
     }
     
-    private int writeIntOrHeapRef(int kind, int index, int prev) throws IOException {
-      final int v = heap[index];
-      writeDiff(kind, v, prev);
-      return v;
+    private int getPrevArray0HeapRef() {
+      if (isNoPrevArrayValue((CommonArray)prevFs)) return 0;      
+      return fs2seq(((FSArray)prevFs).get(0));
     }
     
-    private long writeLongFromHeapIndex(int index, long prev) throws IOException {
-      final long v = longHeapObj.getHeapValue(heap[index]);      
-      writeLong(v, prev); 
-      return v;
+    private int getPrevArray0Int() {
+      if (isNoPrevArrayValue((CommonArray)prevFs)) return 0;      
+      return ((IntegerArray)prevFs).get(0);      
     }
-    
-    private void serializeNonHeapStoredArray(int iHeap) throws IOException {
-      final int length = serializeArrayLength(iHeap);
-      if (length == 0) {
-        return;
-      }
-      SlotKind refKind = typeInfo.getSlotKind(2);
-      switch (refKind) {
-      case Slot_BooleanRef: case Slot_ByteRef:
-        writeFromByteArray(refKind, heap[iHeap + 2], length);
-        if (doMeasurement) {
-          sm.statDetails[byte_i].incr(1);
-          sm.origAuxByteArrayRefs += 4;
-        }
-        break; 
-      case Slot_ShortRef:
-        writeFromShortArray(heap[iHeap + 2], length);
-        if (doMeasurement) {
-          sm.origAuxShortArrayRefs += 4;
-        }
-        break; 
-      case Slot_LongRef: case Slot_DoubleRef:
-        writeFromLongArray(refKind, heap[iHeap + 2], length);
-        if (doMeasurement) {
-          sm.origAuxLongArrayRefs += 4;
-        }
-        break; 
-      default:
-        throw new RuntimeException();
-      }
+
+    private boolean isNoPrevArrayValue(CommonArray prevFs) {
+      return prevFs == null || prevFs.size() == 0;
     }
-    
-    private void serializeByKind(int iHeap, int offset) throws IOException {
-      SlotKind kind = typeInfo.getSlotKind(offset);      
+                
+    private void serializeByKind(TOP fs, FeatureImpl feat) throws IOException {
+      SlotKind kind = feat.getSlotKind();      
       switch (kind) {
-      //Slot_Int, Slot_Float, Slot_Boolean, Slot_Byte, Slot_Short
-      case Slot_Int: case Slot_Short: case Slot_HeapRef:
-        serializeDiffWithPrevTypeSlot(kind, iHeap, offset);
+      case Slot_Int: {
+        final int prev = (prevFs == null) ? 0 : prevFs.getIntValueNc(feat);
+        final int v = fs.getIntValue(feat);
+//        if (TRACE_INT) System.out.format("writeInt value: %,d prev: %,d%n", v, prev); 
+        writeDiff(kind.ordinal(), v, prev);
+        break;
+      }
+      
+      case Slot_Short: 
+        writeDiff(kind.ordinal(), fs.getShortValueNc(feat), (prevFs == null) ? 0 : prevFs.getShortValueNc(feat));
+        break;
+        
+      case Slot_HeapRef:
+        final TOP ref = fs.getFeatureValueNc(feat);
+        writeDiff(kind.ordinal(), fs2seq(ref), 
+           (prevFs == null) ? 0 : fs2seq(prevFs.getFeatureValueNc(feat)));
         break;
+        
       case Slot_Float:
-        writeFloat(heap[iHeap + offset]);
+        writeFloat(CASImpl.float2int(fs.getFloatValueNc(feat)));
+        break;
+        
+      case Slot_Boolean:
+        byte_dos.write(fs.getBooleanValueNc(feat) ? 1 : 0);
         break;
-      case Slot_Boolean: case Slot_Byte:
-        byte_dos.write(heap[iHeap + offset]);
+        
+      case Slot_Byte:
+        byte_dos.write(fs.getByteValueNc(feat));
         break;
+        
       case Slot_StrRef: 
-        writeString(stringHeapObj.getStringForCode(heap[iHeap + offset]));
+        writeString(fs.getStringValueNc(feat));
         break;
-      case Slot_LongRef: 
-        writeLongFromHeapIndex(iHeap + offset, 
-                  (iPrevHeap == 0) ? 
-                    0L : 
-                    longHeapObj.getHeapValue(heap[iPrevHeap + offset]));
+        
+      case Slot_LongRef:
+        writeLong(fs.getLongValueNc(feat), (prevFs == null) ? 0L : prevFs.getLongValueNc(feat));
         break;
+
       case Slot_DoubleRef: 
-        writeDouble(longHeapObj.getHeapValue(heap[iHeap + offset]));
+        writeDouble(CASImpl.double2long(fs.getDoubleValueNc(feat)));
         break;
-      default: 
-        throw new RuntimeException("internal error");
+        
+      default: Misc.internalError(); 
       } // end of switch
     }
     
-    private int serializeArrayLength(int iHeap) throws IOException {
-      final int length = heap[iHeap + 1];
+    private int serializeArrayLength(TOP fs) throws IOException {
+      int length = ((CommonArray)fs).size();
       writeVnumber(arrayLength_i, length);
       return length;
     }
     
-    private void serializeDiffWithPrevTypeSlot(SlotKind kind, int iHeap, int offset) throws IOException {
-      int prev = (iPrevHeap == 0) ? 0 : heap[iPrevHeap + offset];
-      writeDiff(kind.i, heap[iHeap + offset], prev);
-    }
+//    private void serializeDiffWithPrevTypeSlot(SlotKind kind, TOP fs, FeatureImpl feat) throws IOException {
+//      int prev = (prevFs == null) ? 0 : fs.setIntLikeValue(slotKind, fi, v);getheap[iPrevHeap + offset];
+//      writeDiff(kind.ordinal(), heap[iHeap + offset], prev);
+//    }
     
     /**
      * Method:
@@ -945,13 +1001,6 @@ public class BinaryCasSerDes4 {
 //      }
 //      return dos;
 //    }
-    
-    public DataOutputStream setupOutputStream(int i) {
-      // set up output stream
-      int size = estimatedZipSize[i];
-      baosZipSources[i] = new ByteArrayOutputStream(size);
-      return dosZipSources[i] = new DataOutputStream(baosZipSources[i]); 
-    }
 
     private void writeLong(long v, long prev) throws IOException {
       writeDiff(long_High_i, (int)(v >>> 32), (int)(prev >>> 32));
@@ -1097,6 +1146,9 @@ public class BinaryCasSerDes4 {
       final long mants = (revMants << 1) + ((raw < 0) ? 1 : 0);
       writeVnumber(double_Exponent_dos, exponent);
       writeVnumber(double_Mantissa_Sign_dos, mants);
+      if (TRACE_DOUBLE) {
+        System.out.format("write Double: raw = %,d, exponent = %,d, mantissa + lowbit sign: %,d%n", raw, exponent, mants);
+      }
       if (doMeasurement) {
         sm.statDetails[double_Exponent_i].incr(DataIO.lengthVnumber(exponent));
         sm.statDetails[double_Mantissa_Sign_i].incr(DataIO.lengthVnumber(mants));
@@ -1137,13 +1189,6 @@ public class BinaryCasSerDes4 {
         return;
       }
     
-      if (CHANGE_FS_REFS_TO_SEQUENTIAL && (kind == heapRef_i)) {
-        v = fsStartIndexes.getItemIndex(v);
-        if (prev != 0) {
-          prev = fsStartIndexes.getItemIndex(prev);
-        }
-      }
-
       final int absV = Math.abs(v);
       if (((v > 0) && (prev > 0)) ||
           ((v < 0) && (prev < 0))) {
@@ -1171,37 +1216,60 @@ public class BinaryCasSerDes4 {
         sm.statDetails[kind].valueLeDiff ++;
       }
     }
-
-    private void writeFromByteArray(SlotKind kind, int startPos, int length) throws IOException {
-      byte_dos.write(byteHeapObj.heap, startPos, length);
-    }
-
-    private void writeFromLongArray(SlotKind kind, int startPos, int length) throws IOException {
-      final long[] h = longHeapObj.heap;
-      final int endPos = startPos + length;
-      long prev = 0;
-      for (int i = startPos; i < endPos; i++) {
-        final long e = h[i];
-        if (kind == Slot_DoubleRef) {
-          writeDouble(e);
-        } else {
-          writeLong(e, prev);
-          prev = e;
-        }
+    
+    /**
+     * add strings to the optimizestrings object
+      * 
+     * If delta, only process for fs's that are new; 
+     * modified string values picked up when scanning FsChange items
+     * @param fs feature structure
+     */
+    private void extractStrings(TOP fs) {
+      if (isDelta && !mark.isNew(fs)) {
+        return;
       }
+      TypeImpl type = fs._typeImpl;
+     
+      if (type.isArray()) {
+        if (type.getComponentSlotKind() == SlotKind.Slot_StrRef) {
+          for (String s : ((StringArray)fs)._getTheArray()) {
+            os.add(s);
+          }
+        }
+      } else {  // end of is-array        
+        for (FeatureImpl feat : type.getFeatureImpls()) {
+          if (feat.getSlotKind() == SlotKind.Slot_StrRef) {
+            os.add(fs.getStringValueNc(feat));
+          }
+        } // end of iter over all features
+      } // end of if-is-not-array
     }
     
-    private void writeFromShortArray(int startPos, int length) throws IOException {
-      final short[] h = shortHeapObj.heap;
-      final int endPos = startPos + length;
-      int prev = 0;
-      for (int i = startPos; i < endPos; i++) {
-        final short e = h[i];
-        writeDiff(short_i, e, prev);
-        prev = e;
-      }
+    /**
+     * For delta, for each fsChange element, extract any strings
+     * @param fsChange
+     */
+    private void extractStringsFromModifications(FsChange fsChange) {
+      final TOP fs = fsChange.fs;
+      final TypeImpl type = fs._typeImpl;
+      if (fsChange.arrayUpdates != null) {
+        if (type.getComponentSlotKind() == SlotKind.Slot_StrRef) {
+          String[] sa = ((StringArray)fs)._getTheArray();  
+          fsChange.arrayUpdates.forAllInts(index -> {
+            os.add(sa[index]);
+          });
+        } // end of is string array
+      } else { // end of is array
+        BitSet fm = fsChange.featuresModified;
+        for (int offset = fm.nextSetBit(0); offset >= 0; offset = fm.nextSetBit(offset + 1)) {
+          FeatureImpl feat = type.getFeatureImpls().get(offset);
+          if (feat.getSlotKind() == SlotKind.Slot_StrRef) {
+            os.add(fs.getStringValueNc(feat));
+          }
+        } // end of iter over features
+      } // end of is-not-array
     }
-
+    
     /******************************************************************************
      * Modified Values
      * Output:
@@ -1209,47 +1277,24 @@ public class BinaryCasSerDes4 {
      *     write the heap addr of the FS
      *     
      *     For all modified values within the FS:
-     *       if it is an aux array element, write the index in the aux array and the new value
+     *       if it is an aux array element, write the index in the individual array instance and the new value
      *       otherwise, write the slot offset and the new value
      ******************************************************************************/
     public class SerializeModifiedFSs {
 
-      final int[] modifiedMainHeapAddrs = cas.getModifiedFSHeapAddrs().toArray();
-      final int[] modifiedFSs = cas.getModifiedFSList().toArray();
-      final int[] modifiedByteHeapAddrs = cas.getModifiedByteHeapAddrs().toArray();
-      final int[] modifiedShortHeapAddrs = cas.getModifiedShortHeapAddrs().toArray();
-      final int[] modifiedLongHeapAddrs = cas.getModifiedLongHeapAddrs().toArray();
-
-      {sortModifications();}
-      
-      final int modMainHeapAddrsLength = eliminateDuplicatesInMods(modifiedMainHeapAddrs);
-      final int modFSsLength = eliminateDuplicatesInMods(modifiedFSs);
-      final int modByteHeapAddrsLength = eliminateDuplicatesInMods(modifiedByteHeapAddrs);
-      final int modShortHeapAddrsLength = eliminateDuplicatesInMods(modifiedShortHeapAddrs);
-      final int modLongHeapAddrsLength = eliminateDuplicatesInMods(modifiedLongHeapAddrs);
-
-      // ima           - index into modified arrays
-      // ixx, iPrevxxx - index in heap being changed
-      //                 value comes via the main heap or aux heaps
-      
-      int imaModMainHeap = 0;
-      int imaModByteRef = 0;
-      int imaModShortRef = 0;
-      int imaModLongRef = 0;
- 
       // previous value - for things diff encoded
       int vPrevModInt = 0;
       int vPrevModHeapRef = 0;
       short vPrevModShort = 0;
       long vPrevModLong = 0;
-      
-      int iHeap;
-      TypeInfo typeInfo;
-      
+            
       private void serializeModifiedFSs() throws IOException {
-        iPrevHeap = 0;   
+        
+        int iPrevAddr = 0; 
+        
+        FsChange[] fsChanges = baseCas.getModifiedFSList();  
         // write out number of modified Feature Structures
-        writeVnumber(control_dos, modFSsLength);
+        writeVnumber(control_dos, fsChanges.length);
         // iterate over all modified feature structures
         /**
          * Theorems about these data
@@ -1259,196 +1304,199 @@ public class BinaryCasSerDes4 {
          *   3) Assumption: String array element modifications are main heap slot changes
          *      and recorded as such
          */
-        
-        for (int i = 0; i < modFSsLength; i++) {
-          iHeap = modifiedFSs[i];     
-          final int tCode = heap[iHeap];
-          typeInfo = getTypeInfo(tCode);
-          
+
+        for (FsChange fsChange : fsChanges) {
+
+          TOP fs = fsChange.fs;
+          TypeImpl ti = fs._typeImpl;
+          final int addr = csds.fs2addr.get(fs);
           // write out the address of the modified FS
-          writeVnumber(fsIndexes_dos, iHeap - iPrevHeap);
-          // delay updating iPrevHeap until end of "for" loop
+          writeVnumber(fsIndexes_dos, addr - iPrevAddr);
+          // delay updating iPrevAddr until end of "for" loop
           
           /**************************************************
            * handle aux byte, short, long array modifications
            **************************************************/
-          if (typeInfo.isArray && (!typeInfo.isHeapStoredArray)) {
-            writeAuxHeapMods();           
+          if (ti.isArray() && !ti.isHeapStoredArray()) {
+            writeAuxHeapMods(fsChange);           
           } else { 
-            writeMainHeapMods(); 
+            writeMainHeapMods(fsChange); 
           }  // end of processing 1 modified FS
-          iPrevHeap = iHeap;
+          iPrevAddr = addr;
         }  // end of for loop over all modified FSs
       }  // end of method
-      
-      // sort and remove duplicates
-      private void sortModifications() {
-        Arrays.sort(modifiedMainHeapAddrs);
-        Arrays.sort(modifiedFSs);
-        Arrays.sort(modifiedByteHeapAddrs);
-        Arrays.sort(modifiedShortHeapAddrs);
-        Arrays.sort(modifiedLongHeapAddrs);
-      }
-      
-      private int eliminateDuplicatesInMods(final int[] sorted) {
-        int length = sorted.length;
-        if (length < 2) {
-          return length;
-        }
-        
-        int prev = sorted[0];
-        int to = 1;
-        for(int from = 1; from < length; from++) {
-          int s = sorted[from];
-          if (s == prev) {
-            continue;
-          }
-          prev = s;
-          sorted[to] = s;
-          to++;
-        }    
-        return to;  // to is length
-      }
-
-      private int countModifiedSlotsInFs(int fsLength) {
-        return countModifiedSlots(iHeap, fsLength, modifiedMainHeapAddrs, imaModMainHeap, modMainHeapAddrsLength);
-      }
-      
-      private int countModifiedSlotsInAuxHeap(int[] modifiedAddrs, int indexInModAddrs, int length) {
-        return countModifiedSlots(heap[iHeap + 2], heap[iHeap + 1], modifiedAddrs, indexInModAddrs, length);
-      }
-      
-      private int countModifiedSlots(int firstAddr, int length, int[] modifiedAddrs, int indexInModAddrs, int modAddrsLength) {
-        if (0 == length) {
-          throw new RuntimeException();  // can't happen
-        }
-        final int nextAddr = firstAddr + length;
-        int nextModAddr = modifiedAddrs[indexInModAddrs]; 
-        if ((firstAddr > nextModAddr) ||
-            (nextModAddr >= nextAddr)) {
-          throw new RuntimeException(); // never happen - must have one slot at least modified in this fs          
-        }
-        int i = 1;
-        for (;; i++) {
-          if ((indexInModAddrs + i) == modAddrsLength) {
-            break;
-          }
-          nextModAddr = modifiedAddrs[indexInModAddrs + i];
-          if (nextModAddr >= nextAddr) {
-            break;
-          }
-        }
-        return i;
-      }
-      
-      private void writeMainHeapMods() throws IOException {
-        final int fsLength = incrToNextFs(heap, iHeap, typeInfo);
-        final int numberOfModsInFs = countModifiedSlotsInFs(fsLength);
-        writeVnumber(fsIndexes_dos, numberOfModsInFs);
-        int iPrevOffsetInFs = 0;
-
-        for (int i = 0; i < numberOfModsInFs; i++) {
-          final int nextMainHeapIndex = modifiedMainHeapAddrs[imaModMainHeap++];
-          final int offsetInFs = nextMainHeapIndex - iHeap;
-          
-          writeVnumber(fsIndexes_dos, offsetInFs - iPrevOffsetInFs);
-          iPrevOffsetInFs = offsetInFs;
+           
+      private void writeMainHeapMods(FsChange fsChange) throws IOException {
+        int nbrOfMods = (fsChange.arrayUpdates == null) 
+                          ? fsChange.featuresModified.cardinality() 
+                          : fsChange.arrayUpdates.size();
+        writeVnumber(fsIndexes_dos, nbrOfMods);
+        
+        final TOP fs = fsChange.fs;
+        
+        if (fsChange.arrayUpdates == null) {
+          List<FeatureImpl> features = fs._typeImpl.getFeatureImpls();
+          int iPrevOffsetInFs = 0;
+          final BitSet bs = fsChange.featuresModified;
+          for (int i = bs.nextSetBit(0); i >= 0; i = bs.nextSetBit(i+1)) {
+            FeatureImpl feat = features.get(i);
+            // next +1 to conform to v2 encoding of feat offsets
+            writeVnumber(fsIndexes_dos, i + 1 - iPrevOffsetInFs);
+            iPrevOffsetInFs = i + 1;
+            
+            final SlotKind kind = feat.getSlotKind();
+            final int kindi = kind.ordinal();
+            
+            switch (kind) {
+            case Slot_Boolean:
+              byte_dos.write(fs.getBooleanValueNc(feat) ? 1 : 0);
+              break;
+              
+            case Slot_Byte:
+              byte_dos.write(fs.getByteValueNc(feat));
+              break;
+              
+            case Slot_Short: {
+              final short v = fs.getShortValueNc(feat);            
+              writeDiff(kindi, v, vPrevModShort);
+              vPrevModShort = v;
+              break;
+            }
+              
+            case Slot_Int: {
+              final int v = fs.getIntValueNc(feat);
+              writeDiff(kindi, v, vPrevModInt);
+              vPrevModInt = v;
+              break;
+            }
+  
+            case Slot_Float:
+              writeFloat(CASImpl.float2int(fs.getFloatValueNc(feat)));
+              break;
+              
+            case Slot_LongRef: {
+              long v = fs.getLongValueNc(feat);  
+              writeLong(v, vPrevModLong);
+              vPrevModLong = v;
+              break;
+            }
+  
+            case Slot_DoubleRef:
+              writeDouble(CASImpl.double2long(fs.getDoubleValueNc(feat)));
+              break;
+  
+            case Slot_HeapRef: {
+              int v = fs2seq(fs.getFeatureValueNc(feat));
+              writeDiff(kindi, v, vPrevModHeapRef);
+              vPrevModHeapRef = v;
+              break;
+            }
+  
+            case Slot_StrRef: 
+              writeString(fs.getStringValueNc(feat));
+              break;
+              
+            default: Misc.internalError();
+  
+            } // end of switch
+          }  // end of looping for all modified slots in this FS
+        } else { // end of processing of features
+          // heap stored arrays
+          TypeImpl type = fs._typeImpl;
+          SlotKind kind = type.getComponentSlotKind();
+          int kindi = kind.ordinal();
           
-          final SlotKind kind = typeInfo.getSlotKind(typeInfo.isArray ? 2 : offsetInFs);
+          IntListIterator it = fsChange.arrayUpdates.iterator();
+          while (it.hasNext()) {
+            int i = it.next();
+            // write the offset of the of the modified entry
+            //   from the beginning of the fs addr
+            //   i is already the 0 based offset, make it a 2 based one
+            //   to account for the type code and length in v2 layout
+            writeVnumber(fsIndexes_dos, i + 2);
+            
+            switch(kind) {
+            case Slot_Int: {
+              final int v = ((IntegerArray)fs).get(i);
+              writeDiff(kindi, v, vPrevModInt);
+              vPrevModInt = v;
+              break;
+            }
+  
+            case Slot_Float:
+              writeFloat(CASImpl.float2int(((FloatArray)fs).get(i))); 
+              break;
+              
+            case Slot_StrRef:
+              writeString(((StringArray)fs).get(i));
+              break;
+              
+            case Slot_HeapRef:
+              int v = fs2seq(((FSArray)fs).get(i));
+              writeDiff(kindi, v, vPrevModHeapRef);
+              vPrevModHeapRef = v;
+              break;
+              
+            default: Misc.internalError();
+            } // end of switch
+          } // end of iteration over all changed slots in one array
+        } // end of if statement for processing arrays
+      } // end of method
+      
+      private void writeAuxHeapMods(FsChange fsChange) throws IOException {
+        final TOP fs = fsChange.fs;
+        final TypeImpl type = fs._typeImpl;
+        
+        int iPrevOffset = 0;
+        
+        final SlotKind kind = type.getComponentSlotKind();
+        
+        writeVnumber(fsIndexes_dos, fsChange.arrayUpdates.size());
+        
+        IntListIterator it = fsChange.arrayUpdates.iterator();
+        while (it.hasNext()) {
+          int i = it.next();
+                    
+          writeVnumber(fsIndexes_dos, i - iPrevOffset);
+          iPrevOffset = i;
           
-          switch (kind) {
-          case Slot_HeapRef:
-            vPrevModHeapRef = writeIntOrHeapRef(heapRef_i, nextMainHeapIndex, vPrevModHeapRef);
-            break;
-          case Slot_Int:
-            vPrevModInt = writeIntOrHeapRef(int_i, nextMainHeapIndex, vPrevModInt);
-            break;
-          case Slot_Short:
-            vPrevModShort = (short)writeIntOrHeapRef(int_i, nextMainHeapIndex, vPrevModShort);             
-            break;
-          case Slot_LongRef:
-            vPrevModLong = writeLongFromHeapIndex(nextMainHeapIndex, vPrevModLong); 
-            break;
-          case Slot_Byte: case Slot_Boolean:
-            byte_dos.write(heap[nextMainHeapIndex]);
-            break;
-          case Slot_Float:
-            writeFloat(heap[nextMainHeapIndex]);
-            break;
-          case Slot_StrRef:
-            writeString(stringHeapObj.getStringForCode(heap[nextMainHeapIndex]));
-            break;
-          case Slot_DoubleRef:
-            writeDouble(longHeapObj.getHeapValue(heap[nextMainHeapIndex]));
+          switch(kind) {
+          case Slot_BooleanRef: byte_dos.write(((BooleanArray)fs).get(i) ? 1 : 0); break;
+          case Slot_ByteRef:    byte_dos.write(((ByteArray   )fs).get(i)        ); break;
+          case Slot_ShortRef: {
+            short v = ((ShortArray)fs).get(i);
+            writeDiff(int_i, v, vPrevModShort);
+            vPrevModShort = v;
             break;
-          default:
-            throw new RuntimeException();
           }
-
-        }  // end of looping for all modified slots in this FS
-      }
-      
-      private void writeAuxHeapMods() throws IOException {
-        final int auxHeapIndex = heap[iHeap + 2];
-        int iPrevOffsetInAuxArray = 0;
-        
-        final SlotKind kind = typeInfo.getSlotKind(2);  // get kind of element
-        final boolean isAuxByte = ((kind == Slot_BooleanRef) || (kind == Slot_ByteRef));
-        final boolean isAuxShort = (kind == Slot_ShortRef);
-        final boolean isAuxLong = ((kind == Slot_LongRef) || (kind == Slot_DoubleRef));
-        
-        if (!(isAuxByte | isAuxShort | isAuxLong)) {
-          throw new RuntimeException();  // never happen
-        }
-        
-        final int[] modXxxHeapAddrs = isAuxByte  ? modifiedByteHeapAddrs :
-                                      isAuxShort ? modifiedShortHeapAddrs :
-                                                   modifiedLongHeapAddrs;
-        final int modXxxHeapAddrsLength = isAuxByte  ? modByteHeapAddrsLength :
-                                          isAuxShort ? modShortHeapAddrsLength :
-                                                       modLongHeapAddrsLength;
-        int imaModXxxRef = isAuxByte  ? imaModByteRef :
-                                 isAuxShort ? imaModShortRef : 
-                                              imaModLongRef;
-        
-        final int numberOfModsInAuxHeap = countModifiedSlotsInAuxHeap(modXxxHeapAddrs, imaModXxxRef, modXxxHeapAddrsLength);
-        writeVnumber(fsIndexes_dos, numberOfModsInAuxHeap);
-        
-        for (int i = 0; i < numberOfModsInAuxHeap; i++) {
-          final int nextModAuxIndex = modXxxHeapAddrs[imaModXxxRef++];
-          final int offsetInAuxArray = nextModAuxIndex - auxHeapIndex;
           
-          writeVnumber(fsIndexes_dos, offsetInAuxArray - iPrevOffsetInAuxArray);
-          iPrevOffsetInAuxArray = offsetInAuxArray;
-          
-          if (isAuxByte) {
-            writeUnsignedByte(byte_dos, byteHeapObj.getHeapValue(nextModAuxIndex));
-          } else if (isAuxShort) {
-            final short v = shortHeapObj.getHeapValue(nextModAuxIndex);
-            writeDiff(int_i, v, vPrevModShort);
-            vPrevModShort = v;
-          } else {
-            long v = longHeapObj.getHeapValue(nextModAuxIndex);
-            if (kind == Slot_LongRef) {
-              writeLong(v, vPrevModLong);
-              vPrevModLong = v;    
-            } else {
-              writeDouble(v);
-            }
+          case Slot_LongRef: {
+            long v = ((LongArray)fs).get(i);
+            writeLong(v, vPrevModLong);
+            vPrevModLong = v;
+            break;
           }
           
-          if (isAuxByte) {
-            imaModByteRef++;
-          } else if (isAuxShort) {
-            imaModShortRef++;
-          } else {
-            imaModLongRef++;
+          case Slot_DoubleRef: {
+            double v = ((DoubleArray)fs).get(i);
+            writeDouble(CASImpl.double2long(v));
+            break;
           }
           
-        }
-      }
+          default: Misc.internalError();
+          } // end of switch
+        } // end of iteration over items changed in the array
+      } // end of method
     } // end of class definition for SerializeModifiedFSs
     
+    private int fs2seq(TOP fs) {
+      return (fs == null) ? 0 : fs2seq.get(fs);
+    }
+
+//    private TOP seq2fs(int s) {
+//      return (s == 0) ? null : seq2fs.get(s);
+//    }
+    
   }  // end of class definition for Serializer
   
   /**
@@ -1458,39 +1506,51 @@ public class BinaryCasSerDes4 {
    */
   private class Deserializer {
     
-    final private CASImpl cas;  // cas being deserialized into
+    final private CASImpl baseCas;  // cas being deserialized into
+    final private CASImpl ivCas;    // initial view cas - where by default new fs are created
+    final private BinaryCasSerDes bcsd;  
+    final private CommonSerDesSequential csds;
     final private DataInput deserIn;
 
     final private DataInputStream[] dataInputs = new DataInputStream[NBR_SLOT_KIND_ZIP_STREAMS];
     private Inflater[] inflaters = new Inflater[NBR_SLOT_KIND_ZIP_STREAMS];
 
-    private int[] heap;           // main heap
+    /** the FS being deserialized */
+    private TOP currentFs;
+
+    /** 
+     * the deferrals needed when deserializing a subtype of AnnotationBase before the sofa is known
+     * Also for Sofa creation where some fields are final
+     * */
+    final private List<Runnable> singleFsDefer = new ArrayList<>(); 
+    
+    /** used for deferred creation */
+    private int sofaNum;
+    private String sofaName;
+    private String sofaMimeType;
+    private Sofa sofaRef;
+
+//    private int[] heap;           // main heap
     private int heapStart;
     private int heapEnd;
     
-    private IntVector fixupsNeeded;  // for deserialization, the "fixups" for relative heap refs needed
-    private StringHeap stringHeapObj;
-    private LongHeap longHeapObj;
-    private ShortHeap shortHeapObj;
-    private ByteHeap byteHeapObj;
+    /** the "fixups" for relative heap refs */
+    final private List<Runnable> fixupsNeeded = new ArrayList<>();
     
-    private int stringTableOffset;
+    final private StringHeap stringHeapObj = new StringHeap();
+//    private LongHeap longHeapObj;
+//    private ShortHeap shortHeapObj;
+//    private ByteHeap byteHeapObj;
+//    
+//    private int stringTableOffset;
     
-    /**
-     * Cache sharable common values in aux heaps
-     * Values must be in aux heap, but not part of arrays there
-     *   so that rules out boolean, byte, and shorts
-     */
-    private int longZeroIndex = -1; // also used for double 0 indix
-    private int double1Index = -1;
-
     final private boolean isDelta;        // if true, a delta is being deserialized
-    final private ComprItemRefs fsStartIndexes = (CHANGE_FS_REFS_TO_SEQUENTIAL) ? new ComprItemRefs() : null;
     private String[] readCommonString;
 
-    private TypeInfo typeInfo; // type info for the current type being serialized
+//    private TypeInfo typeInfo; // type info for the current type being serialized
+
+//    private int iPrevHeap;        // 0 or heap addr of previous instance of current type
 
-    private int iPrevHeap;        // 0 or heap addr of previous instance of current type
     private boolean only1CommonString;
 
     // speedups
@@ -1515,6 +1575,45 @@ public class BinaryCasSerDes4 {
     final private DataInputStream strSeg_dis;
 
     /**
+     * For differencing when reading.
+     * Also used for arrays to difference the 0th element.
+     * 
+     * Can't use real fs for heap refs - may be forward refs not yet fixedup
+     * 
+     * Hold prev instance of FS which have FSRef slots
+     * 
+     *   for each target typecode, only set if the type 
+     *     - has 1 or more non-array fsref
+     *     - is a (subtype of) FSArray
+     *   set for both 0 and non-0 values !! Different from form6
+     * first index: key is type code
+     * 2nd index: key is slot-offset number (0-based)
+     * 
+     * Also used for array refs, for the 1st entry in the array
+     *   - feature slot 0 is used for this when reading (not when writing - could be made more uniform)
+     */
+    final private int[] [] prevFsRefsByType = new int[ts.getTypeArraySize()][];
+    private int[] prevFsRefs;
+    
+    /**
+     * Used for differencing, except for HeapRef values which
+     * use above
+     */
+    final private TOP[] prevFsByType = new TOP[ts.getTypeArraySize()];
+    private TOP prevFs;
+    /**
+     * If true, adjust type codes in serialized forms > 37 up by 2
+     */
+    private boolean isBeforeV3;  
+    
+    /**
+     * convert between FSs and "sequential" numbers
+     * Note: This may be identity map, but may not in the case for V3 where some FSs are GC'd
+     */    
+    private final Obj2IntIdentityHashMap<TOP> fs2seq = new Obj2IntIdentityHashMap<TOP>(TOP.class, TOP.singleton);
+    private final Int2ObjHashMap<TOP> seq2fs = new Int2ObjHashMap<>(TOP.class);
+
+    /**
      * Called after header was read and determined that
      * this was a compressed binary 
      * @param cas CAS
@@ -1522,16 +1621,13 @@ public class BinaryCasSerDes4 {
      * @throws IOException passthru
      */
     Deserializer(CASImpl cas, DataInput deserIn, boolean isDelta) throws IOException {
-      this.cas = cas;
+      this.baseCas = cas.getBaseCAS();
+      this.ivCas = baseCas.getInitialView();
+      this.bcsd = cas.getBinaryCasSerDes();
+      this.csds = new CommonSerDesSequential(this.baseCas);
       this.deserIn = deserIn;
       this.isDelta = isDelta;
       
-      stringHeapObj = cas.getStringHeap();
-      longHeapObj   = cas.getLongHeap();
-      shortHeapObj  = cas.getShortHeap();
-      byteHeapObj   = cas.getByteHeap();
-
-//      deserIn.readInt();    // reserved to record additional version info  // already read before calling
       final int nbrEntries = deserIn.readInt();  // number of compressed streams
       
       IntVector idxAndLen = new IntVector(nbrEntries * 3);
@@ -1565,15 +1661,19 @@ public class BinaryCasSerDes4 {
      
       control_dis = dataInputs[control_i];
       strSeg_dis = dataInputs[strSeg_i];
-
     }
     
-    private void deserialize() throws IOException {
+    private void deserialize(int version1) throws IOException {
       
+      isBeforeV3 = (version1 & 0xff00) == 0;
+      
+      fs2seq.clear();
+      seq2fs.clear();
       /************************************************
        * Setup all the input streams with inflaters
        ************************************************/
 //      long startTime1 = System.currentTimeMillis();
+      stringHeapObj.reset();
       
       /************************************************
        * Read in the common string(s)
@@ -1587,65 +1687,135 @@ public class BinaryCasSerDes4 {
       /***************************
        * Prepare to walk main heap
        ***************************/
+      csds.setup();
+      int seq = 1;
+      for (TOP fs : csds.sortedFSs) {
+        fs2seq.put(fs, seq);
+        seq2fs.put(seq++, fs);
+      }
+      
       int deltaHeapSize = readVnumber(control_dis);         
-      final Heap heapObj = cas.getHeap();
       
-      heapStart = isDelta ? heapObj.getNextId() : 0;
-      stringTableOffset = isDelta ? (stringHeapObj.getSize() - 1) : 0;
+      heapStart = isDelta ? csds.heapEnd : 0;
+
+//      stringTableOffset = isDelta ? (stringHeapObj.getSize() - 1) : 0;
       
-      if (isDelta) {
-        heapObj.grow(deltaHeapSize);
-      } else {
-        heapObj.reinitSizeOnly(deltaHeapSize);
-      } 
+//      if (isDelta) {
+//        heapObj.grow(deltaHeapSize);
+//      } else {
+//        heapObj.reinitSizeOnly(deltaHeapSize);
+//      } 
       
       heapEnd = heapStart + deltaHeapSize; 
-      heap = heapObj.heap;
+//      heap = heapObj.heap;
 
-      resetIprevious();
+      for (int[] ia : prevFsRefsByType) {
+        if (ia != null) Arrays.fill(ia,  0);
+      }
       
       if (heapStart == 0) {
         heapStart = 1;  // slot 0 not serialized, it's null / 0
       }
 
-      if (CHANGE_FS_REFS_TO_SEQUENTIAL && (heapStart > 1)) {
-        initFsStartIndexes(fsStartIndexes, heap, 1, heapStart, null);
-      }
-      fixupsNeeded = new IntVector(Math.max(16, heap.length / 10));
-
-      /***************************
-       * walk main heap
-       ***************************/
+      
+      
+//      if (CHANGE_FS_REFS_TO_SEQUENTIAL && (heapStart > 1)) {
+//        initFsStartIndexes(fsStartIndexes, heap, 1, heapStart, null);
+//      }
+//      fixupsNeeded = new IntVector(Math.max(16, heap.length / 10));
 
-      for (int iHeap = heapStart; iHeap < heapEnd; iHeap += incrToNextFs(heap, iHeap, typeInfo)) {
-        if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
-          fsStartIndexes.addItemAddr(iHeap);
-        }        
-        int tCode = heap[iHeap] = readVnumber(typeCode_dis); // get type code      
-        typeInfo = getTypeInfo(tCode);
-        iPrevHeap = typeInfo.iPrevHeap;
-
-        if (typeInfo.isHeapStoredArray) {
-          readHeapStoredArray(iHeap);
-        } else if (typeInfo.isArray) {
-          readNonHeapStoredArray(iHeap);
+      /*******************************
+       * walk main heap - deserialize
+       *******************************/
+      TypeImpl type;
+      int arraySize = 0;
+      Arrays.fill(prevFsByType, null);
+      
+      for (int iHeap = heapStart; iHeap < heapEnd; iHeap += type.getFsSpaceReq(arraySize)) {
+        
+        final int typeCode = readVnumber(typeCode_dis);
+        final int adjTypeCode = typeCode + ((isBeforeV3 && typeCode >= TypeSystemImpl.javaObjectTypeCode) ? 2 : 0);
+        type = ts.getTypeForCode(adjTypeCode);
+        
+        prevFs = prevFsByType[adjTypeCode]; // could be null;
+        prevFsRefs = getPrevFsRef(type); // null or int[], only for things having fsrefs (array or not)
+        
+        if (type.isArray()) {
+          currentFs = readArray(iHeap, type);
+          arraySize = ((CommonArray)currentFs).size();
         } else {
-          for (int i = 1; i < typeInfo.slotKinds.length + 1; i++) {
-            readByKind(iHeap, i);
+          if (!ts.annotBaseType.subsumes(type) &&  // defer subtypes of AnnotationBase
+              !(ts.sofaType == type)) {            // defer sofa types
+            currentFs = ivCas.createFS(type);
+              
+          } else {
+            currentFs = null;
+            singleFsDefer.clear();
+            sofaRef = null;
+            sofaNum = -1;
+            sofaName = null;
+            sofaMimeType = null;
+          }
+          for (FeatureImpl feat : type.getFeatureImpls()) {
+            readByKind(feat, type);
+          }
+//          for (int i = 1; i < typeInfo.slotKinds.length + 1; i++) {
+//            readByKind(iHeap, i);
+//          }
+        }
+        
+        if (currentFs == null) {
+          
+          /**
+           * Create single deferred FS
+           *   Either: Sofa (has final fields) or
+           *           Subtype of AnnotationBase - needs to be in the right view
+           *   
+           *   For the latter, handle document annotation specially
+           */
+
+          if (ts.sofaType == type) {
+            currentFs = baseCas.createSofa(sofaNum, sofaName, sofaMimeType);  
+          } else {
+            CASImpl view = (CASImpl) baseCas.getView(sofaRef);
+            if (type.getCode() == TypeSystemImpl.docTypeCode) {
+              currentFs = view.getDocumentAnnotation();  // creates the document annotation if it doesn't exist
+              // we could remove this from the indexes until deserialization is over, but then, other calls to getDocumentAnnotation
+              // would end up creating additional instances
+            } else {
+              currentFs = view.createFS(type);
+            }
+          }
+          if (type.getCode() == TypeSystemImpl.docTypeCode) { 
+            boolean wasRemoved = baseCas.checkForInvalidFeatureSetting(currentFs, baseCas.getAddbackSingle());
+            for (Runnable r : singleFsDefer) {
+              r.run();
+            }
+            baseCas.maybeAddbackSingle(wasRemoved, currentFs);
+          } else {
+            for (Runnable r : singleFsDefer) {
+              r.run();
+            }
           }
         }
         
-        typeInfo.iPrevHeap = iHeap;  // make this one the "prev" one for subsequent testing
+        assert(currentFs != null);
+//        System.out.format("Adding %,d to csds%n", iHeap);
+//        if (isDelta) {
+//          System.out.format("debug adding iHeap: %,d afterAdd: %,d%n", iHeap, iHeap + nextHeapAddrAfterMark);
+//        }
+        csds.addFS(currentFs, iHeap);
+        int s2 = 1 + fs2seq.size();  
+        fs2seq.put(currentFs, s2);  // 1 origin to match v2
+        seq2fs.put(s2, currentFs);
+        
+        prevFsByType[adjTypeCode] = currentFs;
       }
       
-      if (CHANGE_FS_REFS_TO_SEQUENTIAL) {
-        fsStartIndexes.finishSetup();
-        final int end = fixupsNeeded.size();
-        for (int i = 0; i < end; i++) {
-          final int heapAddrToFix = fixupsNeeded.get(i);
-          heap[heapAddrToFix] = fsStartIndexes.getItemAddr(heap[heapAddrToFix]);
-        }        
+      for (Runnable r : fixupsNeeded) {
+        r.run();
       }
+
       
       readIndexedFeatureStructures();
 
@@ -1657,123 +1827,202 @@ public class BinaryCasSerDes4 {
 //      System.out.format("Deserialize took %,d ms%n", System.currentTimeMillis() - startTime1);
     }
     
-    private void readNonHeapStoredArray(int iHeap) throws IOException {
-      final int length = readArrayLength(iHeap);
+    private TOP readArray(int iHeap, TypeImpl type) throws IOException { 
+      final int length = readArrayLength();
+      TOP fs = ivCas.createArray(type, length);
       if (length == 0) {
-        return;
+        return fs;
       }
-      SlotKind refKind = typeInfo.getSlotKind(2);
+      
+      SlotKind refKind = type.getComponentSlotKind();
       switch (refKind) {
-      case Slot_BooleanRef: case Slot_ByteRef:
-        heap[iHeap + 2] = readIntoByteArray(length);
-        break; 
-      case Slot_ShortRef:
-        heap[iHeap + 2] = readIntoShortArray(length);
-        break; 
-      case Slot_LongRef: case Slot_DoubleRef:
-        heap[iHeap + 2] = readIntoLongArray(refKind, length);
-        break; 
-      default:
-        throw new RuntimeException();
+      
+      case Slot_BooleanRef: {
+        boolean [] ba = ((BooleanArray)fs)._getTheArray();
+        for (int i = 0; i < length; i++) {
+          ba[i] = byte_dis.readByte() == 1;
+        }
+        break;
       }
-    }
-    
-    private int readArrayLength(int iHeap) throws IOException {
-      return heap[iHeap + 1] = readVnumber(arrayLength_dis);
-    }
+        
+      case Slot_ByteRef:
+        readIntoByteArray(((ByteArray)fs)._getTheArray()); 
+        break;
 
-    private void readHeapStoredArray(int iHeap) throws IOException {
-      final int length = readArrayLength(iHeap);
-      // output values
-      // special case 0 and 1st value
-      if (length == 0) {
-        return;
-      }
-      SlotKind arrayElementKind = typeInfo.slotKinds[1];
-      final int endi = iHeap + length + 2;
-      switch (arrayElementKind) {
-      case Slot_HeapRef: case Slot_Int: case Slot_Short:
-        {
-          int prev = (iPrevHeap == 0) ? 0 :
-                     (heap[iPrevHeap + 1] == 0) ? 0 :
-                      heap[iPrevHeap + 2]; 
-          for (int i = iHeap + 2; i < endi; i++) {
-            final int v = heap[i] = readDiff(arrayElementKind, prev);
-            prev = v;
-            if (arrayElementKind == Slot_HeapRef) {
-              fixupsNeeded.add(i);
-            }
+      case Slot_ShortRef:
+        readIntoShortArray(((ShortArray)fs)._getTheArray());
+        break; 
+      
+      case Slot_Int: {
+        final int[] ia = ((IntegerArray)fs)._getTheArray();
+        int prev = getPrevIntValue(refKind, null);
+        for (int i = 0; i < length; i++) {
+          int v = readDiff(Slot_Int, prev);
+          prev = v;
+          if (i == 0) {
+            savePrevHeapRef(type.getCode(), 1, 0, v);
           }
+          ia[i] = v;
         }
         break;
-      case Slot_Float: 
-        for (int i = iHeap + 2; i < endi; i++) {
-          heap[i] = readFloat();
-        }
+      }
+      
+      case Slot_LongRef: 
+        readIntoLongArray(((LongArray)fs)._getTheArray()); 
         break;
-      case Slot_StrRef:
-        for (int i = iHeap + 2; i < endi; i++) {
-          heap[i] = readString();
+        
+      case Slot_Float: {
+        final float[] fa = ((FloatArray)fs)._getTheArray(); 
+        for (int i = 0; i < length; i++) {
+          final int floatRef = readFloat();
+          fa[i] = Float.intBitsToFloat(floatRef);
         }
         break;
+      }
+
+      case Slot_DoubleRef:
+        readIntoDoubleArray(((DoubleArray)fs)._getTheArray()); ;
+        break;
         

[... 1675 lines stripped ...]