You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by sc...@apache.org on 2015/11/04 21:04:57 UTC
svn commit: r1712626 - in
/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima:
fs_generation/ fs_generation/impl/ internal/ internal/util/ jcas/impl/
Author: schor
Date: Wed Nov 4 20:04:56 2015
New Revision: 1712626
URL: http://svn.apache.org/viewvc?rev=1712626&view=rev
Log:
no Jira - saving some previous code for now
Added:
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/fs_generation/
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/fs_generation/impl/
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/fs_generation/impl/JCasCoverClassFactoryTest.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/ComparableIntPointerIterator.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMap.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMapTest.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntPerfTest.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoader.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoaderInjector.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapCompareTest.java
uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapTest.java
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/fs_generation/impl/JCasCoverClassFactoryTest.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/fs_generation/impl/JCasCoverClassFactoryTest.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/fs_generation/impl/JCasCoverClassFactoryTest.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/fs_generation/impl/JCasCoverClassFactoryTest.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,46 @@
+package org.apache.uima.fs_generation.impl;
+
+import java.io.ByteArrayOutputStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.uima.type_system.TypeSystem;
+import org.apache.uima.type_system.impl.TypeSystemImpl;
+import org.junit.Test;
+
+public class JCasCoverClassFactoryTest {
+
+@Test
+ public void testCreateJCasCoverClass() throws UnsupportedEncodingException {
+
+ TypeSystemImpl tsi = new TypeSystemImpl();
+
+ FeatureStructureClassGen jcf = new FeatureStructureClassGen();
+
+ byte[] br = jcf.createJCasCoverClass(tsi.getType(TypeSystem.TYPE_NAME_ANNOTATION));
+
+ // convert generated byte codes for jvm into java source via decompiling
+
+ UimaDecompiler ud = new UimaDecompiler();
+ String sr = ud.decompile(TypeSystem.TYPE_NAME_ANNOTATION, br).toString("UTF-8");
+ System.out.println(sr);
+
+
+// File file = JUnitExtension.getFile("JCasGen/typeSystemAllKinds.xml");
+// TypeSystemDescription tsDesc = UIMAFramework.getXMLParser().parseTypeSystemDescription(
+// new XMLInputSource(file));
+//
+// CAS cas = CasCreationUtils.createCas(tsDesc, null, null);
+//
+// FeatureStructureClassGen jcf = new FeatureStructureClassGen();
+//
+// byte[] r = jcf.createJCasCoverClass((TypeImpl) cas.getTypeSystem().getType("pkg.sample.name.All"));
+//
+// Path root = Paths.get("."); // should resolve to the project path
+// Path dir = root.resolve("temp/test/JCasGen");
+// dir.toFile().mkdirs();
+// Files.write(dir.resolve("testOutputAllKinds.class"), r);
+//
+// System.out.println("debug: generated byte array");
+ }
+
+}
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/ComparableIntPointerIterator.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/ComparableIntPointerIterator.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/ComparableIntPointerIterator.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/ComparableIntPointerIterator.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.internal.util;
+
+import org.apache.uima.cas.FeatureStructure;
+import org.apache.uima.cas.impl.FSIntIteratorImplBase;
+
+/**
+ * Interface for a comparable IntPointerIterator.
+ *
+ * This allows two iterators to be compared with each other. Two IntIterators are compared according to the
+ * element that would be returned by the next call to next().
+ *
+ * The purpose of this is for keeping multiple iterators, one for each subtype of a type, in a sorted order,
+ * when desiring to iterate over a type and its subtypes, in a combined merged order.
+ *
+ * This is only needed for iterators over Sorted indexes. Iterators over Bags and Sets have no ordering requirement.
+ *
+ * (Note, however, that backwards compatibility may require this)
+ *
+ * All ComparableIntPointerIterators must implement the ConcurrentModificationException mechanism, so that interface is included
+ *
+ */
+public interface ComparableIntPointerIterator<F extends FeatureStructure> extends IntPointerIterator, Comparable<FSIntIteratorImplBase<F>> {
+
+// public void checkConcurrentModification();
+//
+// public void resetConcurrentModification();
+
+}
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMap.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMap.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMap.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMap.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,517 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.internal.util;
+
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+import org.apache.uima.util.Misc;
+
+/**
+ * A map<int, int>
+ *
+ * based on JCasHashMap, but without the multi-threading support
+ *
+ * This impl is for use in a single thread case only
+ *
+ * Supports shrinking (reallocating the big table)
+ *
+ * Implements Map - like interface:
+ * keys and values are ints
+ * Entry set not (yet) impl
+ *
+ * keys must be non-0; 0 is reserved to be an empty slot
+ * values can be anything, but 0 is the value returned by get if not found so
+ * values probably should not be 0
+ *
+ * remove not supported
+ *
+ */
+public class Int2IntHashMap {
+
+ private class KeyValueIterator implements IntKeyValueIterator {
+
+ private int curPosition;
+
+ private KeyValueIterator() {
+ moveToFirst();
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#dec()
+ */
+ public void dec() {
+ if (isValid()) {
+ curPosition = moveToPreviousFilled(curPosition - 1);
+ }
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#get()
+ */
+ public int get() {
+ if (!isValid()) {
+ throw new NoSuchElementException();
+ }
+ return keys[curPosition];
+ }
+
+ public int getValue() {
+ if (!isValid()) {
+ throw new NoSuchElementException();
+ }
+ return values[curPosition];
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#inc()
+ */
+ public void inc() {
+ if (isValid()) {
+ curPosition = moveToNextFilled(curPosition + 1);
+ }
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#isValid()
+ */
+ public boolean isValid() {
+ return curPosition >= 0 && curPosition < keys.length;
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#moveToFirst()
+ */
+ public void moveToFirst() {
+ curPosition = moveToNextFilled(0);
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#moveToLast()
+ */
+ public void moveToLast() {
+ curPosition = moveToPreviousFilled(keys.length - 1);
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#copy()
+ */
+ public Object copy() {
+ KeyValueIterator it = new KeyValueIterator();
+ it.curPosition = curPosition;
+ return it;
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntPointerIterator#moveTo(int)
+ */
+ public void moveTo(int i) {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ private class KeyIterator implements IntListIterator {
+
+ private int curPosition;
+
+ private KeyIterator() {
+ this.curPosition = 0;
+ }
+
+
+ public final boolean hasNext() {
+ curPosition = moveToNextFilled(curPosition);
+ return curPosition < keys.length;
+ }
+
+ public final int next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return keys[curPosition++];
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntListIterator#hasPrevious()
+ */
+ public boolean hasPrevious() {
+ curPosition = moveToPreviousFilled(curPosition);
+ return (curPosition >= 0);
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntListIterator#previous()
+ */
+ public int previous() {
+ if (!hasPrevious()) {
+ throw new NoSuchElementException();
+ }
+ return keys[curPosition--];
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntListIterator#moveToEnd()
+ */
+ public void moveToEnd() {
+ curPosition = keys.length - 1;
+ }
+
+ /**
+ * @see org.apache.uima.internal.util.IntListIterator#moveToStart()
+ */
+ public void moveToStart() {
+ curPosition = 0;
+ }
+ }
+
+
+
+ // set to true to collect statistics for tuning
+ // you have to also put a call to showHistogram() at the end of the run
+ private static final boolean TUNE = false;
+
+ private final static int[] EMPTY_INT_ARRAY = new int[0];
+
+ static int nextHigherPowerOf2(int i) {
+ return (i < 1) ? 1 : Integer.highestOneBit(i) << ( (Integer.bitCount(i) == 1 ? 0 : 1));
+ }
+
+ // this load factor gives, for array doubling strategy:
+ // between 1.5 * 8 bytes (2 words, one for key, one for value) = 12 and
+ // 3 * 8 24 bytes per entry
+ // This compares with 20 bytes/entry for the Int2IntRBT impl
+
+ // This can be reduced to 12 to 18 bytes per entry at some complexity cost and performance
+ // cost by implementing a 1 1/2 expansion scheme (not done)
+
+ // With this tuning, the performance is approx 6 to 10 x faster than int2intRBT,
+ // for various sizes from 100 to 100,000.
+
+ // See corresponding Int2IntPerfTest which is disabled normally
+
+ private final float loadFactor = (float)0.66;
+
+ private final int initialCapacity;
+
+ private int histogram [];
+ private int maxProbe = 0;
+
+ private int sizeWhichTriggersExpansion;
+ private int size; // number of elements in the table
+
+ private int [] keys;
+ private int [] values;
+
+ private boolean secondTimeShrinkable = false;
+
+ public Int2IntHashMap() {
+ this(16);
+ }
+
+ public Int2IntHashMap(int initialCapacity) {
+ this.initialCapacity = initialCapacity;
+ newTableKeepSize(initialCapacity);
+ size = 0;
+ if (TUNE) {
+ histogram = new int[200];
+ Arrays.fill(histogram, 0);
+ maxProbe = 0;
+ }
+ }
+
+ private void newTableKeepSize(int capacity) {
+ capacity = Math.max(16, nextHigherPowerOf2(capacity));
+ keys = new int[capacity];
+ values = new int[capacity];
+ sizeWhichTriggersExpansion = (int)(capacity * loadFactor);
+ }
+
+ private void incrementSize() {
+ if (size >= sizeWhichTriggersExpansion) {
+ increaseTableCapacity();
+ }
+ size++;
+ }
+
+ private void increaseTableCapacity() {
+ final int [] oldKeys = keys;
+ final int [] oldValues = values;
+ final int oldCapacity = oldKeys.length;
+ int newCapacity = 2 * oldCapacity;
+
+ if (TUNE) {
+ System.out.println("Capacity increasing from " + oldCapacity + " to " + newCapacity);
+ }
+ newTableKeepSize(newCapacity);
+ int vi = 0;
+ for (int key : oldKeys) {
+ if (key != 0) {
+ putInner(key, oldValues[vi]);
+ }
+ vi++;
+ }
+ }
+
+ // called by clear
+ private void newTable(int capacity) {
+ newTableKeepSize(capacity);
+ size = 0;
+ resetHistogram();
+ }
+
+ private void resetHistogram() {
+ if (TUNE) {
+ histogram = new int[200];
+ Arrays.fill(histogram, 0);
+ maxProbe = 0;
+ }
+ }
+
+ public void clear() {
+ // see if size is less than the 1/2 size that triggers expansion
+ if (size < (sizeWhichTriggersExpansion >>> 1)) {
+ // if 2nd time then shrink by 50%
+ // this is done to avoid thrashing around the threshold
+ if (secondTimeShrinkable) {
+ secondTimeShrinkable = false;
+ final int newCapacity = Math.max(initialCapacity, keys.length >>> 1);
+ if (newCapacity < keys.length) {
+ newTable(newCapacity); // shrink table by 50%
+ } else { // don't shrink below minimum
+ Arrays.fill(keys, 0);
+ Arrays.fill(values, 0);
+ }
+ size = 0;
+ resetHistogram();
+ return;
+ } else {
+ secondTimeShrinkable = true;
+ }
+ } else {
+ secondTimeShrinkable = false; // reset this to require 2 triggers in a row
+ }
+ size = 0;
+ Arrays.fill(keys, 0);
+ Arrays.fill(values, 0);
+ resetHistogram();
+ }
+
+ /** It gets a ref to the current value of table, and then searches that int array.
+ *
+ * @param key -
+ * @return the probeAddr in keys array - might have a 0 value, or the key value if found
+ */
+ private int find(final int key) {
+ if (key == 0) {
+ throw new IllegalArgumentException("0 is an invalid key");
+ }
+
+ final int hash = Misc.hashInt(key);
+
+ final int[] localKeys = keys;
+ final int bitMask = localKeys.length - 1;
+ int probeAddr = hash & bitMask;
+
+ // fast paths
+ final int testKey = localKeys[probeAddr];
+ if (testKey == 0 || testKey == key) {
+ if (TUNE) {
+ updateHistogram(1);
+ }
+ return probeAddr;
+ }
+
+ return find2(localKeys, key, probeAddr);
+ }
+
+
+ private int find2(final int[] localKeys, final int key, int probeAddr) {
+ final int bitMask = localKeys.length - 1;
+ int nbrProbes = 2;
+ int probeDelta = 1;
+ probeAddr = bitMask & (probeAddr + (probeDelta++));
+
+ while (true) {
+ final int testKey = localKeys[probeAddr];
+ if (testKey == 0 || testKey == key) {
+ break;
+ }
+ nbrProbes++;
+ probeAddr = bitMask & (probeAddr + (probeDelta++));
+ }
+
+ if (TUNE) {
+ final int pv = histogram[nbrProbes];
+
+ histogram[nbrProbes] = 1 + pv;
+ if (maxProbe < nbrProbes) {
+ maxProbe = nbrProbes;
+ }
+ }
+ return probeAddr;
+ }
+
+ private void updateHistogram(int nbrProbes) {
+ histogram[nbrProbes] = 1 + histogram[nbrProbes];
+ if (maxProbe < nbrProbes) {
+ maxProbe = nbrProbes;
+ }
+ }
+
+ public int get(int key) {
+ return (key == 0) ? 0 : values[find(key)];
+ }
+
+ public boolean containsKey(int key) {
+ int probeAddr = find(key);
+ return probeAddr != 0 && keys[probeAddr] != 0;
+ }
+
+ public boolean isKeyValid(int position) {
+ return (position != 0) && (keys[position] != 0);
+ }
+
+ public int put(int key, int value) {
+ final int i = find(key);
+ final boolean keyNotFound = keys[i] == 0;
+ final int prevValue = values[i];
+ keys[i] = key;
+ values[i] = value;
+ if (keyNotFound) {
+ incrementSize();
+ }
+ return prevValue;
+ }
+
+ public void putInner(int key, int value) {
+ final int i = find(key);
+ assert(keys[i] == 0);
+ keys[i] = key;
+ values[i] = value;
+ }
+
+ public int size() {
+ return size;
+ }
+
+ /**
+ * advance pos until it points to a non 0 or is 1 past end
+ * @param pos
+ * @return updated pos
+ */
+ private int moveToNextFilled(int pos) {
+ if (pos < 0) {
+ pos = 0;
+ }
+
+ final int max = keys.length;
+ while (true) {
+ if (pos >= max) {
+ return pos;
+ }
+ if (keys[pos] != 0) {
+ return pos;
+ }
+ pos ++;
+ }
+ }
+
+ /**
+ * decrement pos until it points to a non 0 or is -1
+ * @param pos
+ * @return updated pos
+ */
+ private int moveToPreviousFilled(int pos) {
+ final int max = keys.length;
+ if (pos > max) {
+ pos = max - 1;
+ }
+
+ while (true) {
+ if (pos < 0) {
+ return pos;
+ }
+ if (keys[pos] != 0) {
+ return pos;
+ }
+ pos --;
+ }
+ }
+
+ public int[] getSortedKeys() {
+ final int size = size();
+ if (size == 0) {
+ return EMPTY_INT_ARRAY;
+ }
+ final int[] r = new int[size];
+ int i = 0;
+ for (int k : keys) {
+ if (k != 0) {
+ r[i++] = k;
+ }
+ }
+ assert(i == size());
+ Arrays.sort(r);
+ return r;
+ }
+
+ public IntListIterator keyIterator() {
+ return new KeyIterator();
+ }
+
+ public IntListIterator keyIterator(int aKey) {
+ throw new UnsupportedOperationException();// only makes sense for sorted things
+ }
+
+ public IntKeyValueIterator keyValueIterator() {
+ return new KeyValueIterator();
+ }
+
+ public IntKeyValueIterator keyValueIterator(int aKey) {
+ throw new UnsupportedOperationException();// only makes sense for sorted things
+ }
+
+ public void showHistogram() {
+ if (TUNE) {
+ int sumI = 0;
+ for (int i : histogram) {
+ sumI += i;
+ }
+
+ System.out.format(
+ "Histogram of number of probes, loadfactor = %.1f, maxProbe=%,d nbr of find operations at last table size=%,d%n",
+ loadFactor, maxProbe, sumI);
+ for (int i = 0; i <= maxProbe; i++) {
+ if (i == maxProbe && histogram[i] == 0) {
+ System.out.println("huh?");
+ }
+ System.out.println(i + ": " + histogram[i]);
+ }
+
+ System.out.println("bytes / entry = " + (float) (keys.length) * 8 / size());
+ System.out.format("size = %,d, prevExpansionTriggerSize = %,d, next = %,d%n",
+ size(),
+ (int) ((keys.length >>> 1) * loadFactor),
+ (int) (keys.length * loadFactor));
+ }
+ }
+
+}
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMapTest.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMapTest.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMapTest.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntHashMapTest.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.internal.util;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+public class Int2IntHashMapTest extends TestCase {
+
+ public void testIterator() {
+ Int2IntHashMap ia = new Int2IntHashMap();
+ Integer[] vs = new Integer[] {2, 2, 5, 1, 6, 7, 3, 4};
+ for (Integer i : vs) {
+ ia.put(i, i * 2);
+ }
+ Integer[] r = new Integer[vs.length];
+ int i = 0;
+ IntListIterator itl = ia.keyIterator();
+
+ while(itl.hasNext()){
+ r[i++] = itl.next();
+ }
+ assertEquals(i, vs.length - 1);
+ assertTrue(Arrays.equals(r, new Integer[] {3, 2, 1, 4, 7, 6, 5, null}));
+
+ i = 0;
+ for (IntKeyValueIterator it = ia.keyValueIterator(); it.isValid(); it.inc()) {
+ r[i++] = it.getValue();
+// System.out.format("key: %d value: %d%n", it.get(), it.getValue());
+ }
+ assertTrue(Arrays.equals(r, new Integer[] {6, 4, 2, 8, 14, 12, 10, null} ));
+
+ i = 0;
+
+ IntKeyValueIterator it = ia.keyValueIterator();
+ assertTrue(it.isValid());
+ it.dec();
+ assertFalse(it.isValid());
+ it.inc();
+ assertFalse(it.isValid());
+ it.moveToLast();
+ assertTrue(it.isValid());
+ it.inc();
+ assertFalse(it.isValid());
+// it.dec(); // causes infinite loop
+// assertFalse(it.isValid());
+
+ }
+
+ public void testFastLookup() {
+ Int2IntHashMap ia = new Int2IntHashMap();
+ Random r = new Random();
+ Set<Integer> keys = new HashSet<Integer>(1000);
+
+ for (int i = 0; i < 1000; i++) {
+ int k = r.nextInt(1000);
+ while (k == 0) {k = r.nextInt(1000);}
+ keys.add(k);
+ ia.put(k, 10000+k);
+ }
+
+ for (int k : keys) {
+ assertEquals(10000 + k, ia.get(k));
+ }
+ }
+}
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntPerfTest.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntPerfTest.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntPerfTest.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/Int2IntPerfTest.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.internal.util;
+
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.uima.internal.util.rb_trees.Int2IntRBT;
+
+public class Int2IntPerfTest extends TestCase {
+ /**
+ * Set to false to run the performance test
+ */
+ final boolean SKIP = true;
+
+
+ Random r = new Random();
+// Set<Integer> keys = new HashSet<Integer>(1000);
+
+ int dmv = 0;
+
+// Int2IntHashMap m2;
+ Int2IntRBT m1;
+
+ final int[] keys10000 = new int[111111];
+ int k10ki = 0;
+
+
+ public void testPerf() {
+ if (SKIP) return;
+ m1 = new Int2IntRBT(16);
+// m2 = new Int2IntHashMap(16);
+
+ for (int i = 0; i < 111111; i++) {
+ int k = r.nextInt(100000);
+ while (k == 0) { k = r.nextInt(100000);}
+ keys10000[i] = k;
+ }
+
+ System.out.format("%n%n W A R M U P %n%n");
+ warmup(m1);
+// warmup(m2);
+
+ System.out.format("%n%n Time 100 %n%n");
+ timelp(100);
+ System.out.format("%n%n Time 1000 %n%n");
+ timelp(1000);
+ System.out.format("%n%n Time 10000 %n%n");
+ timelp(10000);
+ System.out.format("%n%n Time 100000 %n%n");
+ timelp(100000);
+
+ System.out.println(dmv);
+ }
+ private void time2(int n) {
+ float f1 = time(m1, n);
+// float f2 = time(m2, n);
+// System.out.format(" ratio = %.3f%n", f1/f2);
+ }
+
+ private void timelp(int n) {
+ time2(n);
+ time2(n);
+ time2(n);
+ }
+
+ private void warmup(Object m) {
+ for (int i = 0; i < 500; i++) {
+ inner(m,true, 1000) ; // warm up
+ }
+ }
+
+ private float time(Object m, int ss) {
+ long start = System.nanoTime();
+ for (int i = 0; i < 500; i++) {
+ inner(m,false, ss);
+ }
+ float t = (System.nanoTime() - start) / 1000000.0F;
+ System.out.format("time for %,d: %s is %.3f milliseconds %n", ss,
+ m.getClass().getSimpleName(),
+ t);
+ return t;
+ }
+
+ private int nextKey() {
+ int r = keys10000[k10ki++];
+ if (k10ki >= keys10000.length) {
+ k10ki = 0;
+ }
+ return r;
+ }
+
+ private void inner(Object m, boolean check, int ss) {
+ for (int i = 0; i < ss; i++) {
+
+ int k = nextKey();
+// System.out.print(" " + k);
+// if (i %100 == 0) System.out.println("");
+// keys.add(k);
+// if (m instanceof Int2IntHashMap) {
+// ((Int2IntHashMap)m).put(k, 10000+k);
+// } else {
+ ((Int2IntRBT)m).put(k, 10000+k);
+// }
+ if (check) {
+ assertEquals(10000 + k,
+ ((Int2IntRBT)m).getMostlyClose(k));
+ }
+ }
+ for (int i = 0; i < ss; i++) {
+ dmv +=
+// (m instanceof Int2IntHashMap) ?
+// ((Int2IntHashMap)m).get(keys10000[i]) :
+ ((Int2IntRBT)m).getMostlyClose(keys10000[i]);
+ }
+
+// if (m instanceof Int2IntHashMap) {
+// ((Int2IntHashMap)m).showHistogram();
+//// System.out.println(" # of unique Keys: " + keys.size());
+// ((Int2IntHashMap)m).clear();
+// } else {
+ ((Int2IntRBT)m).clear();
+// }
+// keys.clear();
+
+// for (int k : keys) {
+// assertEquals(10000 + k, (m instanceof Int2IntHashMap) ?
+// ((Int2IntHashMap)m).get(k) :
+// ((Int2IntRBT)m).getMostlyClose(k));
+// }
+
+ }
+
+
+}
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoader.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoader.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoader.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoader.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.internal.util;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.security.AccessControlContext;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.ProtectionDomain;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.apache.uima.cas.impl.TypeImpl;
+import org.apache.uima.cas.impl.TypeSystemImpl;
+import org.apache.uima.util.Misc;
+
+/**
+ * UIMATypeSystemClassLoader is used only to load generated JCas classes, lazily.
+ * Multiple instances of these support having multiple type systems in use in one JVM; it is up to the
+ * application setup code to create these class loaders.
+ *
+ * For simple cases, without multiple type systems, if you don't care about lazy instantiation, you can
+ * have type system commit do a batch generation and load into a specific arbitrary class loader;
+ * @see {@link UIMATypeSystemClassLoaderInjector}.
+ *
+ * For Application, External Resource, and Annotator code to have references to these JCas classes,
+ * that code must be loaded under this ClassLoader (or a child of it).
+ *
+ * This loader has no class path; it is only used for lazy generation and loading of JCas classes
+ *
+ * Because the _Type class has static references to the main JCas class, the main JCas class is loaded first.
+ *
+ * This loader has a reference to the committed type system, that starts out as null. Type system commit
+ * sets it. If it is already set, a second set is an error unless the type systems are the same.
+ *
+ * This loader does nothing unless the type system ref is set. Then it looks up the type being loaded, and if it matches
+ * a JCas type, it generates and loads the main and _Type classes. Otherwise, it delegates to its parent.
+ *
+ */
+public class UIMATypeSystemClassLoader extends ClassLoader {
+
+ static {if (!ClassLoader.registerAsParallelCapable()) {
+ System.err.println("WARNING - Failed to register the UIMA Class loader as parallel-capable - should never happen");
+ }
+ }
+
+ /**
+ * locks for loading more than 1 class at a time (on different threads)
+ * no more than the total number of cores, rounded up to pwr of 2
+ */
+ final private static int nbrLocks = Misc.nextHigherPowerOf2(Runtime.getRuntime().availableProcessors());
+ // not static
+ final private Object[] syncLocks = new Object[nbrLocks];
+ { for (int i = 0; i < nbrLocks; i++) {
+ syncLocks[i] = new Object();
+ }
+ }
+
+ /**
+ * JCas classes (including xxxx_Type) are provided (for built-ins) or generated (from merged type system information)
+ * The generated classes are loaded under an instance of this class loader,
+ * which is also then the loader (or the parent class loader) used for loading the UIMA pipeline's classes,
+ * including the application, external resources, annotators, and customized JCasGen'd classes (if any).
+ */
+ private TypeSystemImpl tsi = null; // The corresponding TypeSystemImpl or null
+
+ /**
+ * Creates a new UIMATypeSystemClassLoader
+ */
+ public UIMATypeSystemClassLoader() {
+ super();
+ }
+
+ /**
+ * Creates a new UIMAClassLoader, for a particular parent ClassLoader
+ *
+ * @param parent
+ * specify the parent of the classloader
+ */
+ public UIMATypeSystemClassLoader(ClassLoader parent) {
+ super(parent);
+ }
+
+ /*
+ * Try to generate and load the JCas class before delegate the class loading to its parent
+ * String is like x.y.Foo
+ */
+ protected Class<?> loadClass(String name, boolean resolve)
+ throws ClassNotFoundException {
+
+ if (tsi == null) {
+ return super.loadClass(name, resolve);
+ }
+
+ String rootname = (name.endsWith("_Type")) ? name.substring(0, name.length() - 5) : name;
+ if (!tsi.isJCasGenerateOnLoad(rootname)) {
+ return super.loadClass(name, resolve);
+ }
+
+ // requirement: ensure that the protected defineClass() method is called only once for each class loader and class name pair.
+ // pick a random syncLock to synchronize
+ // Although the sync locks are not one/per/class, there should be enough of them to make the likelyhood
+ // of needing to wait very low (unless it's the same class-name being loaded, of course).
+ synchronized (syncLocks[name.hashCode() & (nbrLocks - 1)]) {
+ // First, check if the class has already been loaded
+ Class<?> c = findLoadedClass(name);
+ if (c == null) {
+ // package must exist before defineClass is called
+ final int i = name.lastIndexOf('.');
+ final String packageName = (i == -1) ? null : name.substring(0, i);
+ if (packageName != null && getPackage(packageName) == null) {
+ definePackage(packageName, null, null, null, null, null, null, null);
+ }
+
+ byte[] b = tsi.jcasGenerate(rootname); // has no static refs to _Type
+ c = defineClass(rootname, b, 0, b.length);
+
+ b = tsi.jcas_TypeGenerate(rootname);
+ resolveClass(defineClass(rootname + "_Type", b, 0, b.length)); // always resolve _Type classes, static ref by rootclass
+ }
+ if (resolve) {
+ resolveClass(c);
+ }
+ return c;
+ }
+ }
+
+ public void setTsi(TypeSystemImpl tsi) {
+ this.tsi = tsi;
+ }
+
+
+}
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoaderInjector.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoaderInjector.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoaderInjector.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/internal/util/UIMATypeSystemClassLoaderInjector.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.internal.util;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodHandles.Lookup;
+import java.net.URL;
+import java.util.function.Function;
+
+import org.apache.uima.cas.impl.TypeSystemImpl;
+import org.apache.uima.util.Misc;
+
+/**
+ * UIMATypeSystemClassLoaderInjector supports injected generated class definitions into an existing class loader.
+ * This may be used when multiple type systems are not in use. Otherwise, @see {@link UIMATypeSystemClassLoader}
+ *
+ * Redefinition of existing types is supported but may be ineffective if other classes have become "linked" to
+ * classes and methods being defined.
+ *
+ * This is intended to be use in a batch mode, as opposed to using the UIMATypeSystemClassLoader, which permits
+ * lazy generating and loading of JCas Classes.
+ *
+ * To use, first create an instance of this class, specifying the type system and the class loader to inject these
+ * classes into.
+ *
+ * Then call generateAndLoadJCasClass for all types needing to be generated.
+ */
+public class UIMATypeSystemClassLoaderInjector {
+
+ private static final Class<String> sc = String.class;
+ private static final Class<ClassLoader> clc = ClassLoader.class;
+
+ /**
+ * JCas classes (including xxxx_Type) are provided (for built-ins) or generated (from merged type system information)
+ * The generated classes are loaded under an instance of this class loader,
+ * which is also then the loader (or the parent class loader) used for loading the UIMA pipeline's classes,
+ * including the application, external resources, annotators, and customized JCasGen'd classes (if any).
+ */
+ final private TypeSystemImpl tsi;
+
+ final private ClassLoader cl;
+
+ /** use these xxx.invokeExact(args) **/
+ MethodHandle mhFindLoadedClass;
+ MethodHandle mhGetPackage;
+ MethodHandle mhDefinePackage;
+ MethodHandle mhDefineClass;
+ MethodHandle mhResolveClass;
+
+ /**
+ * Creates a new UIMAClassLoader based on a classpath URL's
+ *
+ * @param classpath
+ * an array of wellformed classpath URL's
+ */
+ public UIMATypeSystemClassLoaderInjector(ClassLoader cl, TypeSystemImpl tsi) {
+ this.cl = cl;
+ this.tsi = tsi;
+
+ Lookup methodHandleAccessContext = MethodHandles.lookup();
+
+ mhFindLoadedClass = Misc.getProtectedMethodHandle(clc, methodHandleAccessContext, "findLoadedClass", sc);
+ mhGetPackage = Misc.getProtectedMethodHandle(clc, methodHandleAccessContext, "getPackage", sc);
+ mhDefinePackage = Misc.getProtectedMethodHandle(clc, methodHandleAccessContext, "definePackage", sc, sc, sc, sc, sc, sc, sc, URL.class);
+ mhDefineClass = Misc.getProtectedMethodHandle(clc, methodHandleAccessContext, "defineClass", sc, byte[].class, int.class, int.class);
+ mhResolveClass = Misc.getProtectedMethodHandle(clc, methodHandleAccessContext, "resolveClass", Class.class);
+ }
+
+ /**
+ * Generate a JCas cover class and its _Type variant, and load it using the class loader specified in the constructor
+ * @param rootname
+ * @return the loaded and resolved class
+ */
+ public Class<?> generateAndLoadClass(String rootname) {
+ try {
+ // redefining supported TODO check if this works
+// Class<?> c = (Class<?>) mhFindLoadedClass.invokeExact(rootname);
+//
+// if (c != null) {
+// return c; // assume resolved
+// }
+
+ final int i = rootname.lastIndexOf('.');
+ final String packageName = (i == -1) ? null : rootname.substring(0, i);
+
+ // package must exist before defineClass is called
+ if (mhGetPackage.invokeExact(packageName) == null) {
+ mhDefinePackage.invokeExact(packageName, null, null, null, null, null, null, null);
+ }
+ byte[] b = tsi.jcasGenerate(rootname); // has no static refs to _Type
+ Class<?> c = (Class<?>) mhDefineClass.invokeExact(rootname, b, 0, b.length);
+
+ b = tsi.jcas_TypeGenerate(rootname);
+ mhResolveClass.invokeExact(mhDefineClass.invokeExact(rootname + "_Type", b, 0, b.length)); // always resolve _Type classes, static ref by rootclass
+ mhResolveClass.invokeExact(c);
+
+ return c;
+ } catch (Throwable e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
\ No newline at end of file
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapCompareTest.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapCompareTest.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapCompareTest.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapCompareTest.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.uima.jcas.impl;
+
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.uima.cas.impl.FeatureStructureImpl;
+import org.apache.uima.internal.util.MultiThreadUtils;
+import org.apache.uima.jcas.cas.TOP;
+import org.apache.uima.jcas.cas.TOP_Type;
+import org.apache.uima.util.Misc;
+
+import junit.framework.TestCase;
+
+/**
+ * Run this as a single test with yourkit, and look at the retained storage for both maps.
+ *
+ * Java 8 test: Concurrent Hash Map impl showed ~2.25 MB * 8 (concurrency level)
+ * JCasHashmap showed ~0.835 MB * 8
+ *
+ */
+public class JCasHashMapCompareTest extends TestCase {
+
+ private static class FakeTopType extends TOP_Type {
+ public FakeTopType() {
+ super();
+ }
+ }
+
+ private static final long rm = 0x5deece66dL;
+
+ private static int sizeOfTest = 1024 * 8;
+// private static final int SIZEm1 = SIZE - 1;
+ private static final TOP_Type FAKE_TOP_TYPE_INSTANCE = new FakeTopType();
+// private JCasHashMap jhm;
+ private ConcurrentMap<Integer, FeatureStructureImpl> concurrentMap;
+
+
+ public void testComp() throws Exception {
+ Thread.sleep(0000); // set non-zero to delay so you can get yourkit tooling hooked up, if using yourkit
+ int numberOfThreads = Misc.numberOfCores;
+ numberOfThreads = Math.min(8, Misc.nextHigherPowerOf2(numberOfThreads)); // avoid too big slowdown on giant machines.
+ System.out.format("test JCasHashMapComp with %d threads%n", numberOfThreads);
+ runCustom(numberOfThreads);
+ runConCur(numberOfThreads);
+ runCustom(numberOfThreads*2);
+ runConCur(numberOfThreads*2);
+ runCustom(numberOfThreads*4);
+ runConCur(numberOfThreads*4);
+// stats("custom", runCustom(numberOfThreads)); // not accurate, use yourkit retained size instead
+// stats("concur", runConCur(numberOfThreads));
+ Set<Integer> ints = new HashSet<Integer>();
+ for (Entry<Integer, FeatureStructureImpl> e : concurrentMap.entrySet()) {
+ assertFalse(ints.contains(Integer.valueOf(e.getKey())));
+ assertEquals(e.getValue().getAddress(), (int)(e.getKey()));
+ ints.add(e.getKey());
+ }
+
+// System.out.println("Found " + i);
+
+ // launch yourkit profiler and look at retained sizes for both
+// Thread.sleep(1000000);
+ }
+
+ private int runConCur(int numberOfThreads) throws Exception {
+ final ConcurrentMap<Integer, FeatureStructureImpl> m =
+ new ConcurrentHashMap<Integer, FeatureStructureImpl>(200, 0.75F, numberOfThreads);
+ concurrentMap = m;
+
+ final int numberOfWaiters = numberOfThreads*2;
+ final Object[] waiters = new Object[numberOfWaiters];
+ for (int i = 0; i < numberOfWaiters; i++) {
+ waiters[i] = new Object();
+ }
+ MultiThreadUtils.Run2isb run2isb= new MultiThreadUtils.Run2isb() {
+
+ public void call(int threadNumber, int repeatNumber, StringBuilder sb) {
+// int founds = 0, puts = 0;
+ for (int i = 0; i < sizeOfTest*threadNumber; i++) {
+ final int key = hash(i, threadNumber) / 2;
+ final Object waiter = waiters[key & (numberOfWaiters - 1)];
+ FeatureStructureImpl fs = m.putIfAbsent(key, new TOP(key, JCasHashMapSubMap.RESERVE_TOP_TYPE_INSTANCE));
+ while (fs != null && ((TOP)fs).jcasType == JCasHashMapSubMap.RESERVE_TOP_TYPE_INSTANCE) {
+ // someone else reserved this
+
+ // wait for notify
+ synchronized (waiter) {
+ fs = m.get(key);
+ if (((TOP)fs).jcasType == JCasHashMapSubMap.RESERVE_TOP_TYPE_INSTANCE) {
+ try {
+ waiter.wait();
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+ }
+
+// FeatureStructureImpl fs = m.get(key);
+ if (null == fs) {
+// puts ++;
+ FeatureStructureImpl prev = m.put(key, new TOP(key, FAKE_TOP_TYPE_INSTANCE));
+ if (((TOP)prev).jcasType == JCasHashMapSubMap.RESERVE_TOP_TYPE_INSTANCE) {
+ synchronized (waiter) {
+ waiter.notifyAll();
+ }
+ }
+// puts --; // someone beat us
+// founds ++;
+ }
+
+ }
+// System.out.println("concur Puts = " + puts + ", founds = " + founds);
+ }
+ };
+ long start = System.currentTimeMillis();
+ MultiThreadUtils.tstMultiThread("JCasHashMapTestCompConcur", numberOfThreads, 10, run2isb,
+ new Runnable() {
+ public void run() {
+ m.clear();
+ }});
+ System.out.format("JCasCompTest - using ConcurrentHashMap, threads = %d, time = %,f seconds%n", numberOfThreads, (System.currentTimeMillis() - start) / 1000.f);
+ return m.size();
+ }
+
+ private int runCustom(int numberOfThreads) throws Exception {
+ final JCasHashMap m = new JCasHashMap(256, true); // true = do use cache
+
+ MultiThreadUtils.Run2isb run2isb= new MultiThreadUtils.Run2isb() {
+
+ public void call(int threadNumber, int repeatNumber, StringBuilder sb) {
+// int founds = 0, puts = 0;
+ for (int i = 0; i < sizeOfTest*threadNumber; i++) {
+ final int key = hash(i, threadNumber
+ ) / 2;
+// if (key == 456551)
+// System.out.println("debug");
+ FeatureStructureImpl fs = m.getReserve(key);
+
+ if (null == fs) {
+// puts++;
+ m.put(new TOP(key, FAKE_TOP_TYPE_INSTANCE));
+ } else {
+// founds ++;
+ }
+ }
+// System.out.println("custom Puts = " + puts + ", founds = " + founds);
+ }
+ };
+ long start = System.currentTimeMillis();
+ MultiThreadUtils.tstMultiThread("JCasHashMapTestComp0", numberOfThreads, 10, run2isb,
+ new Runnable() {
+ public void run() {
+ m.clear();
+ }});
+ System.out.format("JCasCompTest - using JCasHashMap, threads = %d, time = %,f seconds%n", numberOfThreads, (System.currentTimeMillis() - start) / 1000.f);
+ m.showHistogram();
+ return m.getApproximateSize();
+ }
+
+ // not accurate, use yourkit retained size instead
+// private void stats(String m, int size) {
+// for (int i = 0; i < 2; i++) {
+// System.gc();
+// }
+// Runtime r = Runtime.getRuntime();
+// long free =r.freeMemory();
+// long total = r.totalMemory();
+// System.out.format("JCasHashMapComp %s used = %,d size = %,d%n",
+// m, total - free, size);
+// }
+
+ private int hash(int i, int threadNumber) {
+ return (int)(((
+ (i + (threadNumber << 4)) * rm + 11 +
+ (threadNumber << 1))
+ >>> 16) & (sizeOfTest*threadNumber - 1));
+ }
+}
Added: uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapTest.java
URL: http://svn.apache.org/viewvc/uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapTest.java?rev=1712626&view=auto
==============================================================================
--- uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapTest.java (added)
+++ uima/uimaj/branches/experiment-v3-jcas/unused-saved/src/org/apache/uima/jcas/impl/JCasHashMapTest.java Wed Nov 4 20:04:56 2015
@@ -0,0 +1,468 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.uima.jcas.impl;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.uima.cas.impl.FeatureStructureImpl;
+import org.apache.uima.internal.util.MultiThreadUtils;
+import org.apache.uima.jcas.cas.TOP;
+import org.apache.uima.jcas.cas.TOP_Type;
+import org.apache.uima.util.Misc;
+
+import junit.framework.TestCase;
+
+public class JCasHashMapTest extends TestCase {
+ static private class FakeTopType extends TOP_Type {
+ public FakeTopType() {
+ super();
+ }
+ }
+
+ static final TOP_Type FAKE_TOP_TYPE_INSTANCE = new FakeTopType();
+ static final int SIZE = 20000; // set > 2 million for cache avoidance timing tests
+ static final long SEED = 12345;
+ static Random r = new Random(SEED);
+ static private int[] addrs = new int[SIZE];
+ static int prev = 0;
+
+ static {
+ // unique numbers
+ for (int i = 0; i < SIZE; i++) {
+ addrs[i] = prev = prev + r.nextInt(14) + 1;
+ }
+ // shuffled
+ for (int i = SIZE - 1; i >= 1; i--) {
+ int ir = r.nextInt(i+1);
+ int temp = addrs[i];
+ addrs[i] = addrs[ir];
+ addrs[ir] = temp;
+ }
+ }
+
+ public void testBasic() {
+ JCasHashMap m;
+
+ for (int i = 1; i <= 128; i *= 2) {
+ JCasHashMap.setDEFAULT_CONCURRENCY_LEVEL(i);
+ // test default concurrency level adjusted down
+ m = new JCasHashMap(32 * i, true);
+ assertEquals( i, m.getConcurrencyLevel());
+ m = new JCasHashMap(16 * i, true);
+ assertEquals(Math.max(1, i / 2), m.getConcurrencyLevel());
+
+ //test capacity adjusted up
+ m = new JCasHashMap(32 * i, true, i);
+ assertEquals( 32 * i, m.getCapacity());
+ m = new JCasHashMap(31 * i, true, i);
+ assertEquals( 32 * i, m.getCapacity());
+ m = new JCasHashMap(16 * i, true, i);
+ assertEquals( 32 * i, m.getCapacity());
+ }
+ }
+
+ public void testWithPerf() {
+
+ for (int i = 0; i < 5; i++ ) {
+ arun(SIZE);
+ }
+
+ arunCk(SIZE);
+
+// for (int i = 0; i < 50; i++ ) {
+// arun2(2000000);
+// }
+
+ }
+
+ public void testMultiThread() throws Exception {
+ final Random random = new Random();
+ int numberOfThreads = Misc.numberOfCores;
+ System.out.format("test JCasHashMap with up to %d threads%n", numberOfThreads);
+
+
+ for (int th = 2; th <= numberOfThreads; th *=2) {
+ JCasHashMap.setDEFAULT_CONCURRENCY_LEVEL(th);
+ final JCasHashMap m = new JCasHashMap(200, true); // true = do use cache
+ MultiThreadUtils.Run2isb run2isb = new MultiThreadUtils.Run2isb() {
+
+ public void call(int threadNumber, int repeatNumber, StringBuilder sb) {
+ for (int k = 0; k < 4; k++) {
+ for (int i = 0; i < SIZE / 4; i++) {
+ final int key = addrs[random.nextInt(SIZE / 16)];
+ FeatureStructureImpl fs = m.getReserve(key);
+ if (null == fs) {
+ m.put(new TOP(key, FAKE_TOP_TYPE_INSTANCE));
+ }
+ }
+ try {
+ Thread.sleep(0, random.nextInt(1000));
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ // System.out.println(sb.toString());
+ }
+ };
+ MultiThreadUtils.tstMultiThread("JCasHashMapTest", numberOfThreads, 10, run2isb,
+ new Runnable() {
+ public void run() {
+ m.clear();
+ }});
+ }
+ }
+
+ public void testMultiThreadCompare() throws Exception {
+ final Random random = new Random();
+ int numberOfThreads = Misc.numberOfCores;
+ System.out.format("test JCasHashMap with compare with up to %d threads%n", numberOfThreads);
+
+ final ConcurrentMap<Integer, FeatureStructureImpl> check =
+ new ConcurrentHashMap<Integer, FeatureStructureImpl>(SIZE, .5F, numberOfThreads * 2);
+
+ for (int th = 2; th <= numberOfThreads; th *= 2) {
+ JCasHashMap.setDEFAULT_CONCURRENCY_LEVEL(th);
+ final JCasHashMap m = new JCasHashMap(200, true); // true = do use cache
+
+ MultiThreadUtils.Run2isb run2isb = new MultiThreadUtils.Run2isb() {
+
+ public void call(int threadNumber, int repeatNumber, StringBuilder sb) {
+ for (int k = 0; k < 4; k++) {
+ for (int i = 0; i < SIZE / 4; i++) {
+ final int key = addrs[random.nextInt(SIZE / 16)];
+ FeatureStructureImpl fs = m.getReserve(key);
+ if (null == fs) {
+ fs = new TOP(key, FAKE_TOP_TYPE_INSTANCE);
+ check.put(key, fs);
+ m.put(fs);
+ } else {
+ FeatureStructureImpl fscheck = check.get(key);
+ if (fscheck == null || fscheck != fs) {
+ String msg = String.format("JCasHashMapTest miscompare, repeat=%,d, count=%,d key=%,d"
+ + ", checkKey=%s JCasHashMapKey=%,d",
+ k, i, key, (null == fscheck) ? "null" : Integer.toString(fscheck.getAddress()), fs.getAddress());
+ System.err.println(msg);
+ throw new RuntimeException(msg);
+ }
+ }
+ }
+ try {
+ Thread.sleep(0, random.nextInt(1000));
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ // System.out.println(sb.toString());
+ }
+ };
+ MultiThreadUtils.tstMultiThread("JCasHashMapTest", numberOfThreads, 10, run2isb,
+ new Runnable() {
+ public void run() {
+ check.clear();
+ m.clear();
+ }
+ });
+ }
+ }
+ /**
+ * Create situation
+ * make a set of indexed fs instances, no JCas
+ * on multiple threads, simultaneously, attempt to get the jcas cover object for this
+ * one getReserve should succeed, but reserve, and the others should "wait".
+ * then put
+ * then the others should "wakeup" and return the same instance
+ *
+ * @throws Exception
+ */
+ public void testMultiThreadCollide() throws Exception {
+ int numberOfThreads = Misc.numberOfCores;
+ if (numberOfThreads < 2) {
+ return;
+ }
+ System.out.format("test JCasHashMap collide with up to %d threads%n", numberOfThreads);
+
+ Thread thisThread = Thread.currentThread();
+ final int subThreadPriority = thisThread.getPriority();
+ thisThread.setPriority(subThreadPriority - 1);
+ final MultiThreadUtils.ThreadM[] threads = new MultiThreadUtils.ThreadM[numberOfThreads];
+ final JCasHashMap m = new JCasHashMap(200, true); // true = do use cache
+ final Random r = new Random(); // used to sleep from 0 to 4 milliseconds
+ final int hashKey = 15;
+ final TOP fs = new TOP(hashKey, FAKE_TOP_TYPE_INSTANCE);
+ final FeatureStructureImpl[] found = new FeatureStructureImpl[numberOfThreads];
+
+ for (int i = 0; i < numberOfThreads; i++) {
+ final int finalI = i;
+ threads[i] = new MultiThreadUtils.ThreadM() {
+ public void run() {
+ while (true) {
+ if (!MultiThreadUtils.wait4go(this)) {
+ break;
+ }
+ MultiThreadUtils.sleep(r.nextInt(500000)); // 0-500 microseconds
+ found[finalI] = m.getReserve(hashKey);
+ }
+ }
+ };
+ threads[i].setPriority(subThreadPriority);
+ threads[i].start();
+ }
+
+ for (int loopCount = 0; loopCount < 10; loopCount ++) {
+ System.out.println(" JCasHashMap collide loop count is " + loopCount);
+
+ // create threads and start them
+ for (int th = 2; th <= numberOfThreads; th *= 2) {
+ JCasHashMap.setDEFAULT_CONCURRENCY_LEVEL(th);
+ Arrays.fill(found, null);
+ m.clear();
+
+ MultiThreadUtils.kickOffThreads(threads);
+
+ Thread.sleep(20);
+ // verify that one thread finished, others are waiting, because of the reserve.
+ // this assumes that all the threads got to run.
+ int numberWaiting = 0;
+ int threadFinished = -1;
+ for (int i = 0; i < numberOfThreads; i++) {
+ if (threads[i].state == MultiThreadUtils.THREAD_RUNNING) {
+ numberWaiting ++;
+ } else {
+ threadFinished = i;
+ }
+ }
+
+ assertEquals(numberOfThreads - 1, numberWaiting); // expected 7 but was 8
+ m.put(fs);
+ found[threadFinished] = fs;
+
+ MultiThreadUtils.waitForAllReady(threads);
+
+// // loop a few times to give enough time for the other threads to finish.
+// long startOfWait = System.currentTimeMillis();
+// while (System.currentTimeMillis() - startOfWait < 30000) { // wait up to 30 seconds in case of machine stall
+//
+// // Attempt to insure we let the threads under test run in preference to this one
+// Thread.sleep(20); // imprecise. Intent is to allow other thread that was waiting, to run
+// // before this thread resumes. Depends on thread priorities, but
+// // multiple threads could be running at the same time.
+//
+// numberWaiting = 0;
+// for (int i = 0; i < numberOfThreads; i++) {
+// if (threads[i].state == MultiThreadUtils.THREAD_RUNNING) {
+// numberWaiting ++;
+// }
+// }
+// if (numberWaiting == 0) {
+// break;
+// }
+// }
+
+// assertEquals(0, numberWaiting); // if not 0 by now, something is likely wrong, or machine stalled more than 30 seconds
+ // System.out.format("JCasHashMapTest collide, found = %s%n", intList(found));
+ for (FeatureStructureImpl f : found) {
+ if (f != fs) {
+ System.err.format("JCasHashMapTest miscompare fs = %s, f = %s%n", fs, (f == null) ? "null" : f);
+ }
+ assertTrue(f == fs);
+ }
+ }
+ }
+
+ MultiThreadUtils.terminateThreads(threads);
+ }
+
+
+
+// private void arun2(int n) {
+// JCasHashMap2 m = new JCasHashMap2(200, true);
+// assertTrue(m.size() == 0);
+// assertTrue(m.getbitsMask() == 0x000000ff);
+//
+// JCas jcas = null;
+//
+// long start = System.currentTimeMillis();
+// for (int i = 0; i < n; i++) {
+// TOP fs = new TOP(7 * i, NULL_TOP_TYPE_INSTANCE);
+// FeatureStructureImpl v = m.get(fs.getAddress());
+// if (null == v) {
+// m.putAtLastProbeAddr(fs);
+// }
+// }
+// System.out.format("time for v2 %,d is %,d ms%n",
+// n, System.currentTimeMillis() - start);
+// m.showHistogram();
+//
+// }
+
+ private void arun(int n) {
+ JCasHashMap m = new JCasHashMap(200, true); // true = do use cache
+ assertTrue(m.getApproximateSize() == 0);
+
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < n; i++) {
+ final int key = addrs[i];
+ TOP fs = new TOP(key, FAKE_TOP_TYPE_INSTANCE);
+// FeatureStructureImpl v = m.get(fs.getAddress());
+// if (null == v) {
+// m.get(7 * i);
+ m.put(fs);
+// }
+ }
+
+ assertEquals(m.getApproximateSize(), n);
+
+ System.out.format("time for v1 %,d is %,d ms%n",
+ n, System.currentTimeMillis() - start);
+ m.showHistogram();
+
+ }
+
+ private void arunCk(int n) {
+ JCasHashMap m = new JCasHashMap(200, true); // true = do use cache
+
+ for (int i = 0; i < n; i++) {
+ final int key = addrs[i];
+ TOP fs = new TOP(key, FAKE_TOP_TYPE_INSTANCE);
+// FeatureStructureImpl v = m.get(fs.getAddress());
+// if (null == v) {
+// m.get(7 * i);
+// m.findEmptySlot(key);
+ m.put(fs);
+// }
+ }
+
+ for (int i = 0; i < n; i++) {
+ final int key = addrs[i];
+ TOP fs = (TOP) m.getReserve(key);
+ if (fs == null) { // for debugging
+ System.out.println("stop");
+ }
+ assertTrue(null != fs);
+ }
+
+ }
+
+ public void testGrowth() {
+ System.out.println("JCasHashMapTest growth");
+ for (int th = 2; th <= 128; th *= 2) {
+ JCasHashMap.setDEFAULT_CONCURRENCY_LEVEL(th);
+ double loadfactor = .6; // from JCasHashMap impl
+ int sub_capacity = 32; // from JCasHashMap impl
+ int subs = th;
+ int agg_capacity = subs * sub_capacity;
+ JCasHashMap m = new JCasHashMap(agg_capacity, true); // true = do use cache
+ assertEquals(0, m.getApproximateSize());
+ assertEquals(agg_capacity, m.getCapacity());
+
+ int switchpoint = (int)Math.floor(agg_capacity * loadfactor);
+ fill(switchpoint, m);
+ System.out.print("JCasHashMapTest: after fill to switch point: ");
+ assertTrue(checkSubsCapacity(m, sub_capacity));
+ System.out.print("JCasHashMapTest: after 1 past switch point: ");
+ m.put(new TOP(addrs[switchpoint + 1], null));
+ assertTrue(checkSubsCapacity(m, sub_capacity));
+
+ m.clear();
+ System.out.print("JCasHashMapTest: after clear: ");
+ assertTrue(checkSubsCapacity(m, sub_capacity));
+
+
+ fill(switchpoint, m);
+ System.out.print("JCasHashMapTest: after fill to switch point: ");
+ assertTrue(checkSubsCapacity(m, sub_capacity));
+ m.put(new TOP(addrs[switchpoint + 1], null));
+ System.out.print("JCasHashMapTest: after 1 past switch point: ");
+ assertTrue(checkSubsCapacity(m, sub_capacity));
+
+ m.clear(); // size is above switchpoint, so no shrinkage
+ System.out.print("JCasHashMapTest: after clear (size above sp: ");
+ assertTrue(checkSubsCapacity(m, sub_capacity));
+ m.clear(); // size is 0, so first time shrinkage a possibility
+ System.out.print("JCasHashMapTest: clear (size below sp: ");
+ assertTrue(checkSubsCapacity(m, sub_capacity)); // but we don't shrink on first time
+ m.clear();
+ System.out.print("JCasHashMapTest: clear (size below 2nd time: ");
+ assertTrue(checkSubsCapacity(m, sub_capacity, sub_capacity)); // but we do on second time
+// m.clear();
+// System.out.print("JCasHashMapTest: clear (size below 3rd time: ");
+// assertTrue(checkSubsCapacity(m, sub_capacity, sub_capacity));
+// m.clear();
+// System.out.print("JCasHashMapTest: clear (size below 4th time: ");
+// assertTrue(checkSubsCapacity(m, sub_capacity, sub_capacity)); // don't shrink below minimum
+ }
+ }
+
+ private boolean checkSubsCapacity(JCasHashMap m, int v) {
+ return checkSubsCapacity(m, v, v * 2);
+ }
+
+ // check: the subMaps should be mostly of size v, but some might be of size v*2.
+ private boolean checkSubsCapacity(JCasHashMap m, int v, int v2) {
+ int[] caps = m.getCapacities();
+ for (int i : caps) {
+ if (i == v || i == v2 ) {
+ continue;
+ }
+ System.err.format("expected %d or %d, but got %s%n", v, v2, intList(caps));
+ return false;
+ }
+ System.out.format("%s%n", intListPm(caps, v));
+ return true;
+ }
+
+ private String intList(int[] a) {
+ StringBuilder sb = new StringBuilder();
+ for (int i : a) {
+ sb.append(i).append(", ");
+ }
+ return sb.toString();
+ }
+
+ private String intListPm(int[] a, int smaller) {
+ StringBuilder sb = new StringBuilder(a.length);
+ for (int i : a) {
+ sb.append(i == smaller ? '.' : '+');
+ }
+ return sb.toString();
+ }
+
+ private String intList(FeatureStructureImpl[] a) {
+ StringBuilder sb = new StringBuilder();
+ for (FeatureStructureImpl i : a) {
+ sb.append(i == null ? "null" : i.getAddress()).append(", ");
+ }
+ return sb.toString();
+ }
+
+ private void fill (int n, JCasHashMap m) {
+ for (int i = 0; i < n; i++) {
+ final int key = addrs[i];
+ TOP fs = new TOP(key, FAKE_TOP_TYPE_INSTANCE);
+ m.put(fs);
+// System.out.format("JCasHashMapTest fill %s%n", intList(m.getCapacities()));
+ }
+ }
+}