You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2018/03/14 04:08:14 UTC
[1/7] asterixdb git commit: [ASTERIXDB-2330][*DB][RT] Add
IFunctionRegistrant for dynamic function registration
Repository: asterixdb
Updated Branches:
refs/heads/master 38c41ac41 -> b8307794c
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index df09ca6..487877b 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@ -22,9 +22,12 @@ package org.apache.asterix.runtime.functions;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
+import java.util.ServiceLoader;
import org.apache.asterix.common.utils.CodeGenHelper;
+import org.apache.asterix.om.functions.IFunctionCollection;
import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.functions.IFunctionRegistrant;
import org.apache.asterix.runtime.aggregates.collections.FirstElementAggregateDescriptor;
import org.apache.asterix.runtime.aggregates.collections.ListifyAggregateDescriptor;
import org.apache.asterix.runtime.aggregates.collections.LocalFirstElementAggregateDescriptor;
@@ -139,8 +142,6 @@ import org.apache.asterix.runtime.evaluators.functions.CastTypeDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CastTypeLaxDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CheckUnknownDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CodePointToStringDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.CountHashedGramTokensDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.CountHashedWordTokensDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateCircleDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateLineDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateMBRDescriptor;
@@ -150,23 +151,15 @@ import org.apache.asterix.runtime.evaluators.functions.CreateQueryUIDDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateRectangleDescriptor;
import org.apache.asterix.runtime.evaluators.functions.CreateUUIDDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DeepEqualityDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.EditDistanceCheckDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.EditDistanceContainsDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.EditDistanceDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.EditDistanceListIsFilterableDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.EditDistanceStringIsFilterableDescriptor;
import org.apache.asterix.runtime.evaluators.functions.FullTextContainsDescriptor;
import org.apache.asterix.runtime.evaluators.functions.FullTextContainsWithoutOptionDescriptor;
import org.apache.asterix.runtime.evaluators.functions.GetItemDescriptor;
import org.apache.asterix.runtime.evaluators.functions.GetJobParameterByNameDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.GramTokensDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.HashedGramTokensDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.HashedWordTokensDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IfInfDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.IfNanOrInfDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IfMissingDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IfMissingOrNullDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IfNanDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.IfNanOrInfDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IfNullDescriptor;
import org.apache.asterix.runtime.evaluators.functions.InjectFailureDescriptor;
import org.apache.asterix.runtime.evaluators.functions.IsArrayDescriptor;
@@ -208,18 +201,10 @@ import org.apache.asterix.runtime.evaluators.functions.NumericTanDescriptor;
import org.apache.asterix.runtime.evaluators.functions.NumericTruncDescriptor;
import org.apache.asterix.runtime.evaluators.functions.NumericUnaryMinusDescriptor;
import org.apache.asterix.runtime.evaluators.functions.OrDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.PrefixLenJaccardDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardCheckDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardPrefixCheckDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardPrefixDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardSortedCheckDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardSortedDescriptor;
import org.apache.asterix.runtime.evaluators.functions.SleepDescriptor;
import org.apache.asterix.runtime.evaluators.functions.SpatialAreaDescriptor;
import org.apache.asterix.runtime.evaluators.functions.SpatialCellDescriptor;
import org.apache.asterix.runtime.evaluators.functions.SpatialDistanceDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.SpatialIntersectDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringConcatDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringContainsDescriptor;
import org.apache.asterix.runtime.evaluators.functions.StringEndsWithDescriptor;
@@ -262,7 +247,6 @@ import org.apache.asterix.runtime.evaluators.functions.ToDoubleDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ToNumberDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ToStringDescriptor;
import org.apache.asterix.runtime.evaluators.functions.UUIDDescriptor;
-import org.apache.asterix.runtime.evaluators.functions.WordTokensDescriptor;
import org.apache.asterix.runtime.evaluators.functions.binary.BinaryConcatDescriptor;
import org.apache.asterix.runtime.evaluators.functions.binary.BinaryLengthDescriptor;
import org.apache.asterix.runtime.evaluators.functions.binary.FindBinaryDescriptor;
@@ -343,16 +327,18 @@ import org.apache.asterix.runtime.unnestingfunctions.std.SubsetCollectionDescrip
/**
* This class holds a list of function descriptor factories.
*/
-public final class FunctionCollection {
+public final class FunctionCollection implements IFunctionCollection {
private static final String FACTORY = "FACTORY";
- private final List<IFunctionDescriptorFactory> descriptorFactories = new ArrayList<>();
+ private final ArrayList<IFunctionDescriptorFactory> descriptorFactories = new ArrayList<>();
+ @Override
public void add(IFunctionDescriptorFactory descriptorFactory) {
descriptorFactories.add(descriptorFactory);
}
+ @Override
public void addGenerated(IFunctionDescriptorFactory descriptorFactory) {
add(getGeneratedFunctionDescriptorFactory(descriptorFactory.createFunctionDescriptor().getClass()));
}
@@ -473,11 +459,6 @@ public final class FunctionCollection {
fc.add(CurrentTimeDescriptor.FACTORY);
fc.add(CurrentDateTimeDescriptor.FACTORY);
- // TODO: decide how should we deal these two weird functions as
- // the number of arguments of the function depend on the first few arguments.
- fc.add(SimilarityJaccardPrefixDescriptor.FACTORY);
- fc.add(SimilarityJaccardPrefixCheckDescriptor.FACTORY);
-
// functions that need generated class for null-handling.
// Element accessors.
@@ -614,7 +595,6 @@ public final class FunctionCollection {
fc.addGenerated(CreateRectangleDescriptor.FACTORY);
fc.addGenerated(SpatialAreaDescriptor.FACTORY);
fc.addGenerated(SpatialDistanceDescriptor.FACTORY);
- fc.addGenerated(SpatialIntersectDescriptor.FACTORY);
fc.addGenerated(CreateMBRDescriptor.FACTORY);
fc.addGenerated(SpatialCellDescriptor.FACTORY);
fc.addGenerated(PointXCoordinateAccessor.FACTORY);
@@ -623,24 +603,6 @@ public final class FunctionCollection {
fc.addGenerated(CircleCenterAccessor.FACTORY);
fc.addGenerated(LineRectanglePolygonAccessor.FACTORY);
- // fuzzyjoin function
- fc.addGenerated(PrefixLenJaccardDescriptor.FACTORY);
- fc.addGenerated(WordTokensDescriptor.FACTORY);
- fc.addGenerated(HashedWordTokensDescriptor.FACTORY);
- fc.addGenerated(CountHashedWordTokensDescriptor.FACTORY);
- fc.addGenerated(GramTokensDescriptor.FACTORY);
- fc.addGenerated(HashedGramTokensDescriptor.FACTORY);
- fc.addGenerated(CountHashedGramTokensDescriptor.FACTORY);
- fc.addGenerated(EditDistanceDescriptor.FACTORY);
- fc.addGenerated(EditDistanceCheckDescriptor.FACTORY);
- fc.addGenerated(EditDistanceStringIsFilterableDescriptor.FACTORY);
- fc.addGenerated(EditDistanceListIsFilterableDescriptor.FACTORY);
- fc.addGenerated(EditDistanceContainsDescriptor.FACTORY);
- fc.addGenerated(SimilarityJaccardDescriptor.FACTORY);
- fc.addGenerated(SimilarityJaccardCheckDescriptor.FACTORY);
- fc.addGenerated(SimilarityJaccardSortedDescriptor.FACTORY);
- fc.addGenerated(SimilarityJaccardSortedCheckDescriptor.FACTORY);
-
// full-text function
fc.addGenerated(FullTextContainsDescriptor.FACTORY);
fc.addGenerated(FullTextContainsWithoutOptionDescriptor.FACTORY);
@@ -742,6 +704,7 @@ public final class FunctionCollection {
// Record function
fc.addGenerated(RecordPairsDescriptor.FACTORY);
+ ServiceLoader.load(IFunctionRegistrant.class).iterator().forEachRemaining(c -> c.register(fc));
return fc;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-server/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-server/pom.xml b/asterixdb/asterix-server/pom.xml
index 8a28a49..7e86edc 100644
--- a/asterixdb/asterix-server/pom.xml
+++ b/asterixdb/asterix-server/pom.xml
@@ -371,6 +371,7 @@
<usedDependency>org.apache.hyracks:hyracks-nc-service</usedDependency>
<usedDependency>org.apache.asterix:asterix-external-data</usedDependency>
<usedDependency>org.codehaus.mojo.appassembler:appassembler-booter</usedDependency>
+ <usedDependency>org.apache.asterix:asterix-fuzzyjoin</usedDependency>
</usedDependencies>
</configuration>
</plugin>
@@ -614,5 +615,10 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-fuzzyjoin</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
[4/7] asterixdb git commit: [ASTERIXDB-2330][*DB][RT] Add
IFunctionRegistrant for dynamic function registration
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/resources/META-INF/services/org.apache.asterix.om.functions.IFunctionRegistrant
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/resources/META-INF/services/org.apache.asterix.om.functions.IFunctionRegistrant b/asterixdb/asterix-fuzzyjoin/src/main/resources/META-INF/services/org.apache.asterix.om.functions.IFunctionRegistrant
new file mode 100644
index 0000000..c981de2
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/resources/META-INF/services/org.apache.asterix.om.functions.IFunctionRegistrant
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+org.apache.asterix.runtime.FuzzyJoinFunctionRegistrant
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionCollection.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionCollection.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionCollection.java
new file mode 100644
index 0000000..90c742c
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionCollection.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.functions;
+
+import java.io.Serializable;
+
+public interface IFunctionCollection extends Serializable {
+ void add(IFunctionDescriptorFactory descriptorFactory);
+
+ void addGenerated(IFunctionDescriptorFactory descriptorFactory);
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionRegistrant.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionRegistrant.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionRegistrant.java
new file mode 100644
index 0000000..2fa83fc
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionRegistrant.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.functions;
+
+import java.io.Serializable;
+
+public interface IFunctionRegistrant extends Serializable {
+ void register(IFunctionCollection collection);
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-om/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java
new file mode 100644
index 0000000..c48fff1
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.base;
+
+import org.apache.asterix.common.functions.FunctionDescriptorTag;
+import org.apache.asterix.om.functions.AbstractFunctionDescriptor;
+
+public abstract class AbstractScalarFunctionDynamicDescriptor extends AbstractFunctionDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FunctionDescriptorTag getFunctionDescriptorTag() {
+ return FunctionDescriptorTag.SCALAR;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/pom.xml b/asterixdb/asterix-runtime/pom.xml
index 239b01a..22d1bfc 100644
--- a/asterixdb/asterix-runtime/pom.xml
+++ b/asterixdb/asterix-runtime/pom.xml
@@ -92,12 +92,6 @@
<scope>compile</scope>
</dependency>
<dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-fuzzyjoin</artifactId>
- <version>${project.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java
deleted file mode 100644
index c48fff1..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/base/AbstractScalarFunctionDynamicDescriptor.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.base;
-
-import org.apache.asterix.common.functions.FunctionDescriptorTag;
-import org.apache.asterix.om.functions.AbstractFunctionDescriptor;
-
-public abstract class AbstractScalarFunctionDynamicDescriptor extends AbstractFunctionDescriptor {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public FunctionDescriptorTag getFunctionDescriptorTag() {
- return FunctionDescriptorTag.SCALAR;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java
deleted file mode 100644
index dece292..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.IOException;
-
-import org.apache.asterix.builders.OrderedListBuilder;
-import org.apache.asterix.common.exceptions.ErrorCode;
-import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.types.AOrderedListType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class EditDistanceCheckEvaluator extends EditDistanceEvaluator {
-
- protected final IScalarEvaluator edThreshEval;
- protected int edThresh;
- private final IPointable argPtrThreshold = new VoidPointable();
- protected final OrderedListBuilder listBuilder;
- protected ArrayBackedValueStorage listItemVal;
- @SuppressWarnings("unchecked")
- protected final ISerializerDeserializer<ABoolean> booleanSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
-
- public EditDistanceCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- super(args, context);
- edThreshEval = args[2].createScalarEvaluator(context);
- listBuilder = new OrderedListBuilder();
- listItemVal = new ArrayBackedValueStorage();
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- firstStringEval.evaluate(tuple, argPtr1);
- firstTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
- secondStringEval.evaluate(tuple, argPtr2);
- secondTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
- edThreshEval.evaluate(tuple, argPtrThreshold);
-
- if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
- result.set(resultStorage);
- return;
- }
- try {
- edThresh = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_CHECK.getName(), 2,
- argPtrThreshold.getByteArray(), argPtrThreshold.getStartOffset());
- if (edThresh < 0) {
- throw new RuntimeDataException(ErrorCode.NEGATIVE_VALUE, BuiltinFunctions.EDIT_DISTANCE_CHECK.getName(),
- 3, edThresh);
- }
- editDistance = computeResult(argPtr1, argPtr2, firstTypeTag);
- writeResult(editDistance);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-
- @Override
- protected int computeResult(IPointable left, IPointable right, ATypeTag argType) throws HyracksDataException {
- byte[] leftBytes = left.getByteArray();
- int leftStartOffset = left.getStartOffset();
- byte[] rightBytes = right.getByteArray();
- int rightStartOffset = right.getStartOffset();
- switch (argType) {
- case STRING: {
- return ed.UTF8StringEditDistance(leftBytes, leftStartOffset + typeIndicatorSize, rightBytes,
- rightStartOffset + typeIndicatorSize, edThresh);
- }
-
- case ARRAY: {
- firstOrdListIter.reset(leftBytes, leftStartOffset);
- secondOrdListIter.reset(rightBytes, rightStartOffset);
- return (int) ed.computeSimilarity(firstOrdListIter, secondOrdListIter, edThresh);
- }
-
- default: {
- throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_CHECK, 0, argType.serialize(),
- ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
- }
-
- }
- }
-
- @Override
- protected void writeResult(int ed) throws IOException {
- listBuilder.reset(new AOrderedListType(BuiltinType.ANY, "list"));
- boolean matches = (ed < 0) ? false : true;
- listItemVal.reset();
- booleanSerde.serialize(matches ? ABoolean.TRUE : ABoolean.FALSE, listItemVal.getDataOutput());
- listBuilder.addItem(listItemVal);
-
- listItemVal.reset();
- aInt64.setValue((matches) ? ed : Integer.MAX_VALUE);
- int64Serde.serialize(aInt64, listItemVal.getDataOutput());
- listBuilder.addItem(listItemVal);
- listBuilder.write(out, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java
deleted file mode 100644
index eaf3368..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-
-public class EditDistanceContainsEvaluator extends EditDistanceCheckEvaluator {
-
- public EditDistanceContainsEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- super(args, context);
- }
-
- @Override
- protected int computeResult(IPointable left, IPointable right, ATypeTag argType) throws HyracksDataException {
- byte[] leftBytes = left.getByteArray();
- int leftStartOffset = left.getStartOffset();
- byte[] rightBytes = right.getByteArray();
- int rightStartOffset = right.getStartOffset();
-
- switch (argType) {
- case STRING: {
- return ed.UTF8StringEditDistanceContains(leftBytes, leftStartOffset + typeIndicatorSize, rightBytes,
- rightStartOffset + typeIndicatorSize, edThresh);
- }
- case ARRAY: {
- firstOrdListIter.reset(leftBytes, leftStartOffset);
- secondOrdListIter.reset(rightBytes, rightStartOffset);
- return ed.getSimilarityContains(firstOrdListIter, secondOrdListIter, edThresh);
- }
- default: {
- throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_CONTAINS, 0, argType.serialize(),
- ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java
deleted file mode 100644
index 85fd334..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetricEditDistance;
-import org.apache.asterix.om.base.AInt64;
-import org.apache.asterix.om.base.AMutableInt64;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.asterix.runtime.exceptions.IncompatibleTypeException;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.asterix.runtime.exceptions.UnsupportedItemTypeException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class EditDistanceEvaluator implements IScalarEvaluator {
-
- // assuming type indicator in serde format
- protected final int typeIndicatorSize = 1;
-
- protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- protected final DataOutput out = resultStorage.getDataOutput();
- protected final IPointable argPtr1 = new VoidPointable();
- protected final IPointable argPtr2 = new VoidPointable();
- protected final IScalarEvaluator firstStringEval;
- protected final IScalarEvaluator secondStringEval;
- protected final SimilarityMetricEditDistance ed = new SimilarityMetricEditDistance();
- protected final OrderedListIterator firstOrdListIter = new OrderedListIterator();
- protected final OrderedListIterator secondOrdListIter = new OrderedListIterator();
- protected int editDistance = 0;
- protected final AMutableInt64 aInt64 = new AMutableInt64(-1);
- @SuppressWarnings("unchecked")
- protected final ISerializerDeserializer<AInt64> int64Serde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
- protected ATypeTag itemTypeTag;
-
- protected ATypeTag firstTypeTag;
- protected ATypeTag secondTypeTag;
-
- public EditDistanceEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- firstStringEval = args[0].createScalarEvaluator(context);
- secondStringEval = args[1].createScalarEvaluator(context);
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- firstStringEval.evaluate(tuple, argPtr1);
- firstTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
- secondStringEval.evaluate(tuple, argPtr2);
- secondTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
-
- if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
- result.set(resultStorage);
- return;
- }
-
- editDistance = computeResult(argPtr1, argPtr2, firstTypeTag);
- try {
- writeResult(editDistance);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-
- protected int computeResult(IPointable left, IPointable right, ATypeTag argType) throws HyracksDataException {
- byte[] leftBytes = left.getByteArray();
- int leftStartOffset = left.getStartOffset();
- byte[] rightBytes = right.getByteArray();
- int rightStartOffset = right.getStartOffset();
-
- switch (argType) {
- case STRING: {
- // Passes -1 as the simThresh to calculate the edit distance
- // without applying any calculation optimizations.
- return ed.getActualUTF8StringEditDistanceVal(leftBytes, leftStartOffset + typeIndicatorSize, rightBytes,
- rightStartOffset + typeIndicatorSize, -1);
- }
- case ARRAY: {
- firstOrdListIter.reset(leftBytes, leftStartOffset);
- secondOrdListIter.reset(rightBytes, rightStartOffset);
- return (int) ed.computeSimilarity(firstOrdListIter, secondOrdListIter);
- }
- default: {
- throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE, 0, argType.serialize(),
- ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
- }
-
- }
- }
-
- protected boolean checkArgTypes(ATypeTag typeTag1, ATypeTag typeTag2) throws HyracksDataException {
- if (typeTag1 != typeTag2) {
- throw new IncompatibleTypeException(BuiltinFunctions.EDIT_DISTANCE, typeTag1.serialize(),
- typeTag2.serialize());
- }
-
- // Since they are equal, check one tag is enough.
- if (typeTag1 != ATypeTag.STRING && typeTag1 != ATypeTag.ARRAY) { // could be an list
- throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE, 0, typeTag1.serialize(),
- ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
- }
-
- if (typeTag1 == ATypeTag.ARRAY) {
- itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER
- .deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset() + 1]);
- if (itemTypeTag == ATypeTag.ANY) {
- throw new UnsupportedItemTypeException(BuiltinFunctions.EDIT_DISTANCE, itemTypeTag.serialize());
- }
- itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER
- .deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset() + 1]);
- if (itemTypeTag == ATypeTag.ANY) {
- throw new UnsupportedItemTypeException(BuiltinFunctions.EDIT_DISTANCE, itemTypeTag.serialize());
- }
- }
- return true;
- }
-
- protected void writeResult(int ed) throws IOException {
- aInt64.setValue(ed);
- int64Serde.serialize(aInt64, out);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java
deleted file mode 100644
index ef727c9..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.builders.OrderedListBuilder;
-import org.apache.asterix.dataflow.data.nontagged.serde.ABooleanSerializerDeserializer;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.types.AOrderedListType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
-
-public class GramTokensEvaluator implements IScalarEvaluator {
-
- // assuming type indicator in serde format
- private final int typeIndicatorSize = 1;
-
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput out = resultStorage.getDataOutput();
- private final IPointable stringArg = new VoidPointable();
- private final IPointable gramLengthArg = new VoidPointable();
- private final IPointable prePostArg = new VoidPointable();
- private final IScalarEvaluator stringEval;
- private final IScalarEvaluator gramLengthEval;
- private final IScalarEvaluator prePostEval;
-
- private final NGramUTF8StringBinaryTokenizer tokenizer;
- private final OrderedListBuilder listBuilder = new OrderedListBuilder();
- private final AOrderedListType listType;
-
- public GramTokensEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context, IBinaryTokenizer tokenizer,
- BuiltinType itemType) throws HyracksDataException {
- stringEval = args[0].createScalarEvaluator(context);
- gramLengthEval = args[1].createScalarEvaluator(context);
- prePostEval = args[2].createScalarEvaluator(context);
- this.tokenizer = (NGramUTF8StringBinaryTokenizer) tokenizer;
- this.listType = new AOrderedListType(itemType, null);
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- stringEval.evaluate(tuple, stringArg);
- gramLengthEval.evaluate(tuple, gramLengthArg);
- prePostEval.evaluate(tuple, prePostArg);
-
- int gramLength = ATypeHierarchy.getIntegerValue(BuiltinFunctions.GRAM_TOKENS.getName(), 1,
- gramLengthArg.getByteArray(), gramLengthArg.getStartOffset());
- tokenizer.setGramlength(gramLength);
- boolean prePost = ABooleanSerializerDeserializer.getBoolean(prePostArg.getByteArray(),
- prePostArg.getStartOffset() + typeIndicatorSize);
- tokenizer.setPrePost(prePost);
- tokenizer.reset(stringArg.getByteArray(), stringArg.getStartOffset(), stringArg.getLength());
-
- try {
- listBuilder.reset(listType);
- while (tokenizer.hasNext()) {
- tokenizer.next();
- listBuilder.addItem(tokenizer.getToken());
- }
- listBuilder.write(out, true);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java
deleted file mode 100644
index 4f5fb69..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.DataInputStream;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityFilters;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityFiltersFactory;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
-import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-
-public class SimilarityFiltersCache {
- private final UTF8StringSerializerDeserializer utf8SerDer = new UTF8StringSerializerDeserializer();
-
- private final ByteBufferInputStream bbis = new ByteBufferInputStream();
- private final DataInputStream dis = new DataInputStream(bbis);
-
- private float similarityThresholdCached = 0;
- private byte[] similarityNameBytesCached = null;
- private SimilarityFilters similarityFiltersCached = null;
-
- public SimilarityFilters get(float similarityThreshold, byte[] similarityNameBytes, int startOffset, int len)
- throws HyracksDataException {
- if (similarityThreshold != similarityThresholdCached || similarityNameBytesCached == null
- || !Arrays.equals(similarityNameBytes, similarityNameBytesCached)) {
- bbis.setByteBuffer(ByteBuffer.wrap(similarityNameBytes), startOffset + 1);
- String similarityName = utf8SerDer.deserialize(dis);
- similarityNameBytesCached = Arrays.copyOfRange(similarityNameBytes, startOffset, len);
- similarityFiltersCached =
- SimilarityFiltersFactory.getSimilarityFilters(similarityName, similarityThreshold);
- }
- return similarityFiltersCached;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
deleted file mode 100644
index 60b5592..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.IOException;
-
-import org.apache.asterix.builders.OrderedListBuilder;
-import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.types.AOrderedListType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.data.std.util.BinaryEntry;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class SimilarityJaccardCheckEvaluator extends SimilarityJaccardEvaluator {
-
- protected final IScalarEvaluator jaccThreshEval;
- protected float jaccThresh = -1f;
- protected IPointable jaccThreshPointable = new VoidPointable();
-
- protected OrderedListBuilder listBuilder;
- protected ArrayBackedValueStorage inputVal;
- @SuppressWarnings("unchecked")
- protected final ISerializerDeserializer<ABoolean> booleanSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
- protected final AOrderedListType listType = new AOrderedListType(BuiltinType.ANY, "list");
-
- public SimilarityJaccardCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- super(args, context);
- jaccThreshEval = args[2].createScalarEvaluator(context);
- listBuilder = new OrderedListBuilder();
- inputVal = new ArrayBackedValueStorage();
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
-
- firstOrdListEval.evaluate(tuple, argPtr1);
- secondOrdListEval.evaluate(tuple, argPtr2);
- jaccThreshEval.evaluate(tuple, jaccThreshPointable);
-
- firstTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
- secondTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
-
- firstItemTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset() + 1]);
- secondItemTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset() + 1]);
-
- jaccThresh = AFloatSerializerDeserializer.getFloat(jaccThreshPointable.getByteArray(),
- jaccThreshPointable.getStartOffset() + TYPE_INDICATOR_SIZE);
-
- if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
- result.set(resultStorage);
- return;
- }
- if (prepareLists(argPtr1, argPtr2)) {
- jaccSim = computeResult();
- } else {
- jaccSim = 0.0f;
- }
- try {
- writeResult(jaccSim);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-
- @Override
- protected int probeHashMap(AbstractAsterixListIterator probeIter, int buildListSize, int probeListSize)
- throws HyracksDataException {
- // Apply length filter.
- int lengthLowerBound = (int) Math.ceil(jaccThresh * probeListSize);
- if ((lengthLowerBound > buildListSize)
- || (buildListSize > (int) Math.floor(1.0f / jaccThresh * probeListSize))) {
- return -1;
- }
- // Probe phase: Probe items from second list, and compute intersection size.
- int intersectionSize = 0;
- int probeListCount = 0;
- int minUnionSize = buildListSize;
- while (probeIter.hasNext()) {
- probeListCount++;
- byte[] buf = probeIter.getData();
- int off = probeIter.getPos();
- int len = probeIter.getItemLen();
- keyEntry.set(buf, off, len);
- BinaryEntry entry = hashMap.get(keyEntry);
- if (entry != null) {
- // Increment second value.
- int firstValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset());
- // Irrelevant for the intersection size.
- if (firstValInt == 0) {
- continue;
- }
- int secondValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset() + 4);
- // Subtract old min value.
- intersectionSize -= (firstValInt < secondValInt) ? firstValInt : secondValInt;
- secondValInt++;
- // Add new min value.
- intersectionSize += (firstValInt < secondValInt) ? firstValInt : secondValInt;
- IntegerPointable.setInteger(entry.getBuf(), entry.getOffset() + 4, secondValInt);
- } else {
- // Could not find element in other set. Increase min union size by 1.
- minUnionSize++;
- // Check whether jaccThresh can still be satisfied if there was a mismatch.
- int maxIntersectionSize = Math.min(buildListSize, intersectionSize + (probeListSize - probeListCount));
- int lowerBound = (int) Math.floor(jaccThresh * minUnionSize);
- if (maxIntersectionSize < lowerBound) {
- // Cannot satisfy jaccThresh.
- return -1;
- }
- }
- probeIter.next();
- }
- return intersectionSize;
- }
-
- @Override
- protected void writeResult(float jacc) throws IOException {
- listBuilder.reset(listType);
- boolean matches = (jacc < jaccThresh) ? false : true;
- inputVal.reset();
- booleanSerde.serialize(matches ? ABoolean.TRUE : ABoolean.FALSE, inputVal.getDataOutput());
- listBuilder.addItem(inputVal);
-
- inputVal.reset();
- aFloat.setValue((matches) ? jacc : 0.0f);
- floatSerde.serialize(aFloat, inputVal.getDataOutput());
- listBuilder.addItem(inputVal);
-
- listBuilder.write(out, true);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
deleted file mode 100644
index 1e5ad3c..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.asterix.dataflow.data.nontagged.comparators.ListItemBinaryComparatorFactory;
-import org.apache.asterix.dataflow.data.nontagged.hash.ListItemBinaryHashFunctionFactory;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.om.base.AFloat;
-import org.apache.asterix.om.base.AMutableFloat;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.asterix.runtime.evaluators.functions.BinaryHashMap;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
-import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.data.std.util.BinaryEntry;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class SimilarityJaccardEvaluator implements IScalarEvaluator {
-
- // Parameters for hash table.
- protected static final int MIN_TABLE_SIZE = 100;
- protected static final int TABLE_FRAME_SIZE = 32768;
-
- // Assuming type indicator in serde format.
- protected static final int TYPE_INDICATOR_SIZE = 1;
-
- protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- protected final DataOutput out = resultStorage.getDataOutput();
- protected final IPointable argPtr1 = new VoidPointable();
- protected final IPointable argPtr2 = new VoidPointable();
- protected final IScalarEvaluator firstOrdListEval;
- protected final IScalarEvaluator secondOrdListEval;
-
- protected final OrderedListIterator fstOrdListIter = new OrderedListIterator();
- protected final OrderedListIterator sndOrdListIter = new OrderedListIterator();
- protected final UnorderedListIterator fstUnordListIter = new UnorderedListIterator();
- protected final UnorderedListIterator sndUnordListIter = new UnorderedListIterator();
-
- protected AbstractAsterixListIterator firstListIter;
- protected AbstractAsterixListIterator secondListIter;
-
- protected final AMutableFloat aFloat = new AMutableFloat(0);
- @SuppressWarnings("unchecked")
- protected final ISerializerDeserializer<AFloat> floatSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AFLOAT);
-
- protected ATypeTag firstTypeTag;
- protected ATypeTag secondTypeTag;
- protected float jaccSim = 0.0f;
- protected ATypeTag firstItemTypeTag;
- protected ATypeTag secondItemTypeTag;
-
- protected BinaryHashMap hashMap;
- protected BinaryEntry keyEntry = new BinaryEntry();
- protected BinaryEntry valEntry = new BinaryEntry();
-
- // Ignore case for strings. Defaults to true.
- protected final boolean ignoreCase = true;
-
- protected int hashTableSize = MIN_TABLE_SIZE;
-
- public SimilarityJaccardEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- firstOrdListEval = args[0].createScalarEvaluator(context);
- secondOrdListEval = args[1].createScalarEvaluator(context);
- byte[] emptyValBuf = new byte[8];
- Arrays.fill(emptyValBuf, (byte) 0);
- valEntry.set(emptyValBuf, 0, 8);
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
-
- firstOrdListEval.evaluate(tuple, argPtr1);
- secondOrdListEval.evaluate(tuple, argPtr2);
-
- firstTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
- secondTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
-
- firstItemTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset() + 1]);
- secondItemTypeTag =
- EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset() + 1]);
-
- if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
- result.set(resultStorage);
- return;
- }
- if (prepareLists(argPtr1, argPtr2)) {
- jaccSim = computeResult();
- } else {
- jaccSim = 0.0f;
- }
- try {
- writeResult(jaccSim);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-
- protected boolean prepareLists(IPointable left, IPointable right) throws HyracksDataException {
- firstListIter.reset(left.getByteArray(), left.getStartOffset());
- secondListIter.reset(right.getByteArray(), right.getStartOffset());
- // Check for special case where one of the lists is empty, since list
- // types won't match.
- if (firstListIter.size() == 0 || secondListIter.size() == 0) {
- return false;
- }
-
- // Set the size of the table dynamically
- hashTableSize = Math.max(Math.max(firstListIter.size(), secondListIter.size()), MIN_TABLE_SIZE);
-
- // TODO: Check item types are compatible.
- return true;
- }
-
- protected float computeResult() throws HyracksDataException {
- // We will subtract the intersection size later to get the real union size.
- int firstListSize = firstListIter.size();
- int secondListSize = secondListIter.size();
- int unionSize = firstListSize + secondListSize;
- // Choose smaller list as build, and larger one as probe.
- AbstractAsterixListIterator buildList = (firstListSize < secondListSize) ? firstListIter : secondListIter;
- AbstractAsterixListIterator probeList = (buildList == firstListIter) ? secondListIter : firstListIter;
- int buildListSize = (buildList == firstListIter) ? firstListSize : secondListSize;
- int probeListSize = (probeList == firstListIter) ? firstListSize : secondListSize;
- ATypeTag buildItemTypeTag = (buildList == firstListIter) ? firstItemTypeTag : secondItemTypeTag;
- ATypeTag probeItemTypeTag = (probeList == firstListIter) ? firstItemTypeTag : secondItemTypeTag;
-
- setHashMap(buildItemTypeTag, probeItemTypeTag);
- buildHashMap(buildList);
- int intersectionSize = probeHashMap(probeList, buildListSize, probeListSize);
- // Special indicator for the "check" version of jaccard.
- if (intersectionSize < 0) {
- return -1;
- }
- unionSize -= intersectionSize;
- return (float) intersectionSize / (float) unionSize;
- }
-
- protected void buildHashMap(AbstractAsterixListIterator buildIter) throws HyracksDataException {
- // Build phase: Add items into hash map, starting with first list.
- // Value in map is a pair of integers. Set first integer to 1.
- IntegerPointable.setInteger(valEntry.getBuf(), 0, 1);
- while (buildIter.hasNext()) {
- byte[] buf = buildIter.getData();
- int off = buildIter.getPos();
- int len = buildIter.getItemLen();
- keyEntry.set(buf, off, len);
- BinaryEntry entry = hashMap.put(keyEntry, valEntry);
- if (entry != null) {
- // Increment value.
- int firstValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset());
- IntegerPointable.setInteger(entry.getBuf(), entry.getOffset(), firstValInt + 1);
- }
- buildIter.next();
- }
- }
-
- protected int probeHashMap(AbstractAsterixListIterator probeIter, int buildListSize, int probeListSize)
- throws HyracksDataException {
- // Probe phase: Probe items from second list, and compute intersection size.
- int intersectionSize = 0;
- while (probeIter.hasNext()) {
- byte[] buf = probeIter.getData();
- int off = probeIter.getPos();
- int len = probeIter.getItemLen();
- keyEntry.set(buf, off, len);
- BinaryEntry entry = hashMap.get(keyEntry);
- if (entry != null) {
- // Increment second value.
- int firstValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset());
- // Irrelevant for the intersection size.
- if (firstValInt == 0) {
- continue;
- }
- int secondValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset() + 4);
- // Subtract old min value.
- intersectionSize -= (firstValInt < secondValInt) ? firstValInt : secondValInt;
- secondValInt++;
- // Add new min value.
- intersectionSize += (firstValInt < secondValInt) ? firstValInt : secondValInt;
- IntegerPointable.setInteger(entry.getBuf(), entry.getOffset() + 4, secondValInt);
- }
- probeIter.next();
- }
- return intersectionSize;
- }
-
- protected void setHashMap(ATypeTag buildItemTypeTag, ATypeTag probeItemTypeTag) {
- if (hashMap != null) {
- hashMap.clear();
- return;
- }
-
- IBinaryHashFunction putHashFunc =
- ListItemBinaryHashFunctionFactory.INSTANCE.createBinaryHashFunction(buildItemTypeTag, ignoreCase);
- IBinaryHashFunction getHashFunc =
- ListItemBinaryHashFunctionFactory.INSTANCE.createBinaryHashFunction(probeItemTypeTag, ignoreCase);
- IBinaryComparator cmp = ListItemBinaryComparatorFactory.INSTANCE.createBinaryComparator(buildItemTypeTag,
- probeItemTypeTag, ignoreCase);
- hashMap = new BinaryHashMap(hashTableSize, TABLE_FRAME_SIZE, putHashFunc, getHashFunc, cmp);
- }
-
- protected boolean checkArgTypes(ATypeTag typeTag1, ATypeTag typeTag2) throws HyracksDataException {
- switch (typeTag1) {
- case ARRAY: {
- firstListIter = fstOrdListIter;
- break;
- }
- case MULTISET: {
- firstListIter = fstUnordListIter;
- break;
- }
- default: {
- throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 0, typeTag1.serialize(),
- ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
- }
- switch (typeTag2) {
- case ARRAY: {
- secondListIter = sndOrdListIter;
- break;
- }
- case MULTISET: {
- secondListIter = sndUnordListIter;
- break;
- }
- default: {
- throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 1, typeTag2.serialize(),
- ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
- }
- return true;
- }
-
- protected void writeResult(float jacc) throws IOException {
- aFloat.setValue(jacc);
- floatSerde.serialize(aFloat, out);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java
deleted file mode 100644
index b70c6ad..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AUnorderedListSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.fuzzyjoin.IntArray;
-import org.apache.asterix.fuzzyjoin.similarity.PartialIntersect;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityFiltersJaccard;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetric;
-import org.apache.asterix.om.base.AFloat;
-import org.apache.asterix.om.base.AMutableFloat;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class SimilarityJaccardPrefixEvaluator implements IScalarEvaluator {
- // assuming type indicator in serde format
- protected final int typeIndicatorSize = 1;
-
- protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- protected final DataOutput out = resultStorage.getDataOutput();
- protected final IPointable inputVal = new VoidPointable();
- protected final IScalarEvaluator evalLen1;
- protected final IScalarEvaluator evalTokens1;
- protected final IScalarEvaluator evalLen2;
- protected final IScalarEvaluator evalTokens2;
- protected final IScalarEvaluator evalTokenPrefix;
- protected final IScalarEvaluator evalThreshold;
-
- protected float similarityThresholdCache;
- protected SimilarityFiltersJaccard similarityFilters;
- protected final IntArray tokens1 = new IntArray();
- protected final IntArray tokens2 = new IntArray();
- protected final PartialIntersect parInter = new PartialIntersect();
-
- protected float sim = 0.0f;
-
- // result
- protected final AMutableFloat res = new AMutableFloat(0);
- @SuppressWarnings("unchecked")
- protected final ISerializerDeserializer<AFloat> reusSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AFLOAT);
-
- public SimilarityJaccardPrefixEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- evalLen1 = args[0].createScalarEvaluator(context);
- evalTokens1 = args[1].createScalarEvaluator(context);
- evalLen2 = args[2].createScalarEvaluator(context);
- evalTokens2 = args[3].createScalarEvaluator(context);
- evalTokenPrefix = args[4].createScalarEvaluator(context);
- evalThreshold = args[5].createScalarEvaluator(context);
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- // similarity threshold
- sim = 0;
- evalThreshold.evaluate(tuple, inputVal);
- float similarityThreshold =
- AFloatSerializerDeserializer.getFloat(inputVal.getByteArray(), inputVal.getStartOffset() + 1);
-
- if (similarityThreshold != similarityThresholdCache || similarityFilters == null) {
- similarityFilters = new SimilarityFiltersJaccard(similarityThreshold);
- similarityThresholdCache = similarityThreshold;
- }
-
- evalLen1.evaluate(tuple, inputVal);
- int length1 = ATypeHierarchy.getIntegerValue(BuiltinFunctions.SIMILARITY_JACCARD.getName(), 0,
- inputVal.getByteArray(), inputVal.getStartOffset());
- evalLen2.evaluate(tuple, inputVal);
- int length2 = ATypeHierarchy.getIntegerValue(BuiltinFunctions.SIMILARITY_JACCARD.getName(), 2,
- inputVal.getByteArray(), inputVal.getStartOffset());
-
- //
- // -- - length filter - --
- //
- if (similarityFilters.passLengthFilter(length1, length2)) {
-
- // -- - tokens1 - --
- int i;
- tokens1.reset();
- evalTokens1.evaluate(tuple, inputVal);
-
- byte[] serList = inputVal.getByteArray();
- int startOffset = inputVal.getStartOffset();
- if (serList[startOffset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
- && serList[startOffset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 1, serList[startOffset],
- ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
-
- int lengthTokens1;
- if (serList[startOffset] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
- lengthTokens1 =
- AOrderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
- // read tokens
- for (i = 0; i < lengthTokens1; i++) {
- int itemOffset = AOrderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
- int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
- BuiltinFunctions.SIMILARITY_JACCARD.getName(), 1, serList, itemOffset, startOffset + 1);
- tokens1.add(token);
- }
- } else {
- lengthTokens1 =
- AUnorderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
- // read tokens
- for (i = 0; i < lengthTokens1; i++) {
- int itemOffset = AUnorderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
- int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
- BuiltinFunctions.SIMILARITY_JACCARD.getName(), 1, serList, itemOffset, startOffset + 1);
- tokens1.add(token);
- }
- }
- // pad tokens
- for (; i < length1; i++) {
- tokens1.add(Integer.MAX_VALUE);
- }
-
- // -- - tokens2 - --
- tokens2.reset();
- evalTokens2.evaluate(tuple, inputVal);
-
- serList = inputVal.getByteArray();
- startOffset = inputVal.getStartOffset();
- if (serList[startOffset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
- && serList[startOffset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 3, serList[startOffset],
- ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
-
- int lengthTokens2;
- if (serList[startOffset] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
- lengthTokens2 =
- AOrderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
- // read tokens
- for (i = 0; i < lengthTokens2; i++) {
- int itemOffset = AOrderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
- int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
- BuiltinFunctions.SIMILARITY_JACCARD.getName(), 3, serList, itemOffset, startOffset + 1);
- tokens2.add(token);
- }
- } else {
- lengthTokens2 =
- AUnorderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
- // read tokens
- for (i = 0; i < lengthTokens2; i++) {
- int itemOffset = AUnorderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
- int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
- BuiltinFunctions.SIMILARITY_JACCARD.getName(), 3, serList, itemOffset, startOffset + 1);
- tokens2.add(token);
- }
- }
- // pad tokens
- for (; i < length2; i++) {
- tokens2.add(Integer.MAX_VALUE);
- }
-
- // -- - token prefix - --
- evalTokenPrefix.evaluate(tuple, inputVal);
- int tokenPrefix = ATypeHierarchy.getIntegerValue(BuiltinFunctions.SIMILARITY_JACCARD.getName(), 4,
- inputVal.getByteArray(), inputVal.getStartOffset());
-
- //
- // -- - position filter - --
- //
- SimilarityMetric.getPartialIntersectSize(tokens1.get(), 0, tokens1.length(), tokens2.get(), 0,
- tokens2.length(), tokenPrefix, parInter);
- if (similarityFilters.passPositionFilter(parInter.intersectSize, parInter.posXStop, length1,
- parInter.posYStop, length2)) {
-
- //
- // -- - suffix filter - --
- //
- if (similarityFilters.passSuffixFilter(tokens1.get(), 0, tokens1.length(), parInter.posXStart,
- tokens2.get(), 0, tokens2.length(), parInter.posYStart)) {
-
- sim = similarityFilters.passSimilarityFilter(tokens1.get(), 0, tokens1.length(),
- parInter.posXStop + 1, tokens2.get(), 0, tokens2.length(), parInter.posYStop + 1,
- parInter.intersectSize);
- }
- }
- }
-
- try {
- writeResult();
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-
- public void writeResult() throws IOException {
- res.setValue(sim);
- reusSerde.serialize(res, out);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java
deleted file mode 100644
index 0decd9e..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetricJaccard;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class SimilarityJaccardSortedCheckEvaluator extends SimilarityJaccardCheckEvaluator {
-
- protected final SimilarityMetricJaccard jaccard = new SimilarityMetricJaccard();
-
- public SimilarityJaccardSortedCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- super(args, context);
- }
-
- @Override
- protected float computeResult() throws HyracksDataException {
- return jaccard.computeSimilarity(firstListIter, secondListIter, jaccThresh);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java
deleted file mode 100644
index 1cd32c8..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetricJaccard;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-// Assumes that both arguments are sorted by the same ordering.
-public class SimilarityJaccardSortedEvaluator extends SimilarityJaccardEvaluator {
-
- protected final SimilarityMetricJaccard jaccard = new SimilarityMetricJaccard();
-
- public SimilarityJaccardSortedEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- super(args, context);
- }
-
- @Override
- protected float computeResult() throws HyracksDataException {
- return jaccard.computeSimilarity(firstListIter, secondListIter);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java
deleted file mode 100644
index e51d5cf..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.common;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.builders.OrderedListBuilder;
-import org.apache.asterix.om.types.AOrderedListType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
-
-public class WordTokensEvaluator implements IScalarEvaluator {
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput out = resultStorage.getDataOutput();
- private final IPointable argPtr = new VoidPointable();
- private final IScalarEvaluator stringEval;
-
- private final IBinaryTokenizer tokenizer;
- private final OrderedListBuilder listBuilder = new OrderedListBuilder();
- private final AOrderedListType listType;
-
- public WordTokensEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context, IBinaryTokenizer tokenizer,
- BuiltinType itemType) throws HyracksDataException {
- stringEval = args[0].createScalarEvaluator(context);
- this.tokenizer = tokenizer;
- this.listType = new AOrderedListType(itemType, null);
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- stringEval.evaluate(tuple, argPtr);
- tokenizer.reset(argPtr.getByteArray(), argPtr.getStartOffset(), argPtr.getLength());
- try {
- listBuilder.reset(listType);
- while (tokenizer.hasNext()) {
- tokenizer.next();
- listBuilder.addItem(tokenizer.getToken());
- }
- listBuilder.write(out, true);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java
deleted file mode 100644
index dd36671..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.GramTokensEvaluator;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8NGramTokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
-
-public class CountHashedGramTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new CountHashedGramTokensDescriptor();
- }
- };
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.COUNTHASHED_GRAM_TOKENS;
- }
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args)
- throws AlgebricksException {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- ITokenFactory tokenFactory = new HashedUTF8NGramTokenFactory();
- NGramUTF8StringBinaryTokenizer tokenizer =
- new NGramUTF8StringBinaryTokenizer(3, true, false, true, tokenFactory);
- return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java
deleted file mode 100644
index e12ba2e..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.WordTokensEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8WordTokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
-
-public class CountHashedWordTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new CountHashedWordTokensDescriptor();
- }
- };
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.COUNTHASHED_WORD_TOKENS;
- }
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- ITokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
- IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(false, true, tokenFactory);
- return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
- }
- };
- }
-
-}
[7/7] asterixdb git commit: [ASTERIXDB-2330][*DB][RT] Add
IFunctionRegistrant for dynamic function registration
Posted by mb...@apache.org.
[ASTERIXDB-2330][*DB][RT] Add IFunctionRegistrant for dynamic function registration
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Look for IFunctionRegistrant service instances at runtime, and use these to dynamically
register non-core functions with *DB
- Extract fuzzyjoin functions from core runtime
Change-Id: Ia88590280cbf476e08b905d9e1d62c68667a2569
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2480
Reviewed-by: abdullah alamoudi <ba...@gmail.com>
Tested-by: Michael Blow <mb...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/b8307794
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/b8307794
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/b8307794
Branch: refs/heads/master
Commit: b8307794c3483199a88266e8646bfbad79cc70f9
Parents: 38c41ac
Author: Michael Blow <mb...@apache.org>
Authored: Tue Mar 13 20:45:51 2018 -0700
Committer: Michael Blow <mb...@apache.org>
Committed: Tue Mar 13 21:07:36 2018 -0700
----------------------------------------------------------------------
asterixdb/asterix-app/pom.xml | 7 +
.../apache/asterix/runtime/NullMissingTest.java | 5 +-
asterixdb/asterix-fuzzyjoin/pom.xml | 64 +-
.../runtime/FuzzyJoinFunctionRegistrant.java | 72 ++
.../common/EditDistanceCheckEvaluator.java | 134 +++
.../common/EditDistanceContainsEvaluator.java | 59 +
.../common/EditDistanceEvaluator.java | 156 +++
.../evaluators/common/GramTokensEvaluator.java | 95 ++
.../common/SimilarityFiltersCache.java | 53 +
.../common/SimilarityJaccardCheckEvaluator.java | 166 +++
.../common/SimilarityJaccardEvaluator.java | 277 +++++
.../SimilarityJaccardPrefixEvaluator.java | 230 ++++
.../SimilarityJaccardSortedCheckEvaluator.java | 39 +
.../SimilarityJaccardSortedEvaluator.java | 40 +
.../evaluators/common/WordTokensEvaluator.java | 71 ++
.../CountHashedGramTokensDescriptor.java | 68 ++
.../CountHashedWordTokensDescriptor.java | 66 ++
.../functions/EditDistanceCheckDescriptor.java | 58 +
.../EditDistanceContainsDescriptor.java | 58 +
.../functions/EditDistanceDescriptor.java | 59 +
.../EditDistanceListIsFilterableDescriptor.java | 149 +++
...ditDistanceStringIsFilterableDescriptor.java | 65 ++
...EditDistanceStringIsFilterableEvaluator.java | 121 ++
.../functions/GramTokensDescriptor.java | 66 ++
.../functions/HashedGramTokensDescriptor.java | 66 ++
.../functions/HashedWordTokensDescriptor.java | 66 ++
.../functions/PrefixLenDescriptor.java | 141 +++
.../functions/PrefixLenJaccardDescriptor.java | 126 ++
.../functions/SimilarityDescriptor.java | 268 +++++
.../SimilarityJaccardCheckDescriptor.java | 58 +
.../functions/SimilarityJaccardDescriptor.java | 59 +
.../SimilarityJaccardPrefixCheckDescriptor.java | 110 ++
.../SimilarityJaccardPrefixDescriptor.java | 59 +
.../SimilarityJaccardSortedCheckDescriptor.java | 59 +
.../SimilarityJaccardSortedDescriptor.java | 60 +
.../functions/SpatialIntersectDescriptor.java | 1091 ++++++++++++++++++
.../functions/WordTokensDescriptor.java | 66 ++
...che.asterix.om.functions.IFunctionRegistrant | 20 +
.../om/functions/IFunctionCollection.java | 27 +
.../om/functions/IFunctionRegistrant.java | 25 +
...AbstractScalarFunctionDynamicDescriptor.java | 33 +
asterixdb/asterix-runtime/pom.xml | 6 -
...AbstractScalarFunctionDynamicDescriptor.java | 33 -
.../common/EditDistanceCheckEvaluator.java | 134 ---
.../common/EditDistanceContainsEvaluator.java | 59 -
.../common/EditDistanceEvaluator.java | 156 ---
.../evaluators/common/GramTokensEvaluator.java | 95 --
.../common/SimilarityFiltersCache.java | 53 -
.../common/SimilarityJaccardCheckEvaluator.java | 166 ---
.../common/SimilarityJaccardEvaluator.java | 277 -----
.../SimilarityJaccardPrefixEvaluator.java | 230 ----
.../SimilarityJaccardSortedCheckEvaluator.java | 39 -
.../SimilarityJaccardSortedEvaluator.java | 40 -
.../evaluators/common/WordTokensEvaluator.java | 71 --
.../CountHashedGramTokensDescriptor.java | 68 --
.../CountHashedWordTokensDescriptor.java | 66 --
.../functions/EditDistanceCheckDescriptor.java | 58 -
.../EditDistanceContainsDescriptor.java | 58 -
.../functions/EditDistanceDescriptor.java | 59 -
.../EditDistanceListIsFilterableDescriptor.java | 149 ---
...ditDistanceStringIsFilterableDescriptor.java | 65 --
...EditDistanceStringIsFilterableEvaluator.java | 121 --
.../functions/GramTokensDescriptor.java | 66 --
.../functions/HashedGramTokensDescriptor.java | 66 --
.../functions/HashedWordTokensDescriptor.java | 66 --
.../functions/PrefixLenDescriptor.java | 141 ---
.../functions/PrefixLenJaccardDescriptor.java | 126 --
.../functions/SimilarityDescriptor.java | 268 -----
.../SimilarityJaccardCheckDescriptor.java | 58 -
.../functions/SimilarityJaccardDescriptor.java | 59 -
.../SimilarityJaccardPrefixCheckDescriptor.java | 110 --
.../SimilarityJaccardPrefixDescriptor.java | 59 -
.../SimilarityJaccardSortedCheckDescriptor.java | 59 -
.../SimilarityJaccardSortedDescriptor.java | 60 -
.../functions/SpatialIntersectDescriptor.java | 1091 ------------------
.../functions/WordTokensDescriptor.java | 66 --
.../runtime/functions/FunctionCollection.java | 55 +-
asterixdb/asterix-server/pom.xml | 6 +
78 files changed, 4513 insertions(+), 4358 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-app/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index e75ad42..b51d499 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -152,6 +152,7 @@
</ignoredUsedUndeclaredDependencies>
<usedDependencies combine.children="append">
<usedDependency>org.apache.hadoop:hadoop-common</usedDependency>
+ <usedDependency>org.apache.asterix:asterix-fuzzyjoin</usedDependency>
</usedDependencies>
<ignoredUnusedDeclaredDependencies>
<ignoredUnusedDeclaredDependency>org.apache.asterix:asterix-external-data:zip:*</ignoredUnusedDeclaredDependency>
@@ -643,5 +644,11 @@
<artifactId>log4j-jul</artifactId>
<version>2.10.0</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-fuzzyjoin</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
index 3b33868..d96fec9 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java
@@ -55,8 +55,9 @@ public class NullMissingTest {
++testedFunctions;
}
}
- // 208 is the current number of functions with generated code.
- Assert.assertTrue(testedFunctions >= 208);
+ // 217 is the current number of functions with generated code.
+ Assert.assertTrue("expected >= 217 generated functions to be tested, but was " + testedFunctions,
+ testedFunctions >= 217);
}
private void testFunction(IFunctionDescriptorFactory funcFactory) throws Exception {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/pom.xml b/asterixdb/asterix-fuzzyjoin/pom.xml
index 236bcc7..8056cdf 100644
--- a/asterixdb/asterix-fuzzyjoin/pom.xml
+++ b/asterixdb/asterix-fuzzyjoin/pom.xml
@@ -23,7 +23,6 @@
<groupId>org.apache.asterix</groupId>
<version>0.9.4-SNAPSHOT</version>
</parent>
- <groupId>org.apache.asterix</groupId>
<artifactId>asterix-fuzzyjoin</artifactId>
<licenses>
@@ -42,6 +41,23 @@
<build>
<plugins>
<plugin>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-evaluator-generator-maven-plugin</artifactId>
+ <version>${project.version}</version>
+ <configuration>
+ <evaluatorPackagePrefix>org.apache.asterix.runtime.evaluators</evaluatorPackagePrefix>
+ </configuration>
+ <executions>
+ <execution>
+ <id>generate-evaluator</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>generate-evaluator</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
@@ -52,17 +68,6 @@
<phase>test-compile</phase>
</execution>
</executions>
- <configuration>
- <outputDirectory>${basedir}/target</outputDirectory>
- <includes>
- <include>**/*.class</include>
- <include>**/*.txt</include>
- <include>**/README*</include>
- <include>**/NOTICE*</include>
- <include>**/LICENSE*</include>
- <include>**/DEPENDENCIES*</include>
- </includes>
- </configuration>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
@@ -102,6 +107,41 @@
<groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-data-std</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-om</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-runtime</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>algebricks-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>algebricks-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-storage-am-lsm-invertedindex</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>algebricks-runtime</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-dataflow-common</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/FuzzyJoinFunctionRegistrant.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/FuzzyJoinFunctionRegistrant.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/FuzzyJoinFunctionRegistrant.java
new file mode 100644
index 0000000..35d8727
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/FuzzyJoinFunctionRegistrant.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime;
+
+import org.apache.asterix.om.functions.IFunctionCollection;
+import org.apache.asterix.om.functions.IFunctionRegistrant;
+import org.apache.asterix.runtime.evaluators.functions.CountHashedGramTokensDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.CountHashedWordTokensDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.EditDistanceCheckDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.EditDistanceContainsDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.EditDistanceDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.EditDistanceListIsFilterableDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.EditDistanceStringIsFilterableDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.GramTokensDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.HashedGramTokensDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.HashedWordTokensDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.PrefixLenJaccardDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardCheckDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardPrefixCheckDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardPrefixDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardSortedCheckDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.SimilarityJaccardSortedDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.SpatialIntersectDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.WordTokensDescriptor;
+
+public class FuzzyJoinFunctionRegistrant implements IFunctionRegistrant {
+ @Override
+ public void register(IFunctionCollection fc) {
+ // TODO: decide how should we deal these two weird functions as
+ // the number of arguments of the function depend on the first few arguments.
+ fc.add(SimilarityJaccardPrefixDescriptor.FACTORY);
+ fc.add(SimilarityJaccardPrefixCheckDescriptor.FACTORY);
+
+ // Spatial
+ fc.addGenerated(SpatialIntersectDescriptor.FACTORY);
+
+ // fuzzyjoin function
+ fc.addGenerated(PrefixLenJaccardDescriptor.FACTORY);
+ fc.addGenerated(WordTokensDescriptor.FACTORY);
+ fc.addGenerated(HashedWordTokensDescriptor.FACTORY);
+ fc.addGenerated(CountHashedWordTokensDescriptor.FACTORY);
+ fc.addGenerated(GramTokensDescriptor.FACTORY);
+ fc.addGenerated(HashedGramTokensDescriptor.FACTORY);
+ fc.addGenerated(CountHashedGramTokensDescriptor.FACTORY);
+ fc.addGenerated(EditDistanceDescriptor.FACTORY);
+ fc.addGenerated(EditDistanceCheckDescriptor.FACTORY);
+ fc.addGenerated(EditDistanceStringIsFilterableDescriptor.FACTORY);
+ fc.addGenerated(EditDistanceListIsFilterableDescriptor.FACTORY);
+ fc.addGenerated(EditDistanceContainsDescriptor.FACTORY);
+ fc.addGenerated(SimilarityJaccardDescriptor.FACTORY);
+ fc.addGenerated(SimilarityJaccardCheckDescriptor.FACTORY);
+ fc.addGenerated(SimilarityJaccardSortedDescriptor.FACTORY);
+ fc.addGenerated(SimilarityJaccardSortedCheckDescriptor.FACTORY);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java
new file mode 100644
index 0000000..dece292
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceCheckEvaluator.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.IOException;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class EditDistanceCheckEvaluator extends EditDistanceEvaluator {
+
+ protected final IScalarEvaluator edThreshEval;
+ protected int edThresh;
+ private final IPointable argPtrThreshold = new VoidPointable();
+ protected final OrderedListBuilder listBuilder;
+ protected ArrayBackedValueStorage listItemVal;
+ @SuppressWarnings("unchecked")
+ protected final ISerializerDeserializer<ABoolean> booleanSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
+
+ public EditDistanceCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ super(args, context);
+ edThreshEval = args[2].createScalarEvaluator(context);
+ listBuilder = new OrderedListBuilder();
+ listItemVal = new ArrayBackedValueStorage();
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ firstStringEval.evaluate(tuple, argPtr1);
+ firstTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
+ secondStringEval.evaluate(tuple, argPtr2);
+ secondTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
+ edThreshEval.evaluate(tuple, argPtrThreshold);
+
+ if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
+ result.set(resultStorage);
+ return;
+ }
+ try {
+ edThresh = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_CHECK.getName(), 2,
+ argPtrThreshold.getByteArray(), argPtrThreshold.getStartOffset());
+ if (edThresh < 0) {
+ throw new RuntimeDataException(ErrorCode.NEGATIVE_VALUE, BuiltinFunctions.EDIT_DISTANCE_CHECK.getName(),
+ 3, edThresh);
+ }
+ editDistance = computeResult(argPtr1, argPtr2, firstTypeTag);
+ writeResult(editDistance);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+
+ @Override
+ protected int computeResult(IPointable left, IPointable right, ATypeTag argType) throws HyracksDataException {
+ byte[] leftBytes = left.getByteArray();
+ int leftStartOffset = left.getStartOffset();
+ byte[] rightBytes = right.getByteArray();
+ int rightStartOffset = right.getStartOffset();
+ switch (argType) {
+ case STRING: {
+ return ed.UTF8StringEditDistance(leftBytes, leftStartOffset + typeIndicatorSize, rightBytes,
+ rightStartOffset + typeIndicatorSize, edThresh);
+ }
+
+ case ARRAY: {
+ firstOrdListIter.reset(leftBytes, leftStartOffset);
+ secondOrdListIter.reset(rightBytes, rightStartOffset);
+ return (int) ed.computeSimilarity(firstOrdListIter, secondOrdListIter, edThresh);
+ }
+
+ default: {
+ throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_CHECK, 0, argType.serialize(),
+ ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
+ }
+
+ }
+ }
+
+ @Override
+ protected void writeResult(int ed) throws IOException {
+ listBuilder.reset(new AOrderedListType(BuiltinType.ANY, "list"));
+ boolean matches = (ed < 0) ? false : true;
+ listItemVal.reset();
+ booleanSerde.serialize(matches ? ABoolean.TRUE : ABoolean.FALSE, listItemVal.getDataOutput());
+ listBuilder.addItem(listItemVal);
+
+ listItemVal.reset();
+ aInt64.setValue((matches) ? ed : Integer.MAX_VALUE);
+ int64Serde.serialize(aInt64, listItemVal.getDataOutput());
+ listBuilder.addItem(listItemVal);
+ listBuilder.write(out, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java
new file mode 100644
index 0000000..eaf3368
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceContainsEvaluator.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+
+public class EditDistanceContainsEvaluator extends EditDistanceCheckEvaluator {
+
+ public EditDistanceContainsEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ super(args, context);
+ }
+
+ @Override
+ protected int computeResult(IPointable left, IPointable right, ATypeTag argType) throws HyracksDataException {
+ byte[] leftBytes = left.getByteArray();
+ int leftStartOffset = left.getStartOffset();
+ byte[] rightBytes = right.getByteArray();
+ int rightStartOffset = right.getStartOffset();
+
+ switch (argType) {
+ case STRING: {
+ return ed.UTF8StringEditDistanceContains(leftBytes, leftStartOffset + typeIndicatorSize, rightBytes,
+ rightStartOffset + typeIndicatorSize, edThresh);
+ }
+ case ARRAY: {
+ firstOrdListIter.reset(leftBytes, leftStartOffset);
+ secondOrdListIter.reset(rightBytes, rightStartOffset);
+ return ed.getSimilarityContains(firstOrdListIter, secondOrdListIter, edThresh);
+ }
+ default: {
+ throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_CONTAINS, 0, argType.serialize(),
+ ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java
new file mode 100644
index 0000000..85fd334
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/EditDistanceEvaluator.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetricEditDistance;
+import org.apache.asterix.om.base.AInt64;
+import org.apache.asterix.om.base.AMutableInt64;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.runtime.exceptions.IncompatibleTypeException;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.asterix.runtime.exceptions.UnsupportedItemTypeException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class EditDistanceEvaluator implements IScalarEvaluator {
+
+ // assuming type indicator in serde format
+ protected final int typeIndicatorSize = 1;
+
+ protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ protected final DataOutput out = resultStorage.getDataOutput();
+ protected final IPointable argPtr1 = new VoidPointable();
+ protected final IPointable argPtr2 = new VoidPointable();
+ protected final IScalarEvaluator firstStringEval;
+ protected final IScalarEvaluator secondStringEval;
+ protected final SimilarityMetricEditDistance ed = new SimilarityMetricEditDistance();
+ protected final OrderedListIterator firstOrdListIter = new OrderedListIterator();
+ protected final OrderedListIterator secondOrdListIter = new OrderedListIterator();
+ protected int editDistance = 0;
+ protected final AMutableInt64 aInt64 = new AMutableInt64(-1);
+ @SuppressWarnings("unchecked")
+ protected final ISerializerDeserializer<AInt64> int64Serde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
+ protected ATypeTag itemTypeTag;
+
+ protected ATypeTag firstTypeTag;
+ protected ATypeTag secondTypeTag;
+
+ public EditDistanceEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ firstStringEval = args[0].createScalarEvaluator(context);
+ secondStringEval = args[1].createScalarEvaluator(context);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ firstStringEval.evaluate(tuple, argPtr1);
+ firstTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
+ secondStringEval.evaluate(tuple, argPtr2);
+ secondTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
+
+ if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
+ result.set(resultStorage);
+ return;
+ }
+
+ editDistance = computeResult(argPtr1, argPtr2, firstTypeTag);
+ try {
+ writeResult(editDistance);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+
+ protected int computeResult(IPointable left, IPointable right, ATypeTag argType) throws HyracksDataException {
+ byte[] leftBytes = left.getByteArray();
+ int leftStartOffset = left.getStartOffset();
+ byte[] rightBytes = right.getByteArray();
+ int rightStartOffset = right.getStartOffset();
+
+ switch (argType) {
+ case STRING: {
+ // Passes -1 as the simThresh to calculate the edit distance
+ // without applying any calculation optimizations.
+ return ed.getActualUTF8StringEditDistanceVal(leftBytes, leftStartOffset + typeIndicatorSize, rightBytes,
+ rightStartOffset + typeIndicatorSize, -1);
+ }
+ case ARRAY: {
+ firstOrdListIter.reset(leftBytes, leftStartOffset);
+ secondOrdListIter.reset(rightBytes, rightStartOffset);
+ return (int) ed.computeSimilarity(firstOrdListIter, secondOrdListIter);
+ }
+ default: {
+ throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE, 0, argType.serialize(),
+ ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
+ }
+
+ }
+ }
+
+ protected boolean checkArgTypes(ATypeTag typeTag1, ATypeTag typeTag2) throws HyracksDataException {
+ if (typeTag1 != typeTag2) {
+ throw new IncompatibleTypeException(BuiltinFunctions.EDIT_DISTANCE, typeTag1.serialize(),
+ typeTag2.serialize());
+ }
+
+ // Since they are equal, check one tag is enough.
+ if (typeTag1 != ATypeTag.STRING && typeTag1 != ATypeTag.ARRAY) { // could be an list
+ throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE, 0, typeTag1.serialize(),
+ ATypeTag.SERIALIZED_STRING_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
+ }
+
+ if (typeTag1 == ATypeTag.ARRAY) {
+ itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER
+ .deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset() + 1]);
+ if (itemTypeTag == ATypeTag.ANY) {
+ throw new UnsupportedItemTypeException(BuiltinFunctions.EDIT_DISTANCE, itemTypeTag.serialize());
+ }
+ itemTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER
+ .deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset() + 1]);
+ if (itemTypeTag == ATypeTag.ANY) {
+ throw new UnsupportedItemTypeException(BuiltinFunctions.EDIT_DISTANCE, itemTypeTag.serialize());
+ }
+ }
+ return true;
+ }
+
+ protected void writeResult(int ed) throws IOException {
+ aInt64.setValue(ed);
+ int64Serde.serialize(aInt64, out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java
new file mode 100644
index 0000000..ef727c9
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/GramTokensEvaluator.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.dataflow.data.nontagged.serde.ABooleanSerializerDeserializer;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
+
+public class GramTokensEvaluator implements IScalarEvaluator {
+
+ // assuming type indicator in serde format
+ private final int typeIndicatorSize = 1;
+
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable stringArg = new VoidPointable();
+ private final IPointable gramLengthArg = new VoidPointable();
+ private final IPointable prePostArg = new VoidPointable();
+ private final IScalarEvaluator stringEval;
+ private final IScalarEvaluator gramLengthEval;
+ private final IScalarEvaluator prePostEval;
+
+ private final NGramUTF8StringBinaryTokenizer tokenizer;
+ private final OrderedListBuilder listBuilder = new OrderedListBuilder();
+ private final AOrderedListType listType;
+
+ public GramTokensEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context, IBinaryTokenizer tokenizer,
+ BuiltinType itemType) throws HyracksDataException {
+ stringEval = args[0].createScalarEvaluator(context);
+ gramLengthEval = args[1].createScalarEvaluator(context);
+ prePostEval = args[2].createScalarEvaluator(context);
+ this.tokenizer = (NGramUTF8StringBinaryTokenizer) tokenizer;
+ this.listType = new AOrderedListType(itemType, null);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ stringEval.evaluate(tuple, stringArg);
+ gramLengthEval.evaluate(tuple, gramLengthArg);
+ prePostEval.evaluate(tuple, prePostArg);
+
+ int gramLength = ATypeHierarchy.getIntegerValue(BuiltinFunctions.GRAM_TOKENS.getName(), 1,
+ gramLengthArg.getByteArray(), gramLengthArg.getStartOffset());
+ tokenizer.setGramlength(gramLength);
+ boolean prePost = ABooleanSerializerDeserializer.getBoolean(prePostArg.getByteArray(),
+ prePostArg.getStartOffset() + typeIndicatorSize);
+ tokenizer.setPrePost(prePost);
+ tokenizer.reset(stringArg.getByteArray(), stringArg.getStartOffset(), stringArg.getLength());
+
+ try {
+ listBuilder.reset(listType);
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
+ listBuilder.addItem(tokenizer.getToken());
+ }
+ listBuilder.write(out, true);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java
new file mode 100644
index 0000000..4f5fb69
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityFiltersCache.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.DataInputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityFilters;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityFiltersFactory;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+
+public class SimilarityFiltersCache {
+ private final UTF8StringSerializerDeserializer utf8SerDer = new UTF8StringSerializerDeserializer();
+
+ private final ByteBufferInputStream bbis = new ByteBufferInputStream();
+ private final DataInputStream dis = new DataInputStream(bbis);
+
+ private float similarityThresholdCached = 0;
+ private byte[] similarityNameBytesCached = null;
+ private SimilarityFilters similarityFiltersCached = null;
+
+ public SimilarityFilters get(float similarityThreshold, byte[] similarityNameBytes, int startOffset, int len)
+ throws HyracksDataException {
+ if (similarityThreshold != similarityThresholdCached || similarityNameBytesCached == null
+ || !Arrays.equals(similarityNameBytes, similarityNameBytesCached)) {
+ bbis.setByteBuffer(ByteBuffer.wrap(similarityNameBytes), startOffset + 1);
+ String similarityName = utf8SerDer.deserialize(dis);
+ similarityNameBytesCached = Arrays.copyOfRange(similarityNameBytes, startOffset, len);
+ similarityFiltersCached =
+ SimilarityFiltersFactory.getSimilarityFilters(similarityName, similarityThreshold);
+ }
+ return similarityFiltersCached;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
new file mode 100644
index 0000000..60b5592
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardCheckEvaluator.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.IOException;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.data.std.util.BinaryEntry;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class SimilarityJaccardCheckEvaluator extends SimilarityJaccardEvaluator {
+
+ protected final IScalarEvaluator jaccThreshEval;
+ protected float jaccThresh = -1f;
+ protected IPointable jaccThreshPointable = new VoidPointable();
+
+ protected OrderedListBuilder listBuilder;
+ protected ArrayBackedValueStorage inputVal;
+ @SuppressWarnings("unchecked")
+ protected final ISerializerDeserializer<ABoolean> booleanSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
+ protected final AOrderedListType listType = new AOrderedListType(BuiltinType.ANY, "list");
+
+ public SimilarityJaccardCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ super(args, context);
+ jaccThreshEval = args[2].createScalarEvaluator(context);
+ listBuilder = new OrderedListBuilder();
+ inputVal = new ArrayBackedValueStorage();
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+
+ firstOrdListEval.evaluate(tuple, argPtr1);
+ secondOrdListEval.evaluate(tuple, argPtr2);
+ jaccThreshEval.evaluate(tuple, jaccThreshPointable);
+
+ firstTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
+ secondTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
+
+ firstItemTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset() + 1]);
+ secondItemTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset() + 1]);
+
+ jaccThresh = AFloatSerializerDeserializer.getFloat(jaccThreshPointable.getByteArray(),
+ jaccThreshPointable.getStartOffset() + TYPE_INDICATOR_SIZE);
+
+ if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
+ result.set(resultStorage);
+ return;
+ }
+ if (prepareLists(argPtr1, argPtr2)) {
+ jaccSim = computeResult();
+ } else {
+ jaccSim = 0.0f;
+ }
+ try {
+ writeResult(jaccSim);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+
+ @Override
+ protected int probeHashMap(AbstractAsterixListIterator probeIter, int buildListSize, int probeListSize)
+ throws HyracksDataException {
+ // Apply length filter.
+ int lengthLowerBound = (int) Math.ceil(jaccThresh * probeListSize);
+ if ((lengthLowerBound > buildListSize)
+ || (buildListSize > (int) Math.floor(1.0f / jaccThresh * probeListSize))) {
+ return -1;
+ }
+ // Probe phase: Probe items from second list, and compute intersection size.
+ int intersectionSize = 0;
+ int probeListCount = 0;
+ int minUnionSize = buildListSize;
+ while (probeIter.hasNext()) {
+ probeListCount++;
+ byte[] buf = probeIter.getData();
+ int off = probeIter.getPos();
+ int len = probeIter.getItemLen();
+ keyEntry.set(buf, off, len);
+ BinaryEntry entry = hashMap.get(keyEntry);
+ if (entry != null) {
+ // Increment second value.
+ int firstValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset());
+ // Irrelevant for the intersection size.
+ if (firstValInt == 0) {
+ continue;
+ }
+ int secondValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset() + 4);
+ // Subtract old min value.
+ intersectionSize -= (firstValInt < secondValInt) ? firstValInt : secondValInt;
+ secondValInt++;
+ // Add new min value.
+ intersectionSize += (firstValInt < secondValInt) ? firstValInt : secondValInt;
+ IntegerPointable.setInteger(entry.getBuf(), entry.getOffset() + 4, secondValInt);
+ } else {
+ // Could not find element in other set. Increase min union size by 1.
+ minUnionSize++;
+ // Check whether jaccThresh can still be satisfied if there was a mismatch.
+ int maxIntersectionSize = Math.min(buildListSize, intersectionSize + (probeListSize - probeListCount));
+ int lowerBound = (int) Math.floor(jaccThresh * minUnionSize);
+ if (maxIntersectionSize < lowerBound) {
+ // Cannot satisfy jaccThresh.
+ return -1;
+ }
+ }
+ probeIter.next();
+ }
+ return intersectionSize;
+ }
+
+ @Override
+ protected void writeResult(float jacc) throws IOException {
+ listBuilder.reset(listType);
+ boolean matches = (jacc < jaccThresh) ? false : true;
+ inputVal.reset();
+ booleanSerde.serialize(matches ? ABoolean.TRUE : ABoolean.FALSE, inputVal.getDataOutput());
+ listBuilder.addItem(inputVal);
+
+ inputVal.reset();
+ aFloat.setValue((matches) ? jacc : 0.0f);
+ floatSerde.serialize(aFloat, inputVal.getDataOutput());
+ listBuilder.addItem(inputVal);
+
+ listBuilder.write(out, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
new file mode 100644
index 0000000..1e5ad3c
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardEvaluator.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.asterix.dataflow.data.nontagged.comparators.ListItemBinaryComparatorFactory;
+import org.apache.asterix.dataflow.data.nontagged.hash.ListItemBinaryHashFunctionFactory;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.AFloat;
+import org.apache.asterix.om.base.AMutableFloat;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.runtime.evaluators.functions.BinaryHashMap;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.data.std.util.BinaryEntry;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class SimilarityJaccardEvaluator implements IScalarEvaluator {
+
+ // Parameters for hash table.
+ protected static final int MIN_TABLE_SIZE = 100;
+ protected static final int TABLE_FRAME_SIZE = 32768;
+
+ // Assuming type indicator in serde format.
+ protected static final int TYPE_INDICATOR_SIZE = 1;
+
+ protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ protected final DataOutput out = resultStorage.getDataOutput();
+ protected final IPointable argPtr1 = new VoidPointable();
+ protected final IPointable argPtr2 = new VoidPointable();
+ protected final IScalarEvaluator firstOrdListEval;
+ protected final IScalarEvaluator secondOrdListEval;
+
+ protected final OrderedListIterator fstOrdListIter = new OrderedListIterator();
+ protected final OrderedListIterator sndOrdListIter = new OrderedListIterator();
+ protected final UnorderedListIterator fstUnordListIter = new UnorderedListIterator();
+ protected final UnorderedListIterator sndUnordListIter = new UnorderedListIterator();
+
+ protected AbstractAsterixListIterator firstListIter;
+ protected AbstractAsterixListIterator secondListIter;
+
+ protected final AMutableFloat aFloat = new AMutableFloat(0);
+ @SuppressWarnings("unchecked")
+ protected final ISerializerDeserializer<AFloat> floatSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AFLOAT);
+
+ protected ATypeTag firstTypeTag;
+ protected ATypeTag secondTypeTag;
+ protected float jaccSim = 0.0f;
+ protected ATypeTag firstItemTypeTag;
+ protected ATypeTag secondItemTypeTag;
+
+ protected BinaryHashMap hashMap;
+ protected BinaryEntry keyEntry = new BinaryEntry();
+ protected BinaryEntry valEntry = new BinaryEntry();
+
+ // Ignore case for strings. Defaults to true.
+ protected final boolean ignoreCase = true;
+
+ protected int hashTableSize = MIN_TABLE_SIZE;
+
+ public SimilarityJaccardEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ firstOrdListEval = args[0].createScalarEvaluator(context);
+ secondOrdListEval = args[1].createScalarEvaluator(context);
+ byte[] emptyValBuf = new byte[8];
+ Arrays.fill(emptyValBuf, (byte) 0);
+ valEntry.set(emptyValBuf, 0, 8);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+
+ firstOrdListEval.evaluate(tuple, argPtr1);
+ secondOrdListEval.evaluate(tuple, argPtr2);
+
+ firstTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset()]);
+ secondTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset()]);
+
+ firstItemTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr1.getByteArray()[argPtr1.getStartOffset() + 1]);
+ secondItemTypeTag =
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(argPtr2.getByteArray()[argPtr2.getStartOffset() + 1]);
+
+ if (!checkArgTypes(firstTypeTag, secondTypeTag)) {
+ result.set(resultStorage);
+ return;
+ }
+ if (prepareLists(argPtr1, argPtr2)) {
+ jaccSim = computeResult();
+ } else {
+ jaccSim = 0.0f;
+ }
+ try {
+ writeResult(jaccSim);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+
+ protected boolean prepareLists(IPointable left, IPointable right) throws HyracksDataException {
+ firstListIter.reset(left.getByteArray(), left.getStartOffset());
+ secondListIter.reset(right.getByteArray(), right.getStartOffset());
+ // Check for special case where one of the lists is empty, since list
+ // types won't match.
+ if (firstListIter.size() == 0 || secondListIter.size() == 0) {
+ return false;
+ }
+
+ // Set the size of the table dynamically
+ hashTableSize = Math.max(Math.max(firstListIter.size(), secondListIter.size()), MIN_TABLE_SIZE);
+
+ // TODO: Check item types are compatible.
+ return true;
+ }
+
+ protected float computeResult() throws HyracksDataException {
+ // We will subtract the intersection size later to get the real union size.
+ int firstListSize = firstListIter.size();
+ int secondListSize = secondListIter.size();
+ int unionSize = firstListSize + secondListSize;
+ // Choose smaller list as build, and larger one as probe.
+ AbstractAsterixListIterator buildList = (firstListSize < secondListSize) ? firstListIter : secondListIter;
+ AbstractAsterixListIterator probeList = (buildList == firstListIter) ? secondListIter : firstListIter;
+ int buildListSize = (buildList == firstListIter) ? firstListSize : secondListSize;
+ int probeListSize = (probeList == firstListIter) ? firstListSize : secondListSize;
+ ATypeTag buildItemTypeTag = (buildList == firstListIter) ? firstItemTypeTag : secondItemTypeTag;
+ ATypeTag probeItemTypeTag = (probeList == firstListIter) ? firstItemTypeTag : secondItemTypeTag;
+
+ setHashMap(buildItemTypeTag, probeItemTypeTag);
+ buildHashMap(buildList);
+ int intersectionSize = probeHashMap(probeList, buildListSize, probeListSize);
+ // Special indicator for the "check" version of jaccard.
+ if (intersectionSize < 0) {
+ return -1;
+ }
+ unionSize -= intersectionSize;
+ return (float) intersectionSize / (float) unionSize;
+ }
+
+ protected void buildHashMap(AbstractAsterixListIterator buildIter) throws HyracksDataException {
+ // Build phase: Add items into hash map, starting with first list.
+ // Value in map is a pair of integers. Set first integer to 1.
+ IntegerPointable.setInteger(valEntry.getBuf(), 0, 1);
+ while (buildIter.hasNext()) {
+ byte[] buf = buildIter.getData();
+ int off = buildIter.getPos();
+ int len = buildIter.getItemLen();
+ keyEntry.set(buf, off, len);
+ BinaryEntry entry = hashMap.put(keyEntry, valEntry);
+ if (entry != null) {
+ // Increment value.
+ int firstValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset());
+ IntegerPointable.setInteger(entry.getBuf(), entry.getOffset(), firstValInt + 1);
+ }
+ buildIter.next();
+ }
+ }
+
+ protected int probeHashMap(AbstractAsterixListIterator probeIter, int buildListSize, int probeListSize)
+ throws HyracksDataException {
+ // Probe phase: Probe items from second list, and compute intersection size.
+ int intersectionSize = 0;
+ while (probeIter.hasNext()) {
+ byte[] buf = probeIter.getData();
+ int off = probeIter.getPos();
+ int len = probeIter.getItemLen();
+ keyEntry.set(buf, off, len);
+ BinaryEntry entry = hashMap.get(keyEntry);
+ if (entry != null) {
+ // Increment second value.
+ int firstValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset());
+ // Irrelevant for the intersection size.
+ if (firstValInt == 0) {
+ continue;
+ }
+ int secondValInt = IntegerPointable.getInteger(entry.getBuf(), entry.getOffset() + 4);
+ // Subtract old min value.
+ intersectionSize -= (firstValInt < secondValInt) ? firstValInt : secondValInt;
+ secondValInt++;
+ // Add new min value.
+ intersectionSize += (firstValInt < secondValInt) ? firstValInt : secondValInt;
+ IntegerPointable.setInteger(entry.getBuf(), entry.getOffset() + 4, secondValInt);
+ }
+ probeIter.next();
+ }
+ return intersectionSize;
+ }
+
+ protected void setHashMap(ATypeTag buildItemTypeTag, ATypeTag probeItemTypeTag) {
+ if (hashMap != null) {
+ hashMap.clear();
+ return;
+ }
+
+ IBinaryHashFunction putHashFunc =
+ ListItemBinaryHashFunctionFactory.INSTANCE.createBinaryHashFunction(buildItemTypeTag, ignoreCase);
+ IBinaryHashFunction getHashFunc =
+ ListItemBinaryHashFunctionFactory.INSTANCE.createBinaryHashFunction(probeItemTypeTag, ignoreCase);
+ IBinaryComparator cmp = ListItemBinaryComparatorFactory.INSTANCE.createBinaryComparator(buildItemTypeTag,
+ probeItemTypeTag, ignoreCase);
+ hashMap = new BinaryHashMap(hashTableSize, TABLE_FRAME_SIZE, putHashFunc, getHashFunc, cmp);
+ }
+
+ protected boolean checkArgTypes(ATypeTag typeTag1, ATypeTag typeTag2) throws HyracksDataException {
+ switch (typeTag1) {
+ case ARRAY: {
+ firstListIter = fstOrdListIter;
+ break;
+ }
+ case MULTISET: {
+ firstListIter = fstUnordListIter;
+ break;
+ }
+ default: {
+ throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 0, typeTag1.serialize(),
+ ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ }
+ }
+ switch (typeTag2) {
+ case ARRAY: {
+ secondListIter = sndOrdListIter;
+ break;
+ }
+ case MULTISET: {
+ secondListIter = sndUnordListIter;
+ break;
+ }
+ default: {
+ throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 1, typeTag2.serialize(),
+ ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ }
+ }
+ return true;
+ }
+
+ protected void writeResult(float jacc) throws IOException {
+ aFloat.setValue(jacc);
+ floatSerde.serialize(aFloat, out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java
new file mode 100644
index 0000000..b70c6ad
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardPrefixEvaluator.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AUnorderedListSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.fuzzyjoin.IntArray;
+import org.apache.asterix.fuzzyjoin.similarity.PartialIntersect;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityFiltersJaccard;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetric;
+import org.apache.asterix.om.base.AFloat;
+import org.apache.asterix.om.base.AMutableFloat;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class SimilarityJaccardPrefixEvaluator implements IScalarEvaluator {
+ // assuming type indicator in serde format
+ protected final int typeIndicatorSize = 1;
+
+ protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ protected final DataOutput out = resultStorage.getDataOutput();
+ protected final IPointable inputVal = new VoidPointable();
+ protected final IScalarEvaluator evalLen1;
+ protected final IScalarEvaluator evalTokens1;
+ protected final IScalarEvaluator evalLen2;
+ protected final IScalarEvaluator evalTokens2;
+ protected final IScalarEvaluator evalTokenPrefix;
+ protected final IScalarEvaluator evalThreshold;
+
+ protected float similarityThresholdCache;
+ protected SimilarityFiltersJaccard similarityFilters;
+ protected final IntArray tokens1 = new IntArray();
+ protected final IntArray tokens2 = new IntArray();
+ protected final PartialIntersect parInter = new PartialIntersect();
+
+ protected float sim = 0.0f;
+
+ // result
+ protected final AMutableFloat res = new AMutableFloat(0);
+ @SuppressWarnings("unchecked")
+ protected final ISerializerDeserializer<AFloat> reusSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AFLOAT);
+
+ public SimilarityJaccardPrefixEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ evalLen1 = args[0].createScalarEvaluator(context);
+ evalTokens1 = args[1].createScalarEvaluator(context);
+ evalLen2 = args[2].createScalarEvaluator(context);
+ evalTokens2 = args[3].createScalarEvaluator(context);
+ evalTokenPrefix = args[4].createScalarEvaluator(context);
+ evalThreshold = args[5].createScalarEvaluator(context);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ // similarity threshold
+ sim = 0;
+ evalThreshold.evaluate(tuple, inputVal);
+ float similarityThreshold =
+ AFloatSerializerDeserializer.getFloat(inputVal.getByteArray(), inputVal.getStartOffset() + 1);
+
+ if (similarityThreshold != similarityThresholdCache || similarityFilters == null) {
+ similarityFilters = new SimilarityFiltersJaccard(similarityThreshold);
+ similarityThresholdCache = similarityThreshold;
+ }
+
+ evalLen1.evaluate(tuple, inputVal);
+ int length1 = ATypeHierarchy.getIntegerValue(BuiltinFunctions.SIMILARITY_JACCARD.getName(), 0,
+ inputVal.getByteArray(), inputVal.getStartOffset());
+ evalLen2.evaluate(tuple, inputVal);
+ int length2 = ATypeHierarchy.getIntegerValue(BuiltinFunctions.SIMILARITY_JACCARD.getName(), 2,
+ inputVal.getByteArray(), inputVal.getStartOffset());
+
+ //
+ // -- - length filter - --
+ //
+ if (similarityFilters.passLengthFilter(length1, length2)) {
+
+ // -- - tokens1 - --
+ int i;
+ tokens1.reset();
+ evalTokens1.evaluate(tuple, inputVal);
+
+ byte[] serList = inputVal.getByteArray();
+ int startOffset = inputVal.getStartOffset();
+ if (serList[startOffset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
+ && serList[startOffset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
+ throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 1, serList[startOffset],
+ ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ }
+
+ int lengthTokens1;
+ if (serList[startOffset] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ lengthTokens1 =
+ AOrderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
+ // read tokens
+ for (i = 0; i < lengthTokens1; i++) {
+ int itemOffset = AOrderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
+ int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
+ BuiltinFunctions.SIMILARITY_JACCARD.getName(), 1, serList, itemOffset, startOffset + 1);
+ tokens1.add(token);
+ }
+ } else {
+ lengthTokens1 =
+ AUnorderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
+ // read tokens
+ for (i = 0; i < lengthTokens1; i++) {
+ int itemOffset = AUnorderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
+ int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
+ BuiltinFunctions.SIMILARITY_JACCARD.getName(), 1, serList, itemOffset, startOffset + 1);
+ tokens1.add(token);
+ }
+ }
+ // pad tokens
+ for (; i < length1; i++) {
+ tokens1.add(Integer.MAX_VALUE);
+ }
+
+ // -- - tokens2 - --
+ tokens2.reset();
+ evalTokens2.evaluate(tuple, inputVal);
+
+ serList = inputVal.getByteArray();
+ startOffset = inputVal.getStartOffset();
+ if (serList[startOffset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
+ && serList[startOffset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
+ throw new TypeMismatchException(BuiltinFunctions.SIMILARITY_JACCARD, 3, serList[startOffset],
+ ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ }
+
+ int lengthTokens2;
+ if (serList[startOffset] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ lengthTokens2 =
+ AOrderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
+ // read tokens
+ for (i = 0; i < lengthTokens2; i++) {
+ int itemOffset = AOrderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
+ int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
+ BuiltinFunctions.SIMILARITY_JACCARD.getName(), 3, serList, itemOffset, startOffset + 1);
+ tokens2.add(token);
+ }
+ } else {
+ lengthTokens2 =
+ AUnorderedListSerializerDeserializer.getNumberOfItems(inputVal.getByteArray(), startOffset);
+ // read tokens
+ for (i = 0; i < lengthTokens2; i++) {
+ int itemOffset = AUnorderedListSerializerDeserializer.getItemOffset(serList, startOffset, i);
+ int token = ATypeHierarchy.getIntegerValueWithDifferentTypeTagPosition(
+ BuiltinFunctions.SIMILARITY_JACCARD.getName(), 3, serList, itemOffset, startOffset + 1);
+ tokens2.add(token);
+ }
+ }
+ // pad tokens
+ for (; i < length2; i++) {
+ tokens2.add(Integer.MAX_VALUE);
+ }
+
+ // -- - token prefix - --
+ evalTokenPrefix.evaluate(tuple, inputVal);
+ int tokenPrefix = ATypeHierarchy.getIntegerValue(BuiltinFunctions.SIMILARITY_JACCARD.getName(), 4,
+ inputVal.getByteArray(), inputVal.getStartOffset());
+
+ //
+ // -- - position filter - --
+ //
+ SimilarityMetric.getPartialIntersectSize(tokens1.get(), 0, tokens1.length(), tokens2.get(), 0,
+ tokens2.length(), tokenPrefix, parInter);
+ if (similarityFilters.passPositionFilter(parInter.intersectSize, parInter.posXStop, length1,
+ parInter.posYStop, length2)) {
+
+ //
+ // -- - suffix filter - --
+ //
+ if (similarityFilters.passSuffixFilter(tokens1.get(), 0, tokens1.length(), parInter.posXStart,
+ tokens2.get(), 0, tokens2.length(), parInter.posYStart)) {
+
+ sim = similarityFilters.passSimilarityFilter(tokens1.get(), 0, tokens1.length(),
+ parInter.posXStop + 1, tokens2.get(), 0, tokens2.length(), parInter.posYStop + 1,
+ parInter.intersectSize);
+ }
+ }
+ }
+
+ try {
+ writeResult();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+
+ public void writeResult() throws IOException {
+ res.setValue(sim);
+ reusSerde.serialize(res, out);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java
new file mode 100644
index 0000000..0decd9e
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedCheckEvaluator.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetricJaccard;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class SimilarityJaccardSortedCheckEvaluator extends SimilarityJaccardCheckEvaluator {
+
+ protected final SimilarityMetricJaccard jaccard = new SimilarityMetricJaccard();
+
+ public SimilarityJaccardSortedCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ super(args, context);
+ }
+
+ @Override
+ protected float computeResult() throws HyracksDataException {
+ return jaccard.computeSimilarity(firstListIter, secondListIter, jaccThresh);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java
new file mode 100644
index 0000000..1cd32c8
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/SimilarityJaccardSortedEvaluator.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetricJaccard;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+// Assumes that both arguments are sorted by the same ordering.
+public class SimilarityJaccardSortedEvaluator extends SimilarityJaccardEvaluator {
+
+ protected final SimilarityMetricJaccard jaccard = new SimilarityMetricJaccard();
+
+ public SimilarityJaccardSortedEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ super(args, context);
+ }
+
+ @Override
+ protected float computeResult() throws HyracksDataException {
+ return jaccard.computeSimilarity(firstListIter, secondListIter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java
new file mode 100644
index 0000000..e51d5cf
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/common/WordTokensEvaluator.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.common;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
+
+public class WordTokensEvaluator implements IScalarEvaluator {
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable argPtr = new VoidPointable();
+ private final IScalarEvaluator stringEval;
+
+ private final IBinaryTokenizer tokenizer;
+ private final OrderedListBuilder listBuilder = new OrderedListBuilder();
+ private final AOrderedListType listType;
+
+ public WordTokensEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context, IBinaryTokenizer tokenizer,
+ BuiltinType itemType) throws HyracksDataException {
+ stringEval = args[0].createScalarEvaluator(context);
+ this.tokenizer = tokenizer;
+ this.listType = new AOrderedListType(itemType, null);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ stringEval.evaluate(tuple, argPtr);
+ tokenizer.reset(argPtr.getByteArray(), argPtr.getStartOffset(), argPtr.getLength());
+ try {
+ listBuilder.reset(listType);
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
+ listBuilder.addItem(tokenizer.getToken());
+ }
+ listBuilder.write(out, true);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+}
[3/7] asterixdb git commit: [ASTERIXDB-2330][*DB][RT] Add
IFunctionRegistrant for dynamic function registration
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java
deleted file mode 100644
index e4b40b1..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.EditDistanceCheckEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class EditDistanceCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new EditDistanceCheckDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new EditDistanceCheckEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.EDIT_DISTANCE_CHECK;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java
deleted file mode 100644
index 4c7c257..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.EditDistanceContainsEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class EditDistanceContainsDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new EditDistanceContainsDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new EditDistanceContainsEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.EDIT_DISTANCE_CONTAINS;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java
deleted file mode 100644
index 8c6c9ed..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.EditDistanceEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class EditDistanceDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new EditDistanceDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new EditDistanceEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.EDIT_DISTANCE;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java
deleted file mode 100644
index 0f4ebee..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AUnorderedListSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-/**
- * Checks whether a list with an edit distance threshold can be filtered with a lower bounding on the number
- * of common list elements. This function returns 'true' if the lower bound on the number of common elements
- * is positive, 'false' otherwise. For example, this function is used during an indexed nested-loop join based
- * on edit distance. We partition the tuples of the probing dataset into those that are filterable and those
- * that are not. Those that are filterable are forwarded to the index. The others are are fed into a (non
- * indexed) nested-loop join.
- */
-public class EditDistanceListIsFilterableDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
-
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new EditDistanceListIsFilterableDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new EditDistanceListIsFilterableEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE;
- }
-
- private static class EditDistanceListIsFilterableEvaluator implements IScalarEvaluator {
-
- protected final IPointable listPtr = new VoidPointable();
- protected final IPointable edThreshPtr = new VoidPointable();
- protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- protected final DataOutput output = resultStorage.getDataOutput();
-
- protected final IScalarEvaluator listEval;
- protected final IScalarEvaluator edThreshEval;
-
- @SuppressWarnings("unchecked")
- private final ISerializerDeserializer<ABoolean> booleanSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
-
- public EditDistanceListIsFilterableEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- listEval = args[0].createScalarEvaluator(context);
- edThreshEval = args[1].createScalarEvaluator(context);
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
-
- listEval.evaluate(tuple, listPtr);
- edThreshEval.evaluate(tuple, edThreshPtr);
-
- // Check type and compute string length.
- byte[] bytes = listPtr.getByteArray();
- int offset = listPtr.getStartOffset();
-
- ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[offset]);
- long listLen;
- switch (typeTag) {
- case MULTISET:
- listLen = AUnorderedListSerializerDeserializer.getNumberOfItems(bytes, offset);
- break;
- case ARRAY:
- listLen = AOrderedListSerializerDeserializer.getNumberOfItems(bytes, offset);
- break;
- default:
- throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE, 0,
- ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
- }
-
- // Check type and extract edit-distance threshold.
- bytes = edThreshPtr.getByteArray();
- offset = edThreshPtr.getStartOffset();
- long edThresh = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE.getName(),
- 1, bytes, offset);
-
- // Compute result.
- long lowerBound = listLen - edThresh;
- try {
- if (lowerBound <= 0) {
- booleanSerde.serialize(ABoolean.FALSE, output);
- } else {
- booleanSerde.serialize(ABoolean.TRUE, output);
- }
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java
deleted file mode 100644
index ddb18fc..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * Checks whether a string with an edit distance threshold can be filtered with a lower bounding
- * on number of common grams. This function returns 'true' if the lower bound on the number of
- * common grams is positive, 'false' otherwise. For example, this function is used during an indexed
- * nested-loop join based on edit distance. We partition the tuples of the probing dataset into those
- * that are filterable and those that are not. Those that are filterable are forwarded to the index.
- * The others are fed into a (non indexed) nested-loop join.
- */
-public class EditDistanceStringIsFilterableDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new EditDistanceStringIsFilterableDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new EditDistanceStringIsFilterableEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.EDIT_DISTANCE_STRING_IS_FILTERABLE;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java
deleted file mode 100644
index 0509f51..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.ABooleanSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class EditDistanceStringIsFilterableEvaluator implements IScalarEvaluator {
-
- protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- protected final DataOutput output = resultStorage.getDataOutput();
- protected final IPointable stringPtr = new VoidPointable();
- protected final IPointable edThreshPtr = new VoidPointable();
- protected final IPointable gramLenPtr = new VoidPointable();
- protected final IPointable usePrePostPtr = new VoidPointable();
-
- protected final IScalarEvaluator stringEval;
- protected final IScalarEvaluator edThreshEval;
- protected final IScalarEvaluator gramLenEval;
- protected final IScalarEvaluator usePrePostEval;
-
- @SuppressWarnings("unchecked")
- private final ISerializerDeserializer<ABoolean> booleanSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
-
- private final UTF8StringPointable utf8Ptr = new UTF8StringPointable();
-
- public EditDistanceStringIsFilterableEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- stringEval = args[0].createScalarEvaluator(context);
- edThreshEval = args[1].createScalarEvaluator(context);
- gramLenEval = args[2].createScalarEvaluator(context);
- usePrePostEval = args[3].createScalarEvaluator(context);
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
-
- stringEval.evaluate(tuple, stringPtr);
- edThreshEval.evaluate(tuple, edThreshPtr);
- gramLenEval.evaluate(tuple, gramLenPtr);
- usePrePostEval.evaluate(tuple, usePrePostPtr);
-
- // Check type and compute string length.
- byte typeTag = stringPtr.getByteArray()[stringPtr.getStartOffset()];
- if (typeTag != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
- throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_STRING_IS_FILTERABLE, 0, typeTag,
- ATypeTag.SERIALIZED_STRING_TYPE_TAG);
- }
- utf8Ptr.set(stringPtr.getByteArray(), stringPtr.getStartOffset() + 1, stringPtr.getLength());
- int strLen = utf8Ptr.getStringLength();
-
- // Check type and extract edit-distance threshold.
- long edThresh = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE.getName(), 1,
- edThreshPtr.getByteArray(), edThreshPtr.getStartOffset());
-
- // Check type and extract gram length.
- long gramLen = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE.getName(), 2,
- gramLenPtr.getByteArray(), gramLenPtr.getStartOffset());
-
- // Check type and extract usePrePost flag.
- typeTag = usePrePostPtr.getByteArray()[usePrePostPtr.getStartOffset()];
- if (typeTag != ATypeTag.SERIALIZED_BOOLEAN_TYPE_TAG) {
- throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_STRING_IS_FILTERABLE, 3, typeTag,
- ATypeTag.SERIALIZED_BOOLEAN_TYPE_TAG);
- }
- boolean usePrePost = ABooleanSerializerDeserializer.getBoolean(usePrePostPtr.getByteArray(),
- usePrePostPtr.getStartOffset() + 1);
-
- // Compute result.
- long numGrams = usePrePost ? strLen + gramLen - 1 : strLen - gramLen + 1;
- long lowerBound = numGrams - edThresh * gramLen;
- try {
- if (lowerBound <= 0 || strLen == 0) {
- booleanSerde.serialize(ABoolean.FALSE, output);
- } else {
- booleanSerde.serialize(ABoolean.TRUE, output);
- }
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java
deleted file mode 100644
index 190013b..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.GramTokensEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8NGramTokenFactory;
-
-public class GramTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new GramTokensDescriptor();
- }
- };
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.GRAM_TOKENS;
- }
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- ITokenFactory tokenFactory = new UTF8NGramTokenFactory();
- NGramUTF8StringBinaryTokenizer tokenizer =
- new NGramUTF8StringBinaryTokenizer(3, true, true, true, tokenFactory);
- return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.ASTRING);
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java
deleted file mode 100644
index 32dc292..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.GramTokensEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8NGramTokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
-
-public class HashedGramTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new HashedGramTokensDescriptor();
- }
- };
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.HASHED_GRAM_TOKENS;
- }
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- ITokenFactory tokenFactory = new HashedUTF8NGramTokenFactory();
- NGramUTF8StringBinaryTokenizer tokenizer =
- new NGramUTF8StringBinaryTokenizer(3, true, true, true, tokenFactory);
- return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java
deleted file mode 100644
index e224fce..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.WordTokensEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8WordTokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
-
-public class HashedWordTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new HashedWordTokensDescriptor();
- }
- };
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.HASHED_WORD_TOKENS;
- }
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- ITokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
- IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, true, tokenFactory);
- return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
- }
- };
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java
deleted file mode 100644
index c9a865b..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.common.functions.FunctionConstants;
-import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityFilters;
-import org.apache.asterix.om.base.AInt32;
-import org.apache.asterix.om.base.AMutableInt32;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityFiltersCache;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class PrefixLenDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- private final static FunctionIdentifier FID =
- new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "prefix-len@3", 3);
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new PrefixLenDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
-
- return new IScalarEvaluator() {
-
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput out = resultStorage.getDataOutput();
- private final IPointable inputVal = new VoidPointable();
- private final IScalarEvaluator evalLen = args[0].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalSimilarity = args[1].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalThreshold = args[2].createScalarEvaluator(ctx);
-
- private final SimilarityFiltersCache similarityFiltersCache = new SimilarityFiltersCache();
-
- // result
- private final AMutableInt32 res = new AMutableInt32(0);
- @SuppressWarnings("unchecked")
- private final ISerializerDeserializer<AInt32> int32Serde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- // length
- evalLen.evaluate(tuple, inputVal);
- byte[] data = inputVal.getByteArray();
- int offset = inputVal.getStartOffset();
- if (data[offset] != ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 0, data[offset],
- ATypeTag.SERIALIZED_INT32_TYPE_TAG);
- }
- int length = IntegerPointable.getInteger(data, offset + 1);
-
- // similarity threshold
- evalThreshold.evaluate(tuple, inputVal);
- data = inputVal.getByteArray();
- offset = inputVal.getStartOffset();
- if (data[offset] != ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 1, data[offset],
- ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG);
- }
- float similarityThreshold = (float) ADoubleSerializerDeserializer.getDouble(data, offset + 1);
-
- // similarity name
- evalSimilarity.evaluate(tuple, inputVal);
- data = inputVal.getByteArray();
- offset = inputVal.getStartOffset();
- int len = inputVal.getLength();
- if (data[offset] != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 2, data[offset],
- ATypeTag.SERIALIZED_STRING_TYPE_TAG);
- }
- SimilarityFilters similarityFilters =
- similarityFiltersCache.get(similarityThreshold, data, offset, len);
-
- int prefixLength = similarityFilters.getPrefixLength(length);
- res.setValue(prefixLength);
-
- try {
- int32Serde.serialize(res, out);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
- };
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return FID;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java
deleted file mode 100644
index 25d4be3..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityFiltersJaccard;
-import org.apache.asterix.om.base.AInt32;
-import org.apache.asterix.om.base.AMutableInt32;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class PrefixLenJaccardDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new PrefixLenJaccardDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
-
- return new IScalarEvaluator() {
-
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput out = resultStorage.getDataOutput();
- private final IPointable lenPtr = new VoidPointable();
- private final IPointable thresholdPtr = new VoidPointable();
- private final IScalarEvaluator evalLen = args[0].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalThreshold = args[1].createScalarEvaluator(ctx);
-
- private float similarityThresholdCache;
- private SimilarityFiltersJaccard similarityFilters;
-
- // result
- private final AMutableInt32 res = new AMutableInt32(0);
- @SuppressWarnings("unchecked")
- private final ISerializerDeserializer<AInt32> int32Serde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- evalLen.evaluate(tuple, lenPtr);
- evalThreshold.evaluate(tuple, thresholdPtr);
-
- // length
- int length = ATypeHierarchy.getIntegerValue(getIdentifier().getName(), 0, lenPtr.getByteArray(),
- lenPtr.getStartOffset());
- // similarity threshold
- byte[] data = thresholdPtr.getByteArray();
- int offset = thresholdPtr.getStartOffset();
- if (data[offset] != ATypeTag.SERIALIZED_FLOAT_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 1, data[offset],
- ATypeTag.SERIALIZED_FLOAT_TYPE_TAG);
- }
- float similarityThreshold = AFloatSerializerDeserializer.getFloat(data, offset + 1);
-
- if (similarityThreshold != similarityThresholdCache || similarityFilters == null) {
- similarityFilters = new SimilarityFiltersJaccard(similarityThreshold);
- }
-
- int prefixLength = similarityFilters.getPrefixLength(length);
- res.setValue(prefixLength);
-
- try {
- int32Serde.serialize(res, out);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- result.set(resultStorage);
- }
- };
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.PREFIX_LEN_JACCARD;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java
deleted file mode 100644
index 8584d06..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.asterix.common.functions.FunctionConstants;
-import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AUnorderedListSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.fuzzyjoin.IntArray;
-import org.apache.asterix.fuzzyjoin.similarity.PartialIntersect;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityFilters;
-import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetric;
-import org.apache.asterix.om.base.ADouble;
-import org.apache.asterix.om.base.AMutableDouble;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityFiltersCache;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class SimilarityDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- private final static FunctionIdentifier FID =
- new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "similarity@7", 7);
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SimilarityDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
-
- return new IScalarEvaluator() {
-
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput out = resultStorage.getDataOutput();
- private final IPointable inputVal = new VoidPointable();
- private final IScalarEvaluator evalLen1 = args[0].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalTokens1 = args[1].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalLen2 = args[2].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalTokens2 = args[3].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalTokenPrefix = args[4].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalSimilarity = args[5].createScalarEvaluator(ctx);
- private final IScalarEvaluator evalThreshold = args[6].createScalarEvaluator(ctx);
-
- private final SimilarityFiltersCache similarityFiltersCache = new SimilarityFiltersCache();
-
- private final IntArray tokens1 = new IntArray();
- private final IntArray tokens2 = new IntArray();
- private final PartialIntersect parInter = new PartialIntersect();
-
- // result
- private final AMutableDouble res = new AMutableDouble(0);
- @SuppressWarnings("unchecked")
- private final ISerializerDeserializer<ADouble> doubleSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- // similarity threshold
- evalThreshold.evaluate(tuple, inputVal);
- byte[] data = inputVal.getByteArray();
- int offset = inputVal.getStartOffset();
-
- if (data[offset] != ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 0, data[offset],
- ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG);
- }
- float similarityThreshold = (float) ADoubleSerializerDeserializer.getDouble(data, offset + 1);
-
- // similarity name
- evalSimilarity.evaluate(tuple, inputVal);
- data = inputVal.getByteArray();
- offset = inputVal.getStartOffset();
- int len = inputVal.getLength();
- if (data[offset] != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 1, data[offset],
- ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG);
- }
- SimilarityFilters similarityFilters =
- similarityFiltersCache.get(similarityThreshold, data, offset, len);
-
- evalLen1.evaluate(tuple, inputVal);
- data = inputVal.getByteArray();
- offset = inputVal.getStartOffset();
- if (data[offset] != ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 2, data[offset],
- ATypeTag.SERIALIZED_INT32_TYPE_TAG);
- }
- int length1 = IntegerPointable.getInteger(data, offset + 1);
-
- evalLen2.evaluate(tuple, inputVal);
- data = inputVal.getByteArray();
- offset = inputVal.getStartOffset();
- if (data[offset] != ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 3, data[offset],
- ATypeTag.SERIALIZED_INT32_TYPE_TAG);
- }
- int length2 = IntegerPointable.getInteger(data, offset + 1);
-
- float sim = 0;
-
- //
- // -- - length filter - --
- //
- if (similarityFilters.passLengthFilter(length1, length2)) {
-
- // -- - tokens1 - --
- int i;
- tokens1.reset();
- evalTokens1.evaluate(tuple, inputVal);
- byte[] serList = inputVal.getByteArray();
- offset = inputVal.getStartOffset();
-
- if (serList[offset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
- && serList[offset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 4, data[offset],
- ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG,
- ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
-
- int lengthTokens1;
- if (serList[offset] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
- lengthTokens1 = AOrderedListSerializerDeserializer.getNumberOfItems(serList, offset);
- // read tokens
- for (i = 0; i < lengthTokens1; i++) {
- int itemOffset =
- AOrderedListSerializerDeserializer.getItemOffset(serList, offset, i);
- tokens1.add(IntegerPointable.getInteger(serList, itemOffset));
- }
- } else {
- lengthTokens1 = AUnorderedListSerializerDeserializer.getNumberOfItems(serList, offset);
- // read tokens
- for (i = 0; i < lengthTokens1; i++) {
- int itemOffset =
- AUnorderedListSerializerDeserializer.getItemOffset(serList, offset, i);
- tokens1.add(IntegerPointable.getInteger(serList, itemOffset));
- }
- }
- // pad tokens
- for (; i < length1; i++) {
- tokens1.add(Integer.MAX_VALUE);
- }
-
- // -- - tokens2 - --
- tokens2.reset();
- evalTokens2.evaluate(tuple, inputVal);
- serList = inputVal.getByteArray();
- offset = inputVal.getStartOffset();
-
- if (serList[offset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
- && serList[offset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
- throw new TypeMismatchException(getIdentifier(), 5, data[offset],
- ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG,
- ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
- }
-
- int lengthTokens2;
- if (serList[0] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
- lengthTokens2 = AOrderedListSerializerDeserializer.getNumberOfItems(serList, offset);
- // read tokens
- for (i = 0; i < lengthTokens2; i++) {
- int itemOffset =
- AOrderedListSerializerDeserializer.getItemOffset(serList, offset, i);
- tokens2.add(IntegerPointable.getInteger(serList, itemOffset));
- }
- } else {
- lengthTokens2 = AUnorderedListSerializerDeserializer.getNumberOfItems(serList, offset);
- // read tokens
- for (i = 0; i < lengthTokens2; i++) {
- int itemOffset =
- AUnorderedListSerializerDeserializer.getItemOffset(serList, offset, i);
- tokens2.add(IntegerPointable.getInteger(serList, itemOffset));
- }
- }
- // pad tokens
- for (; i < length2; i++) {
- tokens2.add(Integer.MAX_VALUE);
- }
-
- // -- - token prefix - --
- evalTokenPrefix.evaluate(tuple, inputVal);
- int tokenPrefix =
- IntegerPointable.getInteger(inputVal.getByteArray(), inputVal.getStartOffset() + 1);
-
- //
- // -- - position filter - --
- //
- SimilarityMetric.getPartialIntersectSize(tokens1.get(), 0, tokens1.length(), tokens2.get(),
- 0, tokens2.length(), tokenPrefix, parInter);
- if (similarityFilters.passPositionFilter(parInter.intersectSize, parInter.posXStop, length1,
- parInter.posYStop, length2)) {
-
- //
- // -- - suffix filter - --
- //
- if (similarityFilters.passSuffixFilter(tokens1.get(), 0, tokens1.length(),
- parInter.posXStart, tokens2.get(), 0, tokens2.length(), parInter.posYStart)) {
-
- sim = similarityFilters.passSimilarityFilter(tokens1.get(), 0, tokens1.length(),
- parInter.posXStop + 1, tokens2.get(), 0, tokens2.length(),
- parInter.posYStop + 1, parInter.intersectSize);
- }
- }
- }
-
- res.setValue(sim);
-
- try {
- doubleSerde.serialize(res, out);
- } catch (IOException e) {
- throw HyracksDataException.create(e);
- }
- result.set(resultStorage);
- }
- };
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return FID;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java
deleted file mode 100644
index 9fc5b34..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardCheckEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class SimilarityJaccardCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SimilarityJaccardCheckDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new SimilarityJaccardCheckEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.SIMILARITY_JACCARD_CHECK;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java
deleted file mode 100644
index 742f2ff..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class SimilarityJaccardDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SimilarityJaccardDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new SimilarityJaccardEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.SIMILARITY_JACCARD;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java
deleted file mode 100644
index e1b5f3f..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.IOException;
-
-import org.apache.asterix.builders.OrderedListBuilder;
-import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.base.AFloat;
-import org.apache.asterix.om.base.AMutableFloat;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.AOrderedListType;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardPrefixEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-
-public class SimilarityJaccardPrefixCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
-
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SimilarityJaccardPrefixCheckDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
- return new SimilarityJaccardPrefixCheckEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.SIMILARITY_JACCARD_PREFIX_CHECK;
- }
-
- private static class SimilarityJaccardPrefixCheckEvaluator extends SimilarityJaccardPrefixEvaluator {
-
- private final OrderedListBuilder listBuilder;
- private ArrayBackedValueStorage inputVal;
- @SuppressWarnings("unchecked")
- private final ISerializerDeserializer<ABoolean> booleanSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
- @SuppressWarnings("unchecked")
- private final ISerializerDeserializer<AFloat> floatSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AFLOAT);
- private final AMutableFloat aFloat = new AMutableFloat(0);
-
- private final AOrderedListType listType = new AOrderedListType(BuiltinType.ANY, "list");
-
- public SimilarityJaccardPrefixCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
- throws HyracksDataException {
- super(args, context);
- listBuilder = new OrderedListBuilder();
- inputVal = new ArrayBackedValueStorage();
- }
-
- @Override
- public void writeResult() throws HyracksDataException, IOException {
- listBuilder.reset(listType);
- boolean matches = (sim <= 0) ? false : true;
- float jaccSim = (matches) ? sim : 0.0f;
-
- inputVal.reset();
- booleanSerde.serialize(matches ? ABoolean.TRUE : ABoolean.FALSE, inputVal.getDataOutput());
- listBuilder.addItem(inputVal);
-
- inputVal.reset();
- aFloat.setValue(jaccSim);
- floatSerde.serialize(aFloat, inputVal.getDataOutput());
- listBuilder.addItem(inputVal);
- listBuilder.write(out, true);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java
deleted file mode 100644
index f6b102a..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardPrefixEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class SimilarityJaccardPrefixDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SimilarityJaccardPrefixDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
- return new SimilarityJaccardPrefixEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.SIMILARITY_JACCARD_PREFIX;
- }
-
-}
[2/7] asterixdb git commit: [ASTERIXDB-2330][*DB][RT] Add
IFunctionRegistrant for dynamic function registration
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java
deleted file mode 100644
index b821661..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardSortedCheckEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-// Assumes that both arguments are sorted by the same ordering.
-public class SimilarityJaccardSortedCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SimilarityJaccardSortedCheckDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new SimilarityJaccardSortedCheckEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.SIMILARITY_JACCARD_SORTED_CHECK;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java
deleted file mode 100644
index 0477c2c..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardSortedEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-// Assumes that both arguments are sorted by the same ordering.
-public class SimilarityJaccardSortedDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SimilarityJaccardSortedDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- return new SimilarityJaccardSortedEvaluator(args, ctx);
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.SIMILARITY_JACCARD_SORTED;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java
deleted file mode 100644
index 4229f87..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java
+++ /dev/null
@@ -1,1091 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import java.io.DataOutput;
-
-import org.apache.asterix.dataflow.data.nontagged.Coordinate;
-import org.apache.asterix.dataflow.data.nontagged.serde.ACircleSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AInt16SerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.ALineSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.AObjectSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.APointSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.APolygonSerializerDeserializer;
-import org.apache.asterix.dataflow.data.nontagged.serde.ARectangleSerializerDeserializer;
-import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
-import org.apache.asterix.fuzzyjoin.IntArray;
-import org.apache.asterix.om.base.ABoolean;
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.EnumDeserializer;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.DoubleArray;
-import org.apache.asterix.runtime.evaluators.common.SpatialUtils;
-import org.apache.asterix.runtime.exceptions.InvalidDataFormatException;
-import org.apache.asterix.runtime.exceptions.TypeMismatchException;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.data.std.primitive.VoidPointable;
-import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class SpatialIntersectDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
-
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new SpatialIntersectDescriptor();
- }
- };
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
- return new IScalarEvaluator() {
-
- private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
- private final DataOutput out = resultStorage.getDataOutput();
- private final IPointable inputArg0 = new VoidPointable();
- private final IPointable inputArg1 = new VoidPointable();
- private final IScalarEvaluator eval0 = args[0].createScalarEvaluator(ctx);
- private final IScalarEvaluator eval1 = args[1].createScalarEvaluator(ctx);
- private final IBinaryComparator ascDoubleComp =
- BinaryComparatorFactoryProvider.DOUBLE_POINTABLE_INSTANCE.createBinaryComparator();
- private final SpatialUtils spatialUtils = new SpatialUtils();
- private final IntArray pointsOffsets0 = new IntArray();
- private final IntArray pointsOffsets1 = new IntArray();
- private final DoubleArray trianglesX0 = new DoubleArray();
- private final DoubleArray trianglesY0 = new DoubleArray();
- private final DoubleArray trianglesX1 = new DoubleArray();
- private final DoubleArray trianglesY1 = new DoubleArray();
- private final AObjectSerializerDeserializer aBooleanSerDer = AObjectSerializerDeserializer.INSTANCE;
-
- private boolean pointOnLine(double pX, double pY, double startX, double startY, double endX,
- double endY) throws HyracksDataException {
- double crossProduct =
- SpatialUtils.crossProduct(pY - startY, pX - startX, endY - startY, endX - startX);
- if (Math.abs(crossProduct) > SpatialUtils.doubleEpsilon()) { // crossProduct != 0
- return false;
- }
-
- double dotProduct =
- SpatialUtils.dotProduct((pX - startX), (pY - startY), (endX - startX), (endY - startY));
- if (dotProduct < 0.0) {
- return false;
- }
-
- double squaredLengthBA = (endX - startX) * (endX - startX) + (endY - startY) * (endY - startY);
- if (dotProduct > squaredLengthBA) {
- return false;
- }
- return true;
- }
-
- private boolean pointInPolygon(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException { // ray casting
-
- double pX = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
- double pY = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
- int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
- offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
-
- if (numOfPoints1 < 3) {
- throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
- }
-
- int counter = 0;
- double xInters;
- double x1, x2, y1, y2;
- x1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.X));
- y1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.Y));
-
- for (int i = 1; i <= numOfPoints1; i++) {
- if (i == numOfPoints1) {
- x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.X));
- y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.Y));
- } else {
- x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.X));
- y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.Y));
- }
-
- if (!pointOnLine(pX, pY, x1, y1, x2, y2)) {
- if (pY > Math.min(y1, y2)) {
- if (pY <= Math.max(y1, y2)) {
- if (pX <= Math.max(x1, x2)) {
- if (y1 != y2) {
- xInters = (pY - y1) * (x2 - x1) / (y2 - y1) + x1;
- if (x1 == x2 || pX <= xInters) {
- counter++;
- }
- }
- }
- }
- }
- }
- x1 = x2;
- y1 = y2;
- }
- if (counter % 2 == 1) {
- return true;
- } else {
- return false;
- }
- }
-
- private boolean pointInCircle(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- double x = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
- double y = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
-
- double cX = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
- double cY = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
- double radius = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getRadiusOffset());
-
- if ((x - cX) * (x - cX) + (y - cY) * (y - cY) <= (radius * radius)) {
- return true;
- }
- return false;
- }
-
- private boolean lineLineIntersection(double startX1, double startY1, double endX1, double endY1,
- double startX2, double startY2, double endX2, double endY2) {
- double A1 = endY1 - startY1;
- double B1 = startX1 - endX1;
- double C1 = A1 * startX1 + B1 * startY1;
-
- double A2 = endY2 - startY2;
- double B2 = startX2 - endX2;
- double C2 = A2 * startX2 + B2 * startY2;
-
- double det = (A1 * B2) - (A2 * B1);
- if (Math.abs(det) > SpatialUtils.doubleEpsilon()) { // det != 0
- double x = (B2 * C1 - B1 * C2) / det;
- double y = (A1 * C2 - A2 * C1) / det;
-
- if ((x >= Math.min(startX1, endX1) && x <= Math.max(startX1, endX1))
- && (y >= Math.min(startY1, endY1) && y <= Math.max(startY1, endY1))) {
- if ((x >= Math.min(startX2, endX2) && x <= Math.max(startX2, endX2))
- && (y >= Math.min(startY2, endY2) && y <= Math.max(startY2, endY2))) {
- return true;
- }
- }
- }
- return false;
- }
-
- private boolean linePolygonIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- double startX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.X));
- double startY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.Y));
- double endX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.X));
- double endY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.Y));
-
- int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
- offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
-
- if (numOfPoints1 < 3) {
- throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
- }
- for (int i = 0; i < numOfPoints1; i++) {
- double startX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.X));
- double startY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.Y));
-
- double endX2;
- double endY2;
- if (i + 1 == numOfPoints1) {
- endX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.X));
- endY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.Y));
- } else {
- endX2 = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
- + APolygonSerializerDeserializer.getCoordinateOffset(i + 1, Coordinate.X));
- endY2 = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
- + APolygonSerializerDeserializer.getCoordinateOffset(i + 1, Coordinate.Y));
- }
-
- boolean intersect = lineLineIntersection(startX1, startY1, endX1, endY1, startX2, startY2,
- endX2, endY2);
- if (intersect) {
- return true;
- }
- }
- return false;
- }
-
- private boolean lineRectangleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- double startX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.X));
- double startY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.Y));
- double endX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.X));
- double endY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.Y));
-
- double x1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.X));
- double y1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.Y));
-
- double x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.X));
- double y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.Y));
-
- if (lineLineIntersection(startX1, startY1, endX1, endY1, x1, y1, x1, y2)
- || lineLineIntersection(startX1, startY1, endX1, endY1, x1, y2, x2, y2)
- || lineLineIntersection(startX1, startY1, endX1, endY1, x2, y2, x2, y1)
- || lineLineIntersection(startX1, startY1, endX1, endY1, x2, y1, x1, y1)) {
- return true;
- }
- return false;
-
- }
-
- private boolean lineCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- double startX = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.X));
- double startY = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.Y));
- double endX = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.X));
- double endY = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.Y));
-
- double cX = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
- double cY = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
- double radius = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getRadiusOffset());
-
- double dx = endX - startX;
- double dy = endY - startY;
- double t = -((startX - cX) * dx + (startY - cY) * dy) / ((dx * dx) + (dy * dy));
-
- if (t < 0.0) {
- t = 0.0;
- } else if (t > 1.0) {
- t = 1.0;
- }
-
- dx = (startX + t * (endX - startX)) - cX;
- dy = (startY + t * (endY - startY)) - cY;
- double rt = (dx * dx) + (dy * dy);
- if (rt <= (radius * radius)) {
- return true;
- }
- return false;
- }
-
- private boolean findEar(byte[] bytes, int offset, int u, int v, int w, int n,
- IntArray pointsOffsets) throws HyracksDataException {
- int p;
- double Ax, Ay, Bx, By, Cx, Cy, Px, Py;
-
- Ax = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(u), Coordinate.X));
- Ay = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(u), Coordinate.Y));
-
- Bx = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(v), Coordinate.X));
- By = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(v), Coordinate.Y));
-
- Cx = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(w), Coordinate.X));
- Cy = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(w), Coordinate.Y));
-
- if (SpatialUtils.doubleEpsilon() > (((Bx - Ax) * (Cy - Ay)) - ((By - Ay) * (Cx - Ax)))) {
-
- return false;
- }
-
- for (p = 0; p < n; p++) {
- if ((p == u) || (p == v) || (p == w)) {
- continue;
- }
- Px = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(p), Coordinate.X));
- Py = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(p), Coordinate.Y));
- if (pointInsideTriangle(Ax, Ay, Bx, By, Cx, Cy, Px, Py)) {
- return false;
- }
- }
-
- return true;
- }
-
- private int triangulatePolygon(byte[] bytes, int offset, int numOfPoints, IntArray pointsOffsets,
- DoubleArray trianglesX, DoubleArray trianglesY, int triangleId,
- int nonSimplePolygonDetection, int middleVertex) throws HyracksDataException { // Ear clipping
-
- if (numOfPoints < 3) {
- return -1;
- }
-
- boolean foundEar = false;
- int v = middleVertex;
- while (!foundEar) {
- if (0 >= (nonSimplePolygonDetection--)) {
- throw new InvalidDataFormatException(getIdentifier(),
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
- }
- int u = v;
- if (numOfPoints <= u) {
- u = 0;
- }
- v = u + 1;
- if (numOfPoints <= v) {
- v = 0;
- }
- int w = v + 1;
- if (numOfPoints <= w) {
- w = 0;
- }
-
- if (findEar(bytes, offset, u, v, w, numOfPoints, pointsOffsets)) {
- int s, t;
-
- addRectangle(trianglesX, trianglesY);
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, triangleId, 0,
- ADoubleSerializerDeserializer.getDouble(bytes,
- offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(u), Coordinate.X)));
-
- SpatialUtils.setTriangleYCoordinate(trianglesY, triangleId, 0,
- ADoubleSerializerDeserializer.getDouble(bytes,
- offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(u), Coordinate.Y)));
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, triangleId, 1,
- ADoubleSerializerDeserializer.getDouble(bytes,
- offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(v), Coordinate.X)));
-
- SpatialUtils.setTriangleYCoordinate(trianglesY, triangleId, 1,
- ADoubleSerializerDeserializer.getDouble(bytes,
- offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(v), Coordinate.Y)));
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, triangleId, 2,
- ADoubleSerializerDeserializer.getDouble(bytes,
- offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(w), Coordinate.X)));
-
- SpatialUtils.setTriangleYCoordinate(trianglesY, triangleId, 2,
- ADoubleSerializerDeserializer.getDouble(bytes,
- offset + APolygonSerializerDeserializer
- .getCoordinateOffset(pointsOffsets.get(w), Coordinate.Y)));
-
- // remove v from polygon
- for (s = v, t = v + 1; t < numOfPoints; s++, t++) {
- pointsOffsets.get()[s] = pointsOffsets.get(t);
- }
- foundEar = true;
- }
- }
-
- return v;
- }
-
- private boolean triangleTriangleIntersection(DoubleArray trianglesX0, DoubleArray trianglesY0,
- int triangleId0, DoubleArray trianglesX1, DoubleArray trianglesY1, int triangleId1)
- throws HyracksDataException { // separating axis theorem
-
- for (int side = 0; side < 3; side++) {
- spatialUtils.findNormals(trianglesX0, trianglesY0, triangleId0, side);
- spatialUtils.projectPolygon(trianglesX0, trianglesY0, triangleId0, spatialUtils.getXAxis(),
- spatialUtils.getYAxis());
- double min1 = spatialUtils.getMinProjection();
- double max1 = spatialUtils.getMaxProjection();
- spatialUtils.projectPolygon(trianglesX1, trianglesY1, triangleId1, spatialUtils.getXAxis(),
- spatialUtils.getYAxis());
- double min2 = spatialUtils.getMinProjection();
- double max2 = spatialUtils.getMaxProjection();
-
- if (max1 < min2 || min1 > max2) {
- return false;
- }
- }
- return true;
- }
-
- private boolean pointInsideTriangle(double x1, double y1, double x2, double y2, double x3,
- double y3, double pX, double pY) {
- return pointsOnSameSide(pX, pY, x1, y1, x2, y2, x3, y3)
- && pointsOnSameSide(pX, pY, x2, y2, x1, y1, x3, y3)
- && pointsOnSameSide(pX, pY, x3, y3, x1, y1, x2, y2);
- }
-
- private boolean pointsOnSameSide(double pX, double pY, double x1, double y1, double x2, double y2,
- double x3, double y3) {
- double cp1 = SpatialUtils.crossProduct(x3 - x2, y3 - y2, pX - x2, pY - y2);
- double cp2 = SpatialUtils.crossProduct(x3 - x2, y3 - y2, x1 - x2, y1 - y2);
- return (cp1 * cp2) >= 0.0;
- }
-
- private boolean circleTriangleIntersection(byte[] bytes0, int offset0, DoubleArray trianglesX,
- DoubleArray trianglesY, int triangleId) throws HyracksDataException { // separating axis theorem
-
- double cX = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
- double cY = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
- double radius = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ACircleSerializerDeserializer.getRadiusOffset());
-
- double distance = Double.MAX_VALUE;
- double distanceSquared;
-
- double temp;
- double closestPointX = 0.0;
- double closestPointY = 0.0;
- for (int i = 0; i < 3; i++) {
- double pX = SpatialUtils.getTriangleXCoordinate(trianglesX, triangleId, i);
- double pY = SpatialUtils.getTriangleXCoordinate(trianglesY, triangleId, i);
-
- distanceSquared = (cX - pX) * (cX - pX) + (cY - pY) * (cY - pY);
- if (distanceSquared < distance) {
- distance = distanceSquared;
- closestPointX = pX;
- closestPointY = pY;
- }
- }
-
- double x = Math.abs(cX - closestPointX);
- double y = Math.abs(cY - closestPointY);
-
- temp = Math.sqrt(SpatialUtils.dotProduct(x, y, x, y));
- x /= temp;
- y /= temp;
-
- spatialUtils.projectPolygon(trianglesX, trianglesY, triangleId, x, y);
-
- double min1 = spatialUtils.getMinProjection();
- double max1 = spatialUtils.getMaxProjection();
-
- double dotProduct = SpatialUtils.dotProduct(x, y, cX, cY);
- double max2 = dotProduct + radius;
- double min2 = dotProduct - radius;
-
- if (max1 < min2 || min1 > max2) {
- return false;
- }
-
- for (int side = 0; side < 3; side++) {
- spatialUtils.findNormals(trianglesX, trianglesY, triangleId, side);
- spatialUtils.projectPolygon(trianglesX, trianglesY, triangleId, spatialUtils.getXAxis(),
- spatialUtils.getYAxis());
- min1 = spatialUtils.getMinProjection();
- max1 = spatialUtils.getMaxProjection();
-
- dotProduct =
- SpatialUtils.dotProduct(spatialUtils.getXAxis(), spatialUtils.getYAxis(), cX, cY);
- max2 = dotProduct + radius;
- min2 = dotProduct - radius;
-
- if (max1 < min2 || min1 > max2) {
- return false;
- }
- }
- return true;
- }
-
- private boolean circleCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- double cX0 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
- double cY0 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
- double radius0 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ACircleSerializerDeserializer.getRadiusOffset());
-
- double cX1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
- double cY1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
- double radius1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ACircleSerializerDeserializer.getRadiusOffset());
-
- double distanceSquared = SpatialUtils.dotProduct(cX0 - cX1, cY0 - cY1, cX0 - cX1, cY0 - cY1);
- double radiusDistanceSquared = (radius0 + radius1) * (radius0 + radius1);
- if (distanceSquared <= radiusDistanceSquared) {
- return true;
- }
- return false;
- }
-
- private void getCounterClockWisePolygon(byte[] bytes, int offset, IntArray pointsOffsets,
- int numOfPoints) throws HyracksDataException {
- pointsOffsets.reset();
- if (SpatialUtils.polygonArea(bytes, offset, numOfPoints) > 0.0) {
- for (int i = 0; i < numOfPoints; i++) {
- pointsOffsets.add(i);
- }
- } else {
- for (int i = 0; i < numOfPoints; i++) {
- pointsOffsets.add((numOfPoints - 1) - i);
- }
- }
- }
-
- private boolean pointInRectangle(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
-
- double pX = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
- double pY = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
-
- double x1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.X));
- double y1 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.Y));
-
- double x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.X));
- double y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.Y));
-
- if (pointInsideTriangle(x1, y1, x1, y2, x2, y2, pX, pY)
- || pointInsideTriangle(x1, y1, x2, y1, x2, y2, pX, pY)) {
- return true;
- }
- return false;
-
- }
-
- private void addRectangle(DoubleArray trianglesX, DoubleArray trianglesY) {
- for (int i = 0; i < 3; i++) {
- double temp = 0;
- trianglesX.add(temp);
- trianglesY.add(temp);
- }
- }
-
- private boolean rectangleCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- triangulateRectangle(bytes0, offset0, trianglesX0, trianglesY0);
- boolean res = false;
- // 2 triangles in a rectangle
- for (int i = 0; i < 2; i++) {
- res = circleTriangleIntersection(bytes1, offset1, trianglesX0, trianglesY0, i);
- if (res) {
- break;
- }
- }
- return res;
- }
-
- private void triangulateRectangle(byte[] bytes, int offset, DoubleArray trianglesX,
- DoubleArray trianglesY) throws HyracksDataException {
- double x1 = ADoubleSerializerDeserializer.getDouble(bytes,
- offset + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.X));
- double y1 = ADoubleSerializerDeserializer.getDouble(bytes,
- offset + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.Y));
-
- double x2 = ADoubleSerializerDeserializer.getDouble(bytes,
- offset + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.X));
- double y2 = ADoubleSerializerDeserializer.getDouble(bytes,
- offset + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.Y));
- trianglesX.reset();
- trianglesY.reset();
-
- addRectangle(trianglesX, trianglesY);
- addRectangle(trianglesX, trianglesY);
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, 0, 0, x1);
- SpatialUtils.setTriangleYCoordinate(trianglesY, 0, 0, y1);
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, 0, 1, x2);
- SpatialUtils.setTriangleYCoordinate(trianglesY, 0, 1, y1);
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, 0, 2, x2);
- SpatialUtils.setTriangleYCoordinate(trianglesY, 0, 2, y2);
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, 1, 0, x2);
- SpatialUtils.setTriangleYCoordinate(trianglesY, 1, 0, y2);
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, 1, 1, x1);
- SpatialUtils.setTriangleYCoordinate(trianglesY, 1, 1, y2);
-
- SpatialUtils.setTriangleXCoordinate(trianglesX, 1, 2, x1);
- SpatialUtils.setTriangleYCoordinate(trianglesY, 1, 2, y1);
- }
-
- private boolean rectanglePolygonIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
- offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
-
- if (numOfPoints1 < 3) {
- throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
- }
-
- getCounterClockWisePolygon(bytes1, offset1, pointsOffsets1, numOfPoints1);
- int nonSimplePolygonDetection1 = 2 * numOfPoints1;
- int middleVertex1 = numOfPoints1 - 1;
- int numOfTriangles1 = 0;
-
- trianglesX1.reset();
- trianglesY1.reset();
- while (true) {
- middleVertex1 =
- triangulatePolygon(bytes1, offset1, numOfPoints1, pointsOffsets1, trianglesX1,
- trianglesY1, numOfTriangles1, nonSimplePolygonDetection1, middleVertex1);
-
- if (middleVertex1 == -1) {
- break;
- }
-
- numOfPoints1--;
- nonSimplePolygonDetection1 = 2 * numOfPoints1;
- numOfTriangles1++;
- }
-
- triangulateRectangle(bytes0, offset0, trianglesX0, trianglesY0);
- boolean res = false;
- // 2 triangles in a rectangle
- for (int j = 0; j < 2; j++) {
- for (int i = 0; i < numOfTriangles1; i++) {
-
- res = triangleTriangleIntersection(trianglesX1, trianglesY1, i, trianglesX0,
- trianglesY0, j);
-
- if (res) {
- res = triangleTriangleIntersection(trianglesX0, trianglesY0, j, trianglesX1,
- trianglesY1, i);
-
- if (res) {
- return true;
- }
- }
- }
- }
- return false;
- }
-
- private boolean polygonCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
- throws HyracksDataException {
- int numOfPoints = AInt16SerializerDeserializer.getShort(bytes0,
- offset0 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
-
- if (numOfPoints < 3) {
- throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
- }
-
- getCounterClockWisePolygon(bytes0, offset0, pointsOffsets0, numOfPoints);
- int nonSimplePolygonDetection = 2 * numOfPoints;
- int middleVertex = numOfPoints - 1;
- int numOfTriangles = 0;
-
- trianglesX0.reset();
- trianglesY0.reset();
- boolean res = false;
- while (true) {
- middleVertex = triangulatePolygon(bytes0, offset0, numOfPoints, pointsOffsets0, trianglesX0,
- trianglesY0, numOfTriangles, nonSimplePolygonDetection, middleVertex);
-
- if (middleVertex == -1) {
- break;
- }
- numOfPoints--;
- nonSimplePolygonDetection = 2 * numOfPoints;
- numOfTriangles++;
- int lastTriangle = (trianglesX0.length() / 3) - 1;
-
- res = circleTriangleIntersection(bytes1, offset1, trianglesX0, trianglesY0, lastTriangle);
- if (res) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
- resultStorage.reset();
- eval0.evaluate(tuple, inputArg0);
- eval1.evaluate(tuple, inputArg1);
-
- byte[] bytes0 = inputArg0.getByteArray();
- byte[] bytes1 = inputArg1.getByteArray();
- int offset0 = inputArg0.getStartOffset();
- int offset1 = inputArg1.getStartOffset();
-
- boolean res = false;
- ATypeTag tag0 = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes0[offset0]);
- ATypeTag tag1 = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes1[offset1]);
-
- switch (tag0) {
- case POINT:
- switch (tag1) {
- case POINT:
- if (ascDoubleComp.compare(bytes0,
- offset0 + APointSerializerDeserializer
- .getCoordinateOffset(Coordinate.X),
- 8, bytes1, offset1 + APointSerializerDeserializer
- .getCoordinateOffset(Coordinate.X),
- 8) == 0) {
- if (ascDoubleComp.compare(bytes0,
- offset0 + APointSerializerDeserializer
- .getCoordinateOffset(Coordinate.Y),
- 8, bytes1, offset1 + APointSerializerDeserializer
- .getCoordinateOffset(Coordinate.Y),
- 8) == 0) {
- res = true;
- }
- }
- break;
- case LINE:
- double pX = ADoubleSerializerDeserializer.getDouble(bytes0, offset0
- + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
- double pY = ADoubleSerializerDeserializer.getDouble(bytes0, offset0
- + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
-
- double startX = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.X));
- double startY = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.Y));
- double endX = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.X));
- double endY = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.Y));
-
- res = pointOnLine(pX, pY, startX, startY, endX, endY);
- break;
- case POLYGON:
- res = pointInPolygon(bytes0, offset0, bytes1, offset1);
- break;
- case CIRCLE:
- res = pointInCircle(bytes0, offset0, bytes1, offset1);
- break;
- case RECTANGLE:
- res = pointInRectangle(bytes0, offset0, bytes1, offset1);
- break;
- default:
- throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
- ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
- ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
- ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
- }
- break;
- case LINE:
- switch (tag1) {
- case POINT:
- double pX = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
- + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
- double pY = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
- + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
-
- double startX = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.X));
- double startY = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.Y));
- double endX = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.X));
- double endY = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.Y));
-
- res = pointOnLine(pX, pY, startX, startY, endX, endY);
- break;
- case LINE:
- double startX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.X));
- double startY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.Y));
- double endX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.X));
- double endY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
- offset0 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.Y));
-
- double startX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.X));
- double startY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getStartPointCoordinateOffset(Coordinate.Y));
- double endX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.X));
- double endY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
- offset1 + ALineSerializerDeserializer
- .getEndPointCoordinateOffset(Coordinate.Y));
- res = lineLineIntersection(startX1, startY1, endX1, endY1, startX2, startY2,
- endX2, endY2);
- break;
- case POLYGON:
- res = linePolygonIntersection(bytes0, offset0, bytes1, offset1);
- break;
- case CIRCLE:
- res = lineCircleIntersection(bytes0, offset0, bytes1, offset1);
- break;
- case RECTANGLE:
- res = lineRectangleIntersection(bytes0, offset0, bytes1, offset1);
- break;
- default:
- throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
- ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
- ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
- ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
- }
- break;
- case POLYGON:
- switch (tag1) {
- case POINT:
- res = pointInPolygon(bytes1, offset1, bytes0, offset0);
- break;
- case LINE:
- res = linePolygonIntersection(bytes1, offset1, bytes0, offset0);
- break;
- case POLYGON:
- int numOfPoints0 = AInt16SerializerDeserializer.getShort(bytes0,
- offset0 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
- int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
- offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
-
- if (numOfPoints0 < 3 || numOfPoints1 < 3) {
- throw new InvalidDataFormatException(getIdentifier(),
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
- }
-
- getCounterClockWisePolygon(bytes0, offset0, pointsOffsets0, numOfPoints0);
- getCounterClockWisePolygon(bytes1, offset1, pointsOffsets1, numOfPoints1);
- int nonSimplePolygonDetection0 = 2 * numOfPoints0;
- int nonSimplePolygonDetection1 = 2 * numOfPoints1;
- boolean intersect = false;
- int middleVertex0 = numOfPoints0 - 1;
-
- int numOfTriangles1 = 0;
- int middleVertex1 = numOfPoints1 - 1;
- trianglesX1.reset();
- trianglesY1.reset();
- while (true) {
- middleVertex1 = triangulatePolygon(bytes1, offset1, numOfPoints1,
- pointsOffsets1, trianglesX1, trianglesY1, numOfTriangles1,
- nonSimplePolygonDetection1, middleVertex1);
-
- if (middleVertex1 == -1) {
- break;
- }
-
- numOfPoints1--;
- nonSimplePolygonDetection1 = 2 * numOfPoints1;
- numOfTriangles1++;
- }
- int numOfTriangles0 = 0;
- trianglesX0.reset();
- trianglesY0.reset();
- while (true) {
- middleVertex0 = triangulatePolygon(bytes0, offset0, numOfPoints0,
- pointsOffsets0, trianglesX0, trianglesY0, numOfTriangles0,
- nonSimplePolygonDetection0, middleVertex0);
-
- if (middleVertex0 == -1) {
- break;
- }
- numOfPoints0--;
- nonSimplePolygonDetection0 = 2 * numOfPoints0;
- numOfTriangles0++;
- int lastTriangle = (trianglesX0.length() / 3) - 1;
-
- for (int i = 0; i < numOfTriangles1; i++) {
-
- res = triangleTriangleIntersection(trianglesX0, trianglesY0,
- lastTriangle, trianglesX1, trianglesY1, i);
-
- if (res) {
- res = triangleTriangleIntersection(trianglesX1, trianglesY1, i,
- trianglesX0, trianglesY0, lastTriangle);
-
- if (res) {
- intersect = true;
- break;
- }
- }
- }
- if (intersect) {
- break;
- }
- }
- break;
- case CIRCLE:
- res = polygonCircleIntersection(bytes0, offset0, bytes1, offset1);
- break;
- case RECTANGLE:
- res = rectanglePolygonIntersection(bytes1, offset1, bytes0, offset0);
- break;
- default:
- throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
- ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
- ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
- ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
- }
- break;
- case CIRCLE:
- switch (tag1) {
- case POINT:
- res = pointInCircle(bytes1, offset1, bytes0, offset0);
- break;
- case LINE:
- res = lineCircleIntersection(bytes1, offset1, bytes0, offset0);
- break;
- case POLYGON:
- res = polygonCircleIntersection(bytes1, offset1, bytes0, offset0);
- break;
- case CIRCLE:
- res = circleCircleIntersection(bytes0, offset0, bytes1, offset1);
- break;
- case RECTANGLE:
- res = rectangleCircleIntersection(bytes1, offset1, bytes0, offset0);
- break;
- default:
- throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
- ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
- ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
- ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
- }
- break;
- case RECTANGLE:
- switch (tag1) {
- case POINT:
- res = pointInRectangle(bytes1, offset1, bytes0, offset0);
- break;
- case LINE:
- res = lineRectangleIntersection(bytes1, offset1, bytes0, offset0);
- break;
- case POLYGON:
- res = rectanglePolygonIntersection(bytes0, offset0, bytes1, offset1);
- break;
- case CIRCLE:
- res = rectangleCircleIntersection(bytes0, offset0, bytes1, offset1);
- break;
- case RECTANGLE:
- triangulateRectangle(bytes0, offset0, trianglesX0, trianglesY0);
- triangulateRectangle(bytes1, offset1, trianglesX1, trianglesY1);
-
- boolean intersect = false;
- // 2 triangles in a rectangle
- for (int j = 0; j < 2; j++) {
- for (int i = 0; i < 2; i++) {
-
- res = triangleTriangleIntersection(trianglesX1, trianglesY1, i,
- trianglesX0, trianglesY0, j);
-
- if (res) {
- res = triangleTriangleIntersection(trianglesX0, trianglesY0, j,
- trianglesX1, trianglesY1, i);
-
- if (res) {
- intersect = true;
- break;
- }
- }
- }
- if (intersect) {
- break;
- }
- }
- break;
- default:
- throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
- ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
- ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
- ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
- }
- break;
- default:
- throw new TypeMismatchException(getIdentifier(), 0, bytes0[offset0],
- ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
- ATypeTag.SERIALIZED_POLYGON_TYPE_TAG, ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
- ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
- }
-
- ABoolean aResult = res ? ABoolean.TRUE : ABoolean.FALSE;
- aBooleanSerDer.serialize(aResult, out);
- result.set(resultStorage);
- }
- };
- }
- };
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.SPATIAL_INTERSECT;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java
deleted file mode 100644
index 1b93a3c..0000000
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.runtime.evaluators.functions;
-
-import org.apache.asterix.om.functions.BuiltinFunctions;
-import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
-import org.apache.asterix.om.types.BuiltinType;
-import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.asterix.runtime.evaluators.common.WordTokensEvaluator;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
-import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8WordTokenFactory;
-
-public class WordTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
-
- private static final long serialVersionUID = 1L;
- public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
- @Override
- public IFunctionDescriptor createFunctionDescriptor() {
- return new WordTokensDescriptor();
- }
- };
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return BuiltinFunctions.WORD_TOKENS;
- }
-
- @Override
- public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
- return new IScalarEvaluatorFactory() {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
- ITokenFactory tokenFactory = new UTF8WordTokenFactory();
- IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, true, tokenFactory);
- return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.ASTRING);
- }
- };
- }
-
-}
[5/7] asterixdb git commit: [ASTERIXDB-2330][*DB][RT] Add
IFunctionRegistrant for dynamic function registration
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java
new file mode 100644
index 0000000..e1b5f3f
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixCheckDescriptor.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.IOException;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.AFloat;
+import org.apache.asterix.om.base.AMutableFloat;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardPrefixEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+public class SimilarityJaccardPrefixCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SimilarityJaccardPrefixCheckDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
+ return new SimilarityJaccardPrefixCheckEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SIMILARITY_JACCARD_PREFIX_CHECK;
+ }
+
+ private static class SimilarityJaccardPrefixCheckEvaluator extends SimilarityJaccardPrefixEvaluator {
+
+ private final OrderedListBuilder listBuilder;
+ private ArrayBackedValueStorage inputVal;
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ABoolean> booleanSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<AFloat> floatSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AFLOAT);
+ private final AMutableFloat aFloat = new AMutableFloat(0);
+
+ private final AOrderedListType listType = new AOrderedListType(BuiltinType.ANY, "list");
+
+ public SimilarityJaccardPrefixCheckEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ super(args, context);
+ listBuilder = new OrderedListBuilder();
+ inputVal = new ArrayBackedValueStorage();
+ }
+
+ @Override
+ public void writeResult() throws HyracksDataException, IOException {
+ listBuilder.reset(listType);
+ boolean matches = (sim <= 0) ? false : true;
+ float jaccSim = (matches) ? sim : 0.0f;
+
+ inputVal.reset();
+ booleanSerde.serialize(matches ? ABoolean.TRUE : ABoolean.FALSE, inputVal.getDataOutput());
+ listBuilder.addItem(inputVal);
+
+ inputVal.reset();
+ aFloat.setValue(jaccSim);
+ floatSerde.serialize(aFloat, inputVal.getDataOutput());
+ listBuilder.addItem(inputVal);
+ listBuilder.write(out, true);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java
new file mode 100644
index 0000000..f6b102a
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardPrefixDescriptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardPrefixEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class SimilarityJaccardPrefixDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SimilarityJaccardPrefixDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
+ return new SimilarityJaccardPrefixEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SIMILARITY_JACCARD_PREFIX;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java
new file mode 100644
index 0000000..b821661
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedCheckDescriptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardSortedCheckEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+// Assumes that both arguments are sorted by the same ordering.
+public class SimilarityJaccardSortedCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SimilarityJaccardSortedCheckDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new SimilarityJaccardSortedCheckEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SIMILARITY_JACCARD_SORTED_CHECK;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java
new file mode 100644
index 0000000..0477c2c
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardSortedDescriptor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardSortedEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+// Assumes that both arguments are sorted by the same ordering.
+public class SimilarityJaccardSortedDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SimilarityJaccardSortedDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new SimilarityJaccardSortedEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SIMILARITY_JACCARD_SORTED;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java
new file mode 100644
index 0000000..4229f87
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SpatialIntersectDescriptor.java
@@ -0,0 +1,1091 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+
+import org.apache.asterix.dataflow.data.nontagged.Coordinate;
+import org.apache.asterix.dataflow.data.nontagged.serde.ACircleSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt16SerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.ALineSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AObjectSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.APointSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.APolygonSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.ARectangleSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider;
+import org.apache.asterix.fuzzyjoin.IntArray;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.DoubleArray;
+import org.apache.asterix.runtime.evaluators.common.SpatialUtils;
+import org.apache.asterix.runtime.exceptions.InvalidDataFormatException;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class SpatialIntersectDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SpatialIntersectDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
+ return new IScalarEvaluator() {
+
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable inputArg0 = new VoidPointable();
+ private final IPointable inputArg1 = new VoidPointable();
+ private final IScalarEvaluator eval0 = args[0].createScalarEvaluator(ctx);
+ private final IScalarEvaluator eval1 = args[1].createScalarEvaluator(ctx);
+ private final IBinaryComparator ascDoubleComp =
+ BinaryComparatorFactoryProvider.DOUBLE_POINTABLE_INSTANCE.createBinaryComparator();
+ private final SpatialUtils spatialUtils = new SpatialUtils();
+ private final IntArray pointsOffsets0 = new IntArray();
+ private final IntArray pointsOffsets1 = new IntArray();
+ private final DoubleArray trianglesX0 = new DoubleArray();
+ private final DoubleArray trianglesY0 = new DoubleArray();
+ private final DoubleArray trianglesX1 = new DoubleArray();
+ private final DoubleArray trianglesY1 = new DoubleArray();
+ private final AObjectSerializerDeserializer aBooleanSerDer = AObjectSerializerDeserializer.INSTANCE;
+
+ private boolean pointOnLine(double pX, double pY, double startX, double startY, double endX,
+ double endY) throws HyracksDataException {
+ double crossProduct =
+ SpatialUtils.crossProduct(pY - startY, pX - startX, endY - startY, endX - startX);
+ if (Math.abs(crossProduct) > SpatialUtils.doubleEpsilon()) { // crossProduct != 0
+ return false;
+ }
+
+ double dotProduct =
+ SpatialUtils.dotProduct((pX - startX), (pY - startY), (endX - startX), (endY - startY));
+ if (dotProduct < 0.0) {
+ return false;
+ }
+
+ double squaredLengthBA = (endX - startX) * (endX - startX) + (endY - startY) * (endY - startY);
+ if (dotProduct > squaredLengthBA) {
+ return false;
+ }
+ return true;
+ }
+
+ private boolean pointInPolygon(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException { // ray casting
+
+ double pX = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
+ double pY = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
+ int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
+ offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
+
+ if (numOfPoints1 < 3) {
+ throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
+ }
+
+ int counter = 0;
+ double xInters;
+ double x1, x2, y1, y2;
+ x1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.X));
+ y1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.Y));
+
+ for (int i = 1; i <= numOfPoints1; i++) {
+ if (i == numOfPoints1) {
+ x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.X));
+ y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.Y));
+ } else {
+ x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.X));
+ y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.Y));
+ }
+
+ if (!pointOnLine(pX, pY, x1, y1, x2, y2)) {
+ if (pY > Math.min(y1, y2)) {
+ if (pY <= Math.max(y1, y2)) {
+ if (pX <= Math.max(x1, x2)) {
+ if (y1 != y2) {
+ xInters = (pY - y1) * (x2 - x1) / (y2 - y1) + x1;
+ if (x1 == x2 || pX <= xInters) {
+ counter++;
+ }
+ }
+ }
+ }
+ }
+ }
+ x1 = x2;
+ y1 = y2;
+ }
+ if (counter % 2 == 1) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private boolean pointInCircle(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ double x = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
+ double y = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
+
+ double cX = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
+ double cY = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
+ double radius = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getRadiusOffset());
+
+ if ((x - cX) * (x - cX) + (y - cY) * (y - cY) <= (radius * radius)) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean lineLineIntersection(double startX1, double startY1, double endX1, double endY1,
+ double startX2, double startY2, double endX2, double endY2) {
+ double A1 = endY1 - startY1;
+ double B1 = startX1 - endX1;
+ double C1 = A1 * startX1 + B1 * startY1;
+
+ double A2 = endY2 - startY2;
+ double B2 = startX2 - endX2;
+ double C2 = A2 * startX2 + B2 * startY2;
+
+ double det = (A1 * B2) - (A2 * B1);
+ if (Math.abs(det) > SpatialUtils.doubleEpsilon()) { // det != 0
+ double x = (B2 * C1 - B1 * C2) / det;
+ double y = (A1 * C2 - A2 * C1) / det;
+
+ if ((x >= Math.min(startX1, endX1) && x <= Math.max(startX1, endX1))
+ && (y >= Math.min(startY1, endY1) && y <= Math.max(startY1, endY1))) {
+ if ((x >= Math.min(startX2, endX2) && x <= Math.max(startX2, endX2))
+ && (y >= Math.min(startY2, endY2) && y <= Math.max(startY2, endY2))) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean linePolygonIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ double startX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.X));
+ double startY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.Y));
+ double endX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.X));
+ double endY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.Y));
+
+ int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
+ offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
+
+ if (numOfPoints1 < 3) {
+ throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
+ }
+ for (int i = 0; i < numOfPoints1; i++) {
+ double startX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.X));
+ double startY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(i, Coordinate.Y));
+
+ double endX2;
+ double endY2;
+ if (i + 1 == numOfPoints1) {
+ endX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.X));
+ endY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + APolygonSerializerDeserializer.getCoordinateOffset(0, Coordinate.Y));
+ } else {
+ endX2 = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
+ + APolygonSerializerDeserializer.getCoordinateOffset(i + 1, Coordinate.X));
+ endY2 = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
+ + APolygonSerializerDeserializer.getCoordinateOffset(i + 1, Coordinate.Y));
+ }
+
+ boolean intersect = lineLineIntersection(startX1, startY1, endX1, endY1, startX2, startY2,
+ endX2, endY2);
+ if (intersect) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean lineRectangleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ double startX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.X));
+ double startY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.Y));
+ double endX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.X));
+ double endY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.Y));
+
+ double x1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.X));
+ double y1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.Y));
+
+ double x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.X));
+ double y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.Y));
+
+ if (lineLineIntersection(startX1, startY1, endX1, endY1, x1, y1, x1, y2)
+ || lineLineIntersection(startX1, startY1, endX1, endY1, x1, y2, x2, y2)
+ || lineLineIntersection(startX1, startY1, endX1, endY1, x2, y2, x2, y1)
+ || lineLineIntersection(startX1, startY1, endX1, endY1, x2, y1, x1, y1)) {
+ return true;
+ }
+ return false;
+
+ }
+
+ private boolean lineCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ double startX = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.X));
+ double startY = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getStartPointCoordinateOffset(Coordinate.Y));
+ double endX = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.X));
+ double endY = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer.getEndPointCoordinateOffset(Coordinate.Y));
+
+ double cX = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
+ double cY = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
+ double radius = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getRadiusOffset());
+
+ double dx = endX - startX;
+ double dy = endY - startY;
+ double t = -((startX - cX) * dx + (startY - cY) * dy) / ((dx * dx) + (dy * dy));
+
+ if (t < 0.0) {
+ t = 0.0;
+ } else if (t > 1.0) {
+ t = 1.0;
+ }
+
+ dx = (startX + t * (endX - startX)) - cX;
+ dy = (startY + t * (endY - startY)) - cY;
+ double rt = (dx * dx) + (dy * dy);
+ if (rt <= (radius * radius)) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean findEar(byte[] bytes, int offset, int u, int v, int w, int n,
+ IntArray pointsOffsets) throws HyracksDataException {
+ int p;
+ double Ax, Ay, Bx, By, Cx, Cy, Px, Py;
+
+ Ax = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(u), Coordinate.X));
+ Ay = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(u), Coordinate.Y));
+
+ Bx = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(v), Coordinate.X));
+ By = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(v), Coordinate.Y));
+
+ Cx = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(w), Coordinate.X));
+ Cy = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(w), Coordinate.Y));
+
+ if (SpatialUtils.doubleEpsilon() > (((Bx - Ax) * (Cy - Ay)) - ((By - Ay) * (Cx - Ax)))) {
+
+ return false;
+ }
+
+ for (p = 0; p < n; p++) {
+ if ((p == u) || (p == v) || (p == w)) {
+ continue;
+ }
+ Px = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(p), Coordinate.X));
+ Py = ADoubleSerializerDeserializer.getDouble(bytes, offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(p), Coordinate.Y));
+ if (pointInsideTriangle(Ax, Ay, Bx, By, Cx, Cy, Px, Py)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private int triangulatePolygon(byte[] bytes, int offset, int numOfPoints, IntArray pointsOffsets,
+ DoubleArray trianglesX, DoubleArray trianglesY, int triangleId,
+ int nonSimplePolygonDetection, int middleVertex) throws HyracksDataException { // Ear clipping
+
+ if (numOfPoints < 3) {
+ return -1;
+ }
+
+ boolean foundEar = false;
+ int v = middleVertex;
+ while (!foundEar) {
+ if (0 >= (nonSimplePolygonDetection--)) {
+ throw new InvalidDataFormatException(getIdentifier(),
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
+ }
+ int u = v;
+ if (numOfPoints <= u) {
+ u = 0;
+ }
+ v = u + 1;
+ if (numOfPoints <= v) {
+ v = 0;
+ }
+ int w = v + 1;
+ if (numOfPoints <= w) {
+ w = 0;
+ }
+
+ if (findEar(bytes, offset, u, v, w, numOfPoints, pointsOffsets)) {
+ int s, t;
+
+ addRectangle(trianglesX, trianglesY);
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, triangleId, 0,
+ ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(u), Coordinate.X)));
+
+ SpatialUtils.setTriangleYCoordinate(trianglesY, triangleId, 0,
+ ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(u), Coordinate.Y)));
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, triangleId, 1,
+ ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(v), Coordinate.X)));
+
+ SpatialUtils.setTriangleYCoordinate(trianglesY, triangleId, 1,
+ ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(v), Coordinate.Y)));
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, triangleId, 2,
+ ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(w), Coordinate.X)));
+
+ SpatialUtils.setTriangleYCoordinate(trianglesY, triangleId, 2,
+ ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + APolygonSerializerDeserializer
+ .getCoordinateOffset(pointsOffsets.get(w), Coordinate.Y)));
+
+ // remove v from polygon
+ for (s = v, t = v + 1; t < numOfPoints; s++, t++) {
+ pointsOffsets.get()[s] = pointsOffsets.get(t);
+ }
+ foundEar = true;
+ }
+ }
+
+ return v;
+ }
+
+ private boolean triangleTriangleIntersection(DoubleArray trianglesX0, DoubleArray trianglesY0,
+ int triangleId0, DoubleArray trianglesX1, DoubleArray trianglesY1, int triangleId1)
+ throws HyracksDataException { // separating axis theorem
+
+ for (int side = 0; side < 3; side++) {
+ spatialUtils.findNormals(trianglesX0, trianglesY0, triangleId0, side);
+ spatialUtils.projectPolygon(trianglesX0, trianglesY0, triangleId0, spatialUtils.getXAxis(),
+ spatialUtils.getYAxis());
+ double min1 = spatialUtils.getMinProjection();
+ double max1 = spatialUtils.getMaxProjection();
+ spatialUtils.projectPolygon(trianglesX1, trianglesY1, triangleId1, spatialUtils.getXAxis(),
+ spatialUtils.getYAxis());
+ double min2 = spatialUtils.getMinProjection();
+ double max2 = spatialUtils.getMaxProjection();
+
+ if (max1 < min2 || min1 > max2) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean pointInsideTriangle(double x1, double y1, double x2, double y2, double x3,
+ double y3, double pX, double pY) {
+ return pointsOnSameSide(pX, pY, x1, y1, x2, y2, x3, y3)
+ && pointsOnSameSide(pX, pY, x2, y2, x1, y1, x3, y3)
+ && pointsOnSameSide(pX, pY, x3, y3, x1, y1, x2, y2);
+ }
+
+ private boolean pointsOnSameSide(double pX, double pY, double x1, double y1, double x2, double y2,
+ double x3, double y3) {
+ double cp1 = SpatialUtils.crossProduct(x3 - x2, y3 - y2, pX - x2, pY - y2);
+ double cp2 = SpatialUtils.crossProduct(x3 - x2, y3 - y2, x1 - x2, y1 - y2);
+ return (cp1 * cp2) >= 0.0;
+ }
+
+ private boolean circleTriangleIntersection(byte[] bytes0, int offset0, DoubleArray trianglesX,
+ DoubleArray trianglesY, int triangleId) throws HyracksDataException { // separating axis theorem
+
+ double cX = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
+ double cY = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
+ double radius = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ACircleSerializerDeserializer.getRadiusOffset());
+
+ double distance = Double.MAX_VALUE;
+ double distanceSquared;
+
+ double temp;
+ double closestPointX = 0.0;
+ double closestPointY = 0.0;
+ for (int i = 0; i < 3; i++) {
+ double pX = SpatialUtils.getTriangleXCoordinate(trianglesX, triangleId, i);
+ double pY = SpatialUtils.getTriangleXCoordinate(trianglesY, triangleId, i);
+
+ distanceSquared = (cX - pX) * (cX - pX) + (cY - pY) * (cY - pY);
+ if (distanceSquared < distance) {
+ distance = distanceSquared;
+ closestPointX = pX;
+ closestPointY = pY;
+ }
+ }
+
+ double x = Math.abs(cX - closestPointX);
+ double y = Math.abs(cY - closestPointY);
+
+ temp = Math.sqrt(SpatialUtils.dotProduct(x, y, x, y));
+ x /= temp;
+ y /= temp;
+
+ spatialUtils.projectPolygon(trianglesX, trianglesY, triangleId, x, y);
+
+ double min1 = spatialUtils.getMinProjection();
+ double max1 = spatialUtils.getMaxProjection();
+
+ double dotProduct = SpatialUtils.dotProduct(x, y, cX, cY);
+ double max2 = dotProduct + radius;
+ double min2 = dotProduct - radius;
+
+ if (max1 < min2 || min1 > max2) {
+ return false;
+ }
+
+ for (int side = 0; side < 3; side++) {
+ spatialUtils.findNormals(trianglesX, trianglesY, triangleId, side);
+ spatialUtils.projectPolygon(trianglesX, trianglesY, triangleId, spatialUtils.getXAxis(),
+ spatialUtils.getYAxis());
+ min1 = spatialUtils.getMinProjection();
+ max1 = spatialUtils.getMaxProjection();
+
+ dotProduct =
+ SpatialUtils.dotProduct(spatialUtils.getXAxis(), spatialUtils.getYAxis(), cX, cY);
+ max2 = dotProduct + radius;
+ min2 = dotProduct - radius;
+
+ if (max1 < min2 || min1 > max2) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean circleCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ double cX0 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
+ double cY0 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
+ double radius0 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ACircleSerializerDeserializer.getRadiusOffset());
+
+ double cX1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.X));
+ double cY1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getCenterPointCoordinateOffset(Coordinate.Y));
+ double radius1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ACircleSerializerDeserializer.getRadiusOffset());
+
+ double distanceSquared = SpatialUtils.dotProduct(cX0 - cX1, cY0 - cY1, cX0 - cX1, cY0 - cY1);
+ double radiusDistanceSquared = (radius0 + radius1) * (radius0 + radius1);
+ if (distanceSquared <= radiusDistanceSquared) {
+ return true;
+ }
+ return false;
+ }
+
+ private void getCounterClockWisePolygon(byte[] bytes, int offset, IntArray pointsOffsets,
+ int numOfPoints) throws HyracksDataException {
+ pointsOffsets.reset();
+ if (SpatialUtils.polygonArea(bytes, offset, numOfPoints) > 0.0) {
+ for (int i = 0; i < numOfPoints; i++) {
+ pointsOffsets.add(i);
+ }
+ } else {
+ for (int i = 0; i < numOfPoints; i++) {
+ pointsOffsets.add((numOfPoints - 1) - i);
+ }
+ }
+ }
+
+ private boolean pointInRectangle(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+
+ double pX = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
+ double pY = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
+
+ double x1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.X));
+ double y1 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.Y));
+
+ double x2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.X));
+ double y2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.Y));
+
+ if (pointInsideTriangle(x1, y1, x1, y2, x2, y2, pX, pY)
+ || pointInsideTriangle(x1, y1, x2, y1, x2, y2, pX, pY)) {
+ return true;
+ }
+ return false;
+
+ }
+
+ private void addRectangle(DoubleArray trianglesX, DoubleArray trianglesY) {
+ for (int i = 0; i < 3; i++) {
+ double temp = 0;
+ trianglesX.add(temp);
+ trianglesY.add(temp);
+ }
+ }
+
+ private boolean rectangleCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ triangulateRectangle(bytes0, offset0, trianglesX0, trianglesY0);
+ boolean res = false;
+ // 2 triangles in a rectangle
+ for (int i = 0; i < 2; i++) {
+ res = circleTriangleIntersection(bytes1, offset1, trianglesX0, trianglesY0, i);
+ if (res) {
+ break;
+ }
+ }
+ return res;
+ }
+
+ private void triangulateRectangle(byte[] bytes, int offset, DoubleArray trianglesX,
+ DoubleArray trianglesY) throws HyracksDataException {
+ double x1 = ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.X));
+ double y1 = ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + ARectangleSerializerDeserializer.getBottomLeftCoordinateOffset(Coordinate.Y));
+
+ double x2 = ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.X));
+ double y2 = ADoubleSerializerDeserializer.getDouble(bytes,
+ offset + ARectangleSerializerDeserializer.getUpperRightCoordinateOffset(Coordinate.Y));
+ trianglesX.reset();
+ trianglesY.reset();
+
+ addRectangle(trianglesX, trianglesY);
+ addRectangle(trianglesX, trianglesY);
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, 0, 0, x1);
+ SpatialUtils.setTriangleYCoordinate(trianglesY, 0, 0, y1);
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, 0, 1, x2);
+ SpatialUtils.setTriangleYCoordinate(trianglesY, 0, 1, y1);
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, 0, 2, x2);
+ SpatialUtils.setTriangleYCoordinate(trianglesY, 0, 2, y2);
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, 1, 0, x2);
+ SpatialUtils.setTriangleYCoordinate(trianglesY, 1, 0, y2);
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, 1, 1, x1);
+ SpatialUtils.setTriangleYCoordinate(trianglesY, 1, 1, y2);
+
+ SpatialUtils.setTriangleXCoordinate(trianglesX, 1, 2, x1);
+ SpatialUtils.setTriangleYCoordinate(trianglesY, 1, 2, y1);
+ }
+
+ private boolean rectanglePolygonIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
+ offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
+
+ if (numOfPoints1 < 3) {
+ throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
+ }
+
+ getCounterClockWisePolygon(bytes1, offset1, pointsOffsets1, numOfPoints1);
+ int nonSimplePolygonDetection1 = 2 * numOfPoints1;
+ int middleVertex1 = numOfPoints1 - 1;
+ int numOfTriangles1 = 0;
+
+ trianglesX1.reset();
+ trianglesY1.reset();
+ while (true) {
+ middleVertex1 =
+ triangulatePolygon(bytes1, offset1, numOfPoints1, pointsOffsets1, trianglesX1,
+ trianglesY1, numOfTriangles1, nonSimplePolygonDetection1, middleVertex1);
+
+ if (middleVertex1 == -1) {
+ break;
+ }
+
+ numOfPoints1--;
+ nonSimplePolygonDetection1 = 2 * numOfPoints1;
+ numOfTriangles1++;
+ }
+
+ triangulateRectangle(bytes0, offset0, trianglesX0, trianglesY0);
+ boolean res = false;
+ // 2 triangles in a rectangle
+ for (int j = 0; j < 2; j++) {
+ for (int i = 0; i < numOfTriangles1; i++) {
+
+ res = triangleTriangleIntersection(trianglesX1, trianglesY1, i, trianglesX0,
+ trianglesY0, j);
+
+ if (res) {
+ res = triangleTriangleIntersection(trianglesX0, trianglesY0, j, trianglesX1,
+ trianglesY1, i);
+
+ if (res) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ private boolean polygonCircleIntersection(byte[] bytes0, int offset0, byte[] bytes1, int offset1)
+ throws HyracksDataException {
+ int numOfPoints = AInt16SerializerDeserializer.getShort(bytes0,
+ offset0 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
+
+ if (numOfPoints < 3) {
+ throw new InvalidDataFormatException(getIdentifier(), ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
+ }
+
+ getCounterClockWisePolygon(bytes0, offset0, pointsOffsets0, numOfPoints);
+ int nonSimplePolygonDetection = 2 * numOfPoints;
+ int middleVertex = numOfPoints - 1;
+ int numOfTriangles = 0;
+
+ trianglesX0.reset();
+ trianglesY0.reset();
+ boolean res = false;
+ while (true) {
+ middleVertex = triangulatePolygon(bytes0, offset0, numOfPoints, pointsOffsets0, trianglesX0,
+ trianglesY0, numOfTriangles, nonSimplePolygonDetection, middleVertex);
+
+ if (middleVertex == -1) {
+ break;
+ }
+ numOfPoints--;
+ nonSimplePolygonDetection = 2 * numOfPoints;
+ numOfTriangles++;
+ int lastTriangle = (trianglesX0.length() / 3) - 1;
+
+ res = circleTriangleIntersection(bytes1, offset1, trianglesX0, trianglesY0, lastTriangle);
+ if (res) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ eval0.evaluate(tuple, inputArg0);
+ eval1.evaluate(tuple, inputArg1);
+
+ byte[] bytes0 = inputArg0.getByteArray();
+ byte[] bytes1 = inputArg1.getByteArray();
+ int offset0 = inputArg0.getStartOffset();
+ int offset1 = inputArg1.getStartOffset();
+
+ boolean res = false;
+ ATypeTag tag0 = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes0[offset0]);
+ ATypeTag tag1 = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes1[offset1]);
+
+ switch (tag0) {
+ case POINT:
+ switch (tag1) {
+ case POINT:
+ if (ascDoubleComp.compare(bytes0,
+ offset0 + APointSerializerDeserializer
+ .getCoordinateOffset(Coordinate.X),
+ 8, bytes1, offset1 + APointSerializerDeserializer
+ .getCoordinateOffset(Coordinate.X),
+ 8) == 0) {
+ if (ascDoubleComp.compare(bytes0,
+ offset0 + APointSerializerDeserializer
+ .getCoordinateOffset(Coordinate.Y),
+ 8, bytes1, offset1 + APointSerializerDeserializer
+ .getCoordinateOffset(Coordinate.Y),
+ 8) == 0) {
+ res = true;
+ }
+ }
+ break;
+ case LINE:
+ double pX = ADoubleSerializerDeserializer.getDouble(bytes0, offset0
+ + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
+ double pY = ADoubleSerializerDeserializer.getDouble(bytes0, offset0
+ + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
+
+ double startX = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.X));
+ double startY = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.Y));
+ double endX = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.X));
+ double endY = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.Y));
+
+ res = pointOnLine(pX, pY, startX, startY, endX, endY);
+ break;
+ case POLYGON:
+ res = pointInPolygon(bytes0, offset0, bytes1, offset1);
+ break;
+ case CIRCLE:
+ res = pointInCircle(bytes0, offset0, bytes1, offset1);
+ break;
+ case RECTANGLE:
+ res = pointInRectangle(bytes0, offset0, bytes1, offset1);
+ break;
+ default:
+ throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
+ ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
+ ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
+ ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
+ }
+ break;
+ case LINE:
+ switch (tag1) {
+ case POINT:
+ double pX = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
+ + APointSerializerDeserializer.getCoordinateOffset(Coordinate.X));
+ double pY = ADoubleSerializerDeserializer.getDouble(bytes1, offset1
+ + APointSerializerDeserializer.getCoordinateOffset(Coordinate.Y));
+
+ double startX = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.X));
+ double startY = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.Y));
+ double endX = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.X));
+ double endY = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.Y));
+
+ res = pointOnLine(pX, pY, startX, startY, endX, endY);
+ break;
+ case LINE:
+ double startX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.X));
+ double startY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.Y));
+ double endX1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.X));
+ double endY1 = ADoubleSerializerDeserializer.getDouble(bytes0,
+ offset0 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.Y));
+
+ double startX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.X));
+ double startY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getStartPointCoordinateOffset(Coordinate.Y));
+ double endX2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.X));
+ double endY2 = ADoubleSerializerDeserializer.getDouble(bytes1,
+ offset1 + ALineSerializerDeserializer
+ .getEndPointCoordinateOffset(Coordinate.Y));
+ res = lineLineIntersection(startX1, startY1, endX1, endY1, startX2, startY2,
+ endX2, endY2);
+ break;
+ case POLYGON:
+ res = linePolygonIntersection(bytes0, offset0, bytes1, offset1);
+ break;
+ case CIRCLE:
+ res = lineCircleIntersection(bytes0, offset0, bytes1, offset1);
+ break;
+ case RECTANGLE:
+ res = lineRectangleIntersection(bytes0, offset0, bytes1, offset1);
+ break;
+ default:
+ throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
+ ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
+ ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
+ ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
+ }
+ break;
+ case POLYGON:
+ switch (tag1) {
+ case POINT:
+ res = pointInPolygon(bytes1, offset1, bytes0, offset0);
+ break;
+ case LINE:
+ res = linePolygonIntersection(bytes1, offset1, bytes0, offset0);
+ break;
+ case POLYGON:
+ int numOfPoints0 = AInt16SerializerDeserializer.getShort(bytes0,
+ offset0 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
+ int numOfPoints1 = AInt16SerializerDeserializer.getShort(bytes1,
+ offset1 + APolygonSerializerDeserializer.getNumberOfPointsOffset());
+
+ if (numOfPoints0 < 3 || numOfPoints1 < 3) {
+ throw new InvalidDataFormatException(getIdentifier(),
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG);
+ }
+
+ getCounterClockWisePolygon(bytes0, offset0, pointsOffsets0, numOfPoints0);
+ getCounterClockWisePolygon(bytes1, offset1, pointsOffsets1, numOfPoints1);
+ int nonSimplePolygonDetection0 = 2 * numOfPoints0;
+ int nonSimplePolygonDetection1 = 2 * numOfPoints1;
+ boolean intersect = false;
+ int middleVertex0 = numOfPoints0 - 1;
+
+ int numOfTriangles1 = 0;
+ int middleVertex1 = numOfPoints1 - 1;
+ trianglesX1.reset();
+ trianglesY1.reset();
+ while (true) {
+ middleVertex1 = triangulatePolygon(bytes1, offset1, numOfPoints1,
+ pointsOffsets1, trianglesX1, trianglesY1, numOfTriangles1,
+ nonSimplePolygonDetection1, middleVertex1);
+
+ if (middleVertex1 == -1) {
+ break;
+ }
+
+ numOfPoints1--;
+ nonSimplePolygonDetection1 = 2 * numOfPoints1;
+ numOfTriangles1++;
+ }
+ int numOfTriangles0 = 0;
+ trianglesX0.reset();
+ trianglesY0.reset();
+ while (true) {
+ middleVertex0 = triangulatePolygon(bytes0, offset0, numOfPoints0,
+ pointsOffsets0, trianglesX0, trianglesY0, numOfTriangles0,
+ nonSimplePolygonDetection0, middleVertex0);
+
+ if (middleVertex0 == -1) {
+ break;
+ }
+ numOfPoints0--;
+ nonSimplePolygonDetection0 = 2 * numOfPoints0;
+ numOfTriangles0++;
+ int lastTriangle = (trianglesX0.length() / 3) - 1;
+
+ for (int i = 0; i < numOfTriangles1; i++) {
+
+ res = triangleTriangleIntersection(trianglesX0, trianglesY0,
+ lastTriangle, trianglesX1, trianglesY1, i);
+
+ if (res) {
+ res = triangleTriangleIntersection(trianglesX1, trianglesY1, i,
+ trianglesX0, trianglesY0, lastTriangle);
+
+ if (res) {
+ intersect = true;
+ break;
+ }
+ }
+ }
+ if (intersect) {
+ break;
+ }
+ }
+ break;
+ case CIRCLE:
+ res = polygonCircleIntersection(bytes0, offset0, bytes1, offset1);
+ break;
+ case RECTANGLE:
+ res = rectanglePolygonIntersection(bytes1, offset1, bytes0, offset0);
+ break;
+ default:
+ throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
+ ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
+ ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
+ ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
+ }
+ break;
+ case CIRCLE:
+ switch (tag1) {
+ case POINT:
+ res = pointInCircle(bytes1, offset1, bytes0, offset0);
+ break;
+ case LINE:
+ res = lineCircleIntersection(bytes1, offset1, bytes0, offset0);
+ break;
+ case POLYGON:
+ res = polygonCircleIntersection(bytes1, offset1, bytes0, offset0);
+ break;
+ case CIRCLE:
+ res = circleCircleIntersection(bytes0, offset0, bytes1, offset1);
+ break;
+ case RECTANGLE:
+ res = rectangleCircleIntersection(bytes1, offset1, bytes0, offset0);
+ break;
+ default:
+ throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
+ ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
+ ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
+ ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
+ }
+ break;
+ case RECTANGLE:
+ switch (tag1) {
+ case POINT:
+ res = pointInRectangle(bytes1, offset1, bytes0, offset0);
+ break;
+ case LINE:
+ res = lineRectangleIntersection(bytes1, offset1, bytes0, offset0);
+ break;
+ case POLYGON:
+ res = rectanglePolygonIntersection(bytes0, offset0, bytes1, offset1);
+ break;
+ case CIRCLE:
+ res = rectangleCircleIntersection(bytes0, offset0, bytes1, offset1);
+ break;
+ case RECTANGLE:
+ triangulateRectangle(bytes0, offset0, trianglesX0, trianglesY0);
+ triangulateRectangle(bytes1, offset1, trianglesX1, trianglesY1);
+
+ boolean intersect = false;
+ // 2 triangles in a rectangle
+ for (int j = 0; j < 2; j++) {
+ for (int i = 0; i < 2; i++) {
+
+ res = triangleTriangleIntersection(trianglesX1, trianglesY1, i,
+ trianglesX0, trianglesY0, j);
+
+ if (res) {
+ res = triangleTriangleIntersection(trianglesX0, trianglesY0, j,
+ trianglesX1, trianglesY1, i);
+
+ if (res) {
+ intersect = true;
+ break;
+ }
+ }
+ }
+ if (intersect) {
+ break;
+ }
+ }
+ break;
+ default:
+ throw new TypeMismatchException(getIdentifier(), 1, bytes1[offset1],
+ ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG,
+ ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
+ ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
+ }
+ break;
+ default:
+ throw new TypeMismatchException(getIdentifier(), 0, bytes0[offset0],
+ ATypeTag.SERIALIZED_POINT_TYPE_TAG, ATypeTag.SERIALIZED_LINE_TYPE_TAG,
+ ATypeTag.SERIALIZED_POLYGON_TYPE_TAG, ATypeTag.SERIALIZED_CIRCLE_TYPE_TAG,
+ ATypeTag.SERIALIZED_RECTANGLE_TYPE_TAG);
+ }
+
+ ABoolean aResult = res ? ABoolean.TRUE : ABoolean.FALSE;
+ aBooleanSerDer.serialize(aResult, out);
+ result.set(resultStorage);
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SPATIAL_INTERSECT;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java
new file mode 100644
index 0000000..1b93a3c
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/WordTokensDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.WordTokensEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8WordTokenFactory;
+
+public class WordTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new WordTokensDescriptor();
+ }
+ };
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.WORD_TOKENS;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ ITokenFactory tokenFactory = new UTF8WordTokenFactory();
+ IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, true, tokenFactory);
+ return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.ASTRING);
+ }
+ };
+ }
+
+}
[6/7] asterixdb git commit: [ASTERIXDB-2330][*DB][RT] Add
IFunctionRegistrant for dynamic function registration
Posted by mb...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java
new file mode 100644
index 0000000..dd36671
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedGramTokensDescriptor.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.GramTokensEvaluator;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8NGramTokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
+
+public class CountHashedGramTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new CountHashedGramTokensDescriptor();
+ }
+ };
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.COUNTHASHED_GRAM_TOKENS;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ ITokenFactory tokenFactory = new HashedUTF8NGramTokenFactory();
+ NGramUTF8StringBinaryTokenizer tokenizer =
+ new NGramUTF8StringBinaryTokenizer(3, true, false, true, tokenFactory);
+ return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java
new file mode 100644
index 0000000..e12ba2e
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/CountHashedWordTokensDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.WordTokensEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8WordTokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
+
+public class CountHashedWordTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new CountHashedWordTokensDescriptor();
+ }
+ };
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.COUNTHASHED_WORD_TOKENS;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ ITokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
+ IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(false, true, tokenFactory);
+ return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java
new file mode 100644
index 0000000..e4b40b1
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceCheckDescriptor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.EditDistanceCheckEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class EditDistanceCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new EditDistanceCheckDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new EditDistanceCheckEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.EDIT_DISTANCE_CHECK;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java
new file mode 100644
index 0000000..4c7c257
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceContainsDescriptor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.EditDistanceContainsEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class EditDistanceContainsDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new EditDistanceContainsDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new EditDistanceContainsEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.EDIT_DISTANCE_CONTAINS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java
new file mode 100644
index 0000000..8c6c9ed
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceDescriptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.EditDistanceEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class EditDistanceDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new EditDistanceDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new EditDistanceEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.EDIT_DISTANCE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java
new file mode 100644
index 0000000..0f4ebee
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceListIsFilterableDescriptor.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AUnorderedListSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.EnumDeserializer;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+/**
+ * Checks whether a list with an edit distance threshold can be filtered with a lower bounding on the number
+ * of common list elements. This function returns 'true' if the lower bound on the number of common elements
+ * is positive, 'false' otherwise. For example, this function is used during an indexed nested-loop join based
+ * on edit distance. We partition the tuples of the probing dataset into those that are filterable and those
+ * that are not. Those that are filterable are forwarded to the index. The others are are fed into a (non
+ * indexed) nested-loop join.
+ */
+public class EditDistanceListIsFilterableDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new EditDistanceListIsFilterableDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new EditDistanceListIsFilterableEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE;
+ }
+
+ private static class EditDistanceListIsFilterableEvaluator implements IScalarEvaluator {
+
+ protected final IPointable listPtr = new VoidPointable();
+ protected final IPointable edThreshPtr = new VoidPointable();
+ protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ protected final DataOutput output = resultStorage.getDataOutput();
+
+ protected final IScalarEvaluator listEval;
+ protected final IScalarEvaluator edThreshEval;
+
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ABoolean> booleanSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
+
+ public EditDistanceListIsFilterableEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ listEval = args[0].createScalarEvaluator(context);
+ edThreshEval = args[1].createScalarEvaluator(context);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+
+ listEval.evaluate(tuple, listPtr);
+ edThreshEval.evaluate(tuple, edThreshPtr);
+
+ // Check type and compute string length.
+ byte[] bytes = listPtr.getByteArray();
+ int offset = listPtr.getStartOffset();
+
+ ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(bytes[offset]);
+ long listLen;
+ switch (typeTag) {
+ case MULTISET:
+ listLen = AUnorderedListSerializerDeserializer.getNumberOfItems(bytes, offset);
+ break;
+ case ARRAY:
+ listLen = AOrderedListSerializerDeserializer.getNumberOfItems(bytes, offset);
+ break;
+ default:
+ throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE, 0,
+ ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
+ }
+
+ // Check type and extract edit-distance threshold.
+ bytes = edThreshPtr.getByteArray();
+ offset = edThreshPtr.getStartOffset();
+ long edThresh = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE.getName(),
+ 1, bytes, offset);
+
+ // Compute result.
+ long lowerBound = listLen - edThresh;
+ try {
+ if (lowerBound <= 0) {
+ booleanSerde.serialize(ABoolean.FALSE, output);
+ } else {
+ booleanSerde.serialize(ABoolean.TRUE, output);
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java
new file mode 100644
index 0000000..ddb18fc
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableDescriptor.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Checks whether a string with an edit distance threshold can be filtered with a lower bounding
+ * on number of common grams. This function returns 'true' if the lower bound on the number of
+ * common grams is positive, 'false' otherwise. For example, this function is used during an indexed
+ * nested-loop join based on edit distance. We partition the tuples of the probing dataset into those
+ * that are filterable and those that are not. Those that are filterable are forwarded to the index.
+ * The others are fed into a (non indexed) nested-loop join.
+ */
+public class EditDistanceStringIsFilterableDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new EditDistanceStringIsFilterableDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new EditDistanceStringIsFilterableEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.EDIT_DISTANCE_STRING_IS_FILTERABLE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java
new file mode 100644
index 0000000..0509f51
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/EditDistanceStringIsFilterableEvaluator.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.ABooleanSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class EditDistanceStringIsFilterableEvaluator implements IScalarEvaluator {
+
+ protected final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ protected final DataOutput output = resultStorage.getDataOutput();
+ protected final IPointable stringPtr = new VoidPointable();
+ protected final IPointable edThreshPtr = new VoidPointable();
+ protected final IPointable gramLenPtr = new VoidPointable();
+ protected final IPointable usePrePostPtr = new VoidPointable();
+
+ protected final IScalarEvaluator stringEval;
+ protected final IScalarEvaluator edThreshEval;
+ protected final IScalarEvaluator gramLenEval;
+ protected final IScalarEvaluator usePrePostEval;
+
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ABoolean> booleanSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
+
+ private final UTF8StringPointable utf8Ptr = new UTF8StringPointable();
+
+ public EditDistanceStringIsFilterableEvaluator(IScalarEvaluatorFactory[] args, IHyracksTaskContext context)
+ throws HyracksDataException {
+ stringEval = args[0].createScalarEvaluator(context);
+ edThreshEval = args[1].createScalarEvaluator(context);
+ gramLenEval = args[2].createScalarEvaluator(context);
+ usePrePostEval = args[3].createScalarEvaluator(context);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+
+ stringEval.evaluate(tuple, stringPtr);
+ edThreshEval.evaluate(tuple, edThreshPtr);
+ gramLenEval.evaluate(tuple, gramLenPtr);
+ usePrePostEval.evaluate(tuple, usePrePostPtr);
+
+ // Check type and compute string length.
+ byte typeTag = stringPtr.getByteArray()[stringPtr.getStartOffset()];
+ if (typeTag != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
+ throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_STRING_IS_FILTERABLE, 0, typeTag,
+ ATypeTag.SERIALIZED_STRING_TYPE_TAG);
+ }
+ utf8Ptr.set(stringPtr.getByteArray(), stringPtr.getStartOffset() + 1, stringPtr.getLength());
+ int strLen = utf8Ptr.getStringLength();
+
+ // Check type and extract edit-distance threshold.
+ long edThresh = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE.getName(), 1,
+ edThreshPtr.getByteArray(), edThreshPtr.getStartOffset());
+
+ // Check type and extract gram length.
+ long gramLen = ATypeHierarchy.getIntegerValue(BuiltinFunctions.EDIT_DISTANCE_LIST_IS_FILTERABLE.getName(), 2,
+ gramLenPtr.getByteArray(), gramLenPtr.getStartOffset());
+
+ // Check type and extract usePrePost flag.
+ typeTag = usePrePostPtr.getByteArray()[usePrePostPtr.getStartOffset()];
+ if (typeTag != ATypeTag.SERIALIZED_BOOLEAN_TYPE_TAG) {
+ throw new TypeMismatchException(BuiltinFunctions.EDIT_DISTANCE_STRING_IS_FILTERABLE, 3, typeTag,
+ ATypeTag.SERIALIZED_BOOLEAN_TYPE_TAG);
+ }
+ boolean usePrePost = ABooleanSerializerDeserializer.getBoolean(usePrePostPtr.getByteArray(),
+ usePrePostPtr.getStartOffset() + 1);
+
+ // Compute result.
+ long numGrams = usePrePost ? strLen + gramLen - 1 : strLen - gramLen + 1;
+ long lowerBound = numGrams - edThresh * gramLen;
+ try {
+ if (lowerBound <= 0 || strLen == 0) {
+ booleanSerde.serialize(ABoolean.FALSE, output);
+ } else {
+ booleanSerde.serialize(ABoolean.TRUE, output);
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java
new file mode 100644
index 0000000..190013b
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/GramTokensDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.GramTokensEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.UTF8NGramTokenFactory;
+
+public class GramTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new GramTokensDescriptor();
+ }
+ };
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.GRAM_TOKENS;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ ITokenFactory tokenFactory = new UTF8NGramTokenFactory();
+ NGramUTF8StringBinaryTokenizer tokenizer =
+ new NGramUTF8StringBinaryTokenizer(3, true, true, true, tokenFactory);
+ return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.ASTRING);
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java
new file mode 100644
index 0000000..32dc292
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedGramTokensDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.GramTokensEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8NGramTokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.NGramUTF8StringBinaryTokenizer;
+
+public class HashedGramTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new HashedGramTokensDescriptor();
+ }
+ };
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.HASHED_GRAM_TOKENS;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ ITokenFactory tokenFactory = new HashedUTF8NGramTokenFactory();
+ NGramUTF8StringBinaryTokenizer tokenizer =
+ new NGramUTF8StringBinaryTokenizer(3, true, true, true, tokenFactory);
+ return new GramTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java
new file mode 100644
index 0000000..e224fce
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashedWordTokensDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.WordTokensEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.HashedUTF8WordTokenFactory;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizer;
+import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.ITokenFactory;
+
+public class HashedWordTokensDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new HashedWordTokensDescriptor();
+ }
+ };
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.HASHED_WORD_TOKENS;
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ ITokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
+ IBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, true, tokenFactory);
+ return new WordTokensEvaluator(args, ctx, tokenizer, BuiltinType.AINT32);
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java
new file mode 100644
index 0000000..c9a865b
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenDescriptor.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.common.functions.FunctionConstants;
+import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityFilters;
+import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AMutableInt32;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityFiltersCache;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class PrefixLenDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ private final static FunctionIdentifier FID =
+ new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "prefix-len@3", 3);
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new PrefixLenDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
+
+ return new IScalarEvaluator() {
+
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable inputVal = new VoidPointable();
+ private final IScalarEvaluator evalLen = args[0].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalSimilarity = args[1].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalThreshold = args[2].createScalarEvaluator(ctx);
+
+ private final SimilarityFiltersCache similarityFiltersCache = new SimilarityFiltersCache();
+
+ // result
+ private final AMutableInt32 res = new AMutableInt32(0);
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<AInt32> int32Serde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ // length
+ evalLen.evaluate(tuple, inputVal);
+ byte[] data = inputVal.getByteArray();
+ int offset = inputVal.getStartOffset();
+ if (data[offset] != ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 0, data[offset],
+ ATypeTag.SERIALIZED_INT32_TYPE_TAG);
+ }
+ int length = IntegerPointable.getInteger(data, offset + 1);
+
+ // similarity threshold
+ evalThreshold.evaluate(tuple, inputVal);
+ data = inputVal.getByteArray();
+ offset = inputVal.getStartOffset();
+ if (data[offset] != ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 1, data[offset],
+ ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG);
+ }
+ float similarityThreshold = (float) ADoubleSerializerDeserializer.getDouble(data, offset + 1);
+
+ // similarity name
+ evalSimilarity.evaluate(tuple, inputVal);
+ data = inputVal.getByteArray();
+ offset = inputVal.getStartOffset();
+ int len = inputVal.getLength();
+ if (data[offset] != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 2, data[offset],
+ ATypeTag.SERIALIZED_STRING_TYPE_TAG);
+ }
+ SimilarityFilters similarityFilters =
+ similarityFiltersCache.get(similarityThreshold, data, offset, len);
+
+ int prefixLength = similarityFilters.getPrefixLength(length);
+ res.setValue(prefixLength);
+
+ try {
+ int32Serde.serialize(res, out);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return FID;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java
new file mode 100644
index 0000000..25d4be3
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/PrefixLenJaccardDescriptor.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AFloatSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityFiltersJaccard;
+import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AMutableInt32;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class PrefixLenJaccardDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new PrefixLenJaccardDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
+
+ return new IScalarEvaluator() {
+
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable lenPtr = new VoidPointable();
+ private final IPointable thresholdPtr = new VoidPointable();
+ private final IScalarEvaluator evalLen = args[0].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalThreshold = args[1].createScalarEvaluator(ctx);
+
+ private float similarityThresholdCache;
+ private SimilarityFiltersJaccard similarityFilters;
+
+ // result
+ private final AMutableInt32 res = new AMutableInt32(0);
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<AInt32> int32Serde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ evalLen.evaluate(tuple, lenPtr);
+ evalThreshold.evaluate(tuple, thresholdPtr);
+
+ // length
+ int length = ATypeHierarchy.getIntegerValue(getIdentifier().getName(), 0, lenPtr.getByteArray(),
+ lenPtr.getStartOffset());
+ // similarity threshold
+ byte[] data = thresholdPtr.getByteArray();
+ int offset = thresholdPtr.getStartOffset();
+ if (data[offset] != ATypeTag.SERIALIZED_FLOAT_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 1, data[offset],
+ ATypeTag.SERIALIZED_FLOAT_TYPE_TAG);
+ }
+ float similarityThreshold = AFloatSerializerDeserializer.getFloat(data, offset + 1);
+
+ if (similarityThreshold != similarityThresholdCache || similarityFilters == null) {
+ similarityFilters = new SimilarityFiltersJaccard(similarityThreshold);
+ }
+
+ int prefixLength = similarityFilters.getPrefixLength(length);
+ res.setValue(prefixLength);
+
+ try {
+ int32Serde.serialize(res, out);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ result.set(resultStorage);
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.PREFIX_LEN_JACCARD;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java
new file mode 100644
index 0000000..8584d06
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityDescriptor.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.asterix.common.functions.FunctionConstants;
+import org.apache.asterix.dataflow.data.nontagged.serde.ADoubleSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AOrderedListSerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AUnorderedListSerializerDeserializer;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.fuzzyjoin.IntArray;
+import org.apache.asterix.fuzzyjoin.similarity.PartialIntersect;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityFilters;
+import org.apache.asterix.fuzzyjoin.similarity.SimilarityMetric;
+import org.apache.asterix.om.base.ADouble;
+import org.apache.asterix.om.base.AMutableDouble;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityFiltersCache;
+import org.apache.asterix.runtime.exceptions.TypeMismatchException;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class SimilarityDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ private final static FunctionIdentifier FID =
+ new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "similarity@7", 7);
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SimilarityDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
+
+ return new IScalarEvaluator() {
+
+ private final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
+ private final DataOutput out = resultStorage.getDataOutput();
+ private final IPointable inputVal = new VoidPointable();
+ private final IScalarEvaluator evalLen1 = args[0].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalTokens1 = args[1].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalLen2 = args[2].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalTokens2 = args[3].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalTokenPrefix = args[4].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalSimilarity = args[5].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalThreshold = args[6].createScalarEvaluator(ctx);
+
+ private final SimilarityFiltersCache similarityFiltersCache = new SimilarityFiltersCache();
+
+ private final IntArray tokens1 = new IntArray();
+ private final IntArray tokens2 = new IntArray();
+ private final PartialIntersect parInter = new PartialIntersect();
+
+ // result
+ private final AMutableDouble res = new AMutableDouble(0);
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<ADouble> doubleSerde =
+ SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
+ resultStorage.reset();
+ // similarity threshold
+ evalThreshold.evaluate(tuple, inputVal);
+ byte[] data = inputVal.getByteArray();
+ int offset = inputVal.getStartOffset();
+
+ if (data[offset] != ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 0, data[offset],
+ ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG);
+ }
+ float similarityThreshold = (float) ADoubleSerializerDeserializer.getDouble(data, offset + 1);
+
+ // similarity name
+ evalSimilarity.evaluate(tuple, inputVal);
+ data = inputVal.getByteArray();
+ offset = inputVal.getStartOffset();
+ int len = inputVal.getLength();
+ if (data[offset] != ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 1, data[offset],
+ ATypeTag.SERIALIZED_DOUBLE_TYPE_TAG);
+ }
+ SimilarityFilters similarityFilters =
+ similarityFiltersCache.get(similarityThreshold, data, offset, len);
+
+ evalLen1.evaluate(tuple, inputVal);
+ data = inputVal.getByteArray();
+ offset = inputVal.getStartOffset();
+ if (data[offset] != ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 2, data[offset],
+ ATypeTag.SERIALIZED_INT32_TYPE_TAG);
+ }
+ int length1 = IntegerPointable.getInteger(data, offset + 1);
+
+ evalLen2.evaluate(tuple, inputVal);
+ data = inputVal.getByteArray();
+ offset = inputVal.getStartOffset();
+ if (data[offset] != ATypeTag.SERIALIZED_INT32_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 3, data[offset],
+ ATypeTag.SERIALIZED_INT32_TYPE_TAG);
+ }
+ int length2 = IntegerPointable.getInteger(data, offset + 1);
+
+ float sim = 0;
+
+ //
+ // -- - length filter - --
+ //
+ if (similarityFilters.passLengthFilter(length1, length2)) {
+
+ // -- - tokens1 - --
+ int i;
+ tokens1.reset();
+ evalTokens1.evaluate(tuple, inputVal);
+ byte[] serList = inputVal.getByteArray();
+ offset = inputVal.getStartOffset();
+
+ if (serList[offset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
+ && serList[offset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 4, data[offset],
+ ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG,
+ ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ }
+
+ int lengthTokens1;
+ if (serList[offset] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ lengthTokens1 = AOrderedListSerializerDeserializer.getNumberOfItems(serList, offset);
+ // read tokens
+ for (i = 0; i < lengthTokens1; i++) {
+ int itemOffset =
+ AOrderedListSerializerDeserializer.getItemOffset(serList, offset, i);
+ tokens1.add(IntegerPointable.getInteger(serList, itemOffset));
+ }
+ } else {
+ lengthTokens1 = AUnorderedListSerializerDeserializer.getNumberOfItems(serList, offset);
+ // read tokens
+ for (i = 0; i < lengthTokens1; i++) {
+ int itemOffset =
+ AUnorderedListSerializerDeserializer.getItemOffset(serList, offset, i);
+ tokens1.add(IntegerPointable.getInteger(serList, itemOffset));
+ }
+ }
+ // pad tokens
+ for (; i < length1; i++) {
+ tokens1.add(Integer.MAX_VALUE);
+ }
+
+ // -- - tokens2 - --
+ tokens2.reset();
+ evalTokens2.evaluate(tuple, inputVal);
+ serList = inputVal.getByteArray();
+ offset = inputVal.getStartOffset();
+
+ if (serList[offset] != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG
+ && serList[offset] != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG) {
+ throw new TypeMismatchException(getIdentifier(), 5, data[offset],
+ ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG,
+ ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG);
+ }
+
+ int lengthTokens2;
+ if (serList[0] == ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
+ lengthTokens2 = AOrderedListSerializerDeserializer.getNumberOfItems(serList, offset);
+ // read tokens
+ for (i = 0; i < lengthTokens2; i++) {
+ int itemOffset =
+ AOrderedListSerializerDeserializer.getItemOffset(serList, offset, i);
+ tokens2.add(IntegerPointable.getInteger(serList, itemOffset));
+ }
+ } else {
+ lengthTokens2 = AUnorderedListSerializerDeserializer.getNumberOfItems(serList, offset);
+ // read tokens
+ for (i = 0; i < lengthTokens2; i++) {
+ int itemOffset =
+ AUnorderedListSerializerDeserializer.getItemOffset(serList, offset, i);
+ tokens2.add(IntegerPointable.getInteger(serList, itemOffset));
+ }
+ }
+ // pad tokens
+ for (; i < length2; i++) {
+ tokens2.add(Integer.MAX_VALUE);
+ }
+
+ // -- - token prefix - --
+ evalTokenPrefix.evaluate(tuple, inputVal);
+ int tokenPrefix =
+ IntegerPointable.getInteger(inputVal.getByteArray(), inputVal.getStartOffset() + 1);
+
+ //
+ // -- - position filter - --
+ //
+ SimilarityMetric.getPartialIntersectSize(tokens1.get(), 0, tokens1.length(), tokens2.get(),
+ 0, tokens2.length(), tokenPrefix, parInter);
+ if (similarityFilters.passPositionFilter(parInter.intersectSize, parInter.posXStop, length1,
+ parInter.posYStop, length2)) {
+
+ //
+ // -- - suffix filter - --
+ //
+ if (similarityFilters.passSuffixFilter(tokens1.get(), 0, tokens1.length(),
+ parInter.posXStart, tokens2.get(), 0, tokens2.length(), parInter.posYStart)) {
+
+ sim = similarityFilters.passSimilarityFilter(tokens1.get(), 0, tokens1.length(),
+ parInter.posXStop + 1, tokens2.get(), 0, tokens2.length(),
+ parInter.posYStop + 1, parInter.intersectSize);
+ }
+ }
+ }
+
+ res.setValue(sim);
+
+ try {
+ doubleSerde.serialize(res, out);
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ result.set(resultStorage);
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return FID;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java
new file mode 100644
index 0000000..9fc5b34
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardCheckDescriptor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardCheckEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class SimilarityJaccardCheckDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SimilarityJaccardCheckDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new SimilarityJaccardCheckEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SIMILARITY_JACCARD_CHECK;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/b8307794/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java
new file mode 100644
index 0000000..742f2ff
--- /dev/null
+++ b/asterixdb/asterix-fuzzyjoin/src/main/java/org/apache/asterix/runtime/evaluators/functions/SimilarityJaccardDescriptor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptor;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.SimilarityJaccardEvaluator;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class SimilarityJaccardDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new SimilarityJaccardDescriptor();
+ }
+ };
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
+ return new SimilarityJaccardEvaluator(args, ctx);
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.SIMILARITY_JACCARD;
+ }
+
+}