You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by bu...@apache.org on 2016/08/20 06:15:43 UTC
[01/16] asterixdb git commit: Add Asterix Extension Manager
Repository: asterixdb
Updated Branches:
refs/heads/master a4815d353 -> ab81748ab
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AString.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AString.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AString.java
index 4034d77..a1a89fe 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AString.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/base/AString.java
@@ -18,15 +18,16 @@
*/
package org.apache.asterix.om.base;
-import org.json.JSONException;
-import org.json.JSONObject;
+import java.io.Serializable;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.visitors.IOMVisitor;
+import org.json.JSONException;
+import org.json.JSONObject;
-public class AString implements IAObject {
+public class AString implements IAObject, Serializable {
protected String value;
@@ -50,8 +51,9 @@ public class AString implements IAObject {
@Override
public boolean equals(Object obj) {
- if (!(obj instanceof AString))
+ if (!(obj instanceof AString)) {
return false;
+ }
return value.equals(((AString) obj).getStringValue());
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
index 248ec3a..fc1108f 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/ARecordType.java
@@ -44,8 +44,8 @@ import org.json.JSONObject;
*/
public class ARecordType extends AbstractComplexType {
- public static final ARecordType FULLY_OPEN_RECORD_TYPE =
- new ARecordType("OpenRecord", new String[0], new IAType[0], true);
+ public static final ARecordType FULLY_OPEN_RECORD_TYPE = new ARecordType("OpenRecord", new String[0], new IAType[0],
+ true);
private static final long serialVersionUID = 1L;
private final String[] fieldNames;
@@ -369,4 +369,14 @@ public class ARecordType extends AbstractComplexType {
}
return false;
}
+
+ /**
+ * Create a fully open record type with the passed name
+ *
+ * @param name
+ * @return
+ */
+ public static ARecordType createOpenRecordType(String name) {
+ return new ARecordType(name, new String[0], new IAType[0], true);
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java
index bb1e554..bf103fc 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/util/AsterixAppContextInfo.java
@@ -18,11 +18,13 @@
*/
package org.apache.asterix.om.util;
+import java.io.IOException;
import java.util.logging.Logger;
import org.apache.asterix.common.cluster.IGlobalRecoveryMaanger;
import org.apache.asterix.common.config.AsterixBuildProperties;
import org.apache.asterix.common.config.AsterixCompilerProperties;
+import org.apache.asterix.common.config.AsterixExtensionProperties;
import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.common.config.AsterixFeedProperties;
import org.apache.asterix.common.config.AsterixMetadataProperties;
@@ -59,12 +61,15 @@ public class AsterixAppContextInfo implements IAsterixApplicationContextInfo, IA
private AsterixFeedProperties feedProperties;
private AsterixBuildProperties buildProperties;
private AsterixReplicationProperties replicationProperties;
+ private AsterixExtensionProperties extensionProperties;
private final IGlobalRecoveryMaanger globalRecoveryMaanger;
private IHyracksClientConnection hcc;
private final ILibraryManager libraryManager;
+ private Object extensionManager;
public static void initialize(ICCApplicationContext ccAppCtx, IHyracksClientConnection hcc,
- IGlobalRecoveryMaanger globalRecoveryMaanger, ILibraryManager libraryManager) throws AsterixException {
+ IGlobalRecoveryMaanger globalRecoveryMaanger, ILibraryManager libraryManager)
+ throws AsterixException, IOException {
if (INSTANCE != null) {
return;
}
@@ -86,8 +91,9 @@ public class AsterixAppContextInfo implements IAsterixApplicationContextInfo, IA
INSTANCE.storageProperties = new AsterixStorageProperties(propertiesAccessor);
INSTANCE.txnProperties = new AsterixTransactionProperties(propertiesAccessor);
INSTANCE.feedProperties = new AsterixFeedProperties(propertiesAccessor);
- INSTANCE.replicationProperties = new AsterixReplicationProperties(propertiesAccessor,
- AsterixClusterProperties.INSTANCE.getCluster());
+ INSTANCE.extensionProperties = new AsterixExtensionProperties(propertiesAccessor);
+ INSTANCE.replicationProperties =
+ new AsterixReplicationProperties(propertiesAccessor, AsterixClusterProperties.INSTANCE.getCluster());
INSTANCE.hcc = hcc;
INSTANCE.buildProperties = new AsterixBuildProperties(propertiesAccessor);
Logger.getLogger("org.apache").setLevel(INSTANCE.externalProperties.getLogLevel());
@@ -173,4 +179,16 @@ public class AsterixAppContextInfo implements IAsterixApplicationContextInfo, IA
public ILibraryManager getLibraryManager() {
return libraryManager;
}
+
+ public Object getExtensionManager() {
+ return extensionManager;
+ }
+
+ public void setExtensionManager(Object extensionManager) {
+ this.extensionManager = extensionManager;
+ }
+
+ public AsterixExtensionProperties getExtensionProperties() {
+ return extensionProperties;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/job/listener/MultiTransactionJobletEventListenerFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/job/listener/MultiTransactionJobletEventListenerFactory.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/job/listener/MultiTransactionJobletEventListenerFactory.java
new file mode 100644
index 0000000..2d23058
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/job/listener/MultiTransactionJobletEventListenerFactory.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.job.listener;
+
+import java.util.List;
+
+import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.transactions.DatasetId;
+import org.apache.asterix.common.transactions.ITransactionContext;
+import org.apache.asterix.common.transactions.ITransactionManager;
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.hyracks.api.context.IHyracksJobletContext;
+import org.apache.hyracks.api.job.IJobletEventListener;
+import org.apache.hyracks.api.job.IJobletEventListenerFactory;
+import org.apache.hyracks.api.job.JobStatus;
+
+/**
+ * This Joblet enable transactions on multiple datasets to take place in the same Hyracks Job
+ * It takes a list of Transaction job ids instead of a single job Id
+ */
+public class MultiTransactionJobletEventListenerFactory implements IJobletEventListenerFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final List<JobId> jobIds;
+ private final boolean transactionalWrite;
+
+ public MultiTransactionJobletEventListenerFactory(List<JobId> jobIds, boolean transactionalWrite) {
+ this.jobIds = jobIds;
+ this.transactionalWrite = transactionalWrite;
+ }
+
+ @Override
+ public IJobletEventListener createListener(final IHyracksJobletContext jobletContext) {
+
+ return new IJobletEventListener() {
+ @Override
+ public void jobletFinish(JobStatus jobStatus) {
+ try {
+ ITransactionManager txnManager =
+ ((IAsterixAppRuntimeContext) jobletContext.getApplicationContext().getApplicationObject())
+ .getTransactionSubsystem().getTransactionManager();
+ for (JobId jobId : jobIds) {
+ ITransactionContext txnContext = txnManager.getTransactionContext(jobId, false);
+ txnContext.setWriteTxn(transactionalWrite);
+ txnManager.completedTransaction(txnContext, new DatasetId(-1), -1,
+ !(jobStatus == JobStatus.FAILURE));
+ }
+ } catch (ACIDException e) {
+ throw new Error(e);
+ }
+ }
+
+ @Override
+ public void jobletStart() {
+ try {
+ for (JobId jobId : jobIds) {
+ ((IAsterixAppRuntimeContext) jobletContext.getApplicationContext().getApplicationObject())
+ .getTransactionSubsystem().getTransactionManager().getTransactionContext(jobId, true);
+ }
+ } catch (ACIDException e) {
+ throw new Error(e);
+ }
+ }
+
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java
index 696726f..003137d 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/server/test/NCServiceExecutionIT.java
@@ -18,6 +18,11 @@
*/
package org.apache.asterix.server.test;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.logging.Logger;
+
import org.apache.asterix.test.aql.TestExecutor;
import org.apache.asterix.test.runtime.HDFSCluster;
import org.apache.asterix.testframework.context.TestCaseContext;
@@ -34,11 +39,6 @@ import org.junit.runners.MethodSorters;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.logging.Logger;
-
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@RunWith(Parameterized.class)
public class NCServiceExecutionIT {
@@ -47,7 +47,7 @@ public class NCServiceExecutionIT {
// The "target" subdirectory of asterix-server. All outputs go here.
private static final String TARGET_DIR = StringUtils
- .join(new String[] { System.getProperty("basedir"), "target" }, File.separator);
+ .join(new String[] { "target" }, File.separator);
// Directory where the NCs create and store all data, as configured by
// src/test/resources/NCServiceExecutionIT/cc.conf.
@@ -75,7 +75,7 @@ public class NCServiceExecutionIT {
// paths in "load" statements in test queries to find the right data. It is
// also used for HDFSCluster.
private static final String ASTERIX_APP_DIR = StringUtils
- .join(new String[] { System.getProperty("basedir"), "..", "asterix-app" },
+ .join(new String[] { "..", "asterix-app" },
File.separator);
// Path to the actual AQL test files, which we borrow from asterix-app. This is
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index a03bc30..7285a5a 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -356,6 +356,19 @@
<ignore />
</action>
</pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>javacc-maven-plugin</artifactId>
+ <versionRange>[2.6,)</versionRange>
+ <goals>
+ <goal>jjdoc</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
</pluginExecutions>
</lifecycleMappingMetadata>
</configuration>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
index 2f2b7e3..afcb86e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
@@ -20,7 +20,7 @@ package org.apache.hyracks.algebricks.core.algebra.functions;
import java.io.Serializable;
-public final class FunctionIdentifier implements Serializable {
+public class FunctionIdentifier implements Serializable {
private static final long serialVersionUID = 1L;
private final String namespace;
@@ -60,6 +60,7 @@ public final class FunctionIdentifier implements Serializable {
return name.hashCode() + namespace.hashCode();
}
+ @Override
public String toString() {
return getNamespace() + ":" + name;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java
index bd40813..278a9d7 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/application/IApplicationConfig.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.api.application;
+import java.util.List;
+import java.util.Map.Entry;
import java.util.Set;
/**
@@ -25,11 +27,22 @@ import java.util.Set;
*/
public interface IApplicationConfig {
String getString(String section, String key);
+
String getString(String section, String key, String defaultValue);
+
int getInt(String section, String key);
+
int getInt(String section, String key, int defaultValue);
+
long getLong(String section, String key);
+
long getLong(String section, String key, long defaultValue);
+
Set<String> getSections();
+
Set<String> getKeys(String section);
+
+ String[] getStringArray(String section, String key);
+
+ List<Set<Entry<String, String>>> getMultiSections(String section);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ActivityId.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ActivityId.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ActivityId.java
index acac345..214627e 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ActivityId.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/ActivityId.java
@@ -49,13 +49,17 @@ public final class ActivityId implements IWritable, Serializable {
return odId;
}
+ public void setOperatorDescriptorId(OperatorDescriptorId odId) {
+ this.odId = odId;
+ }
+
public int getLocalId() {
return id;
}
@Override
public int hashCode() {
- return (int) (odId.hashCode() + id);
+ return odId.hashCode() + id;
}
@Override
@@ -70,6 +74,7 @@ public final class ActivityId implements IWritable, Serializable {
return other.odId.equals(odId) && other.id == id;
}
+ @Override
public String toString() {
return "ANID:" + odId + ":" + id;
}
@@ -78,8 +83,8 @@ public final class ActivityId implements IWritable, Serializable {
if (str.startsWith("ANID:")) {
str = str.substring(5);
int idIdx = str.lastIndexOf(':');
- return new ActivityId(OperatorDescriptorId.parse(str.substring(0, idIdx)), Integer.parseInt(str
- .substring(idIdx + 1)));
+ return new ActivityId(OperatorDescriptorId.parse(str.substring(0, idIdx)),
+ Integer.parseInt(str.substring(idIdx + 1)));
}
throw new IllegalArgumentException("Unable to parse: " + str);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IConnectorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IConnectorDescriptor.java
index 339eb9d..7219040 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IConnectorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IConnectorDescriptor.java
@@ -68,7 +68,7 @@ public interface IConnectorDescriptor extends Serializable {
*/
public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
- throws HyracksDataException;
+ throws HyracksDataException;
/**
* Factory metod to create the receive side reader that reads data from this
@@ -136,4 +136,9 @@ public interface IConnectorDescriptor extends Serializable {
* @throws JSONException
*/
public JSONObject toJSON() throws JSONException;
+
+ /**
+ * Sets the connector Id
+ */
+ public void setConnectorId(ConnectorDescriptorId cdId);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IOperatorDescriptor.java
index 1b85fbd..26561e6 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataflow/IOperatorDescriptor.java
@@ -20,12 +20,11 @@ package org.apache.hyracks.api.dataflow;
import java.io.Serializable;
-import org.json.JSONException;
-import org.json.JSONObject;
-
import org.apache.hyracks.api.application.ICCApplicationContext;
import org.apache.hyracks.api.constraints.IConstraintAcceptor;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.json.JSONException;
+import org.json.JSONObject;
/**
* Descriptor for operators in Hyracks.
@@ -41,6 +40,13 @@ public interface IOperatorDescriptor extends Serializable {
public OperatorDescriptorId getOperatorId();
/**
+ * Sets the id of the operator.
+ *
+ * @param id
+ */
+ void setOperatorId(OperatorDescriptorId id);
+
+ /**
* Returns the number of inputs into this operator.
*
* @return Number of inputs.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index e4385bf..c95c31c 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -26,6 +26,7 @@ public class ErrorCode {
public static final int ERROR_PROCESSING_TUPLE = 0;
public static final int INVALID_OPERATOR_OPERATION = 1;
public static final int FAILURE_ON_NODE = 2;
+ public static final int ILLEGAL_ARGUMENT = 3;
private ErrorCode() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
index 5165a62..7b44ff5 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
@@ -28,10 +28,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
import org.apache.hyracks.api.constraints.Constraint;
import org.apache.hyracks.api.constraints.expressions.ConstantExpression;
import org.apache.hyracks.api.constraints.expressions.PartitionCountExpression;
@@ -43,6 +39,9 @@ import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.dataset.ResultSetId;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
public class JobSpecification implements Serializable, IOperatorDescriptorRegistry, IConnectorDescriptorRegistry {
private static final long serialVersionUID = 1L;
@@ -98,7 +97,7 @@ public class JobSpecification implements Serializable, IOperatorDescriptorRegist
connMap = new HashMap<ConnectorDescriptorId, IConnectorDescriptor>();
opInputMap = new HashMap<OperatorDescriptorId, List<IConnectorDescriptor>>();
opOutputMap = new HashMap<OperatorDescriptorId, List<IConnectorDescriptor>>();
- connectorOpMap = new HashMap<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>>();
+ connectorOpMap = new HashMap<>();
properties = new HashMap<String, Serializable>();
userConstraints = new HashSet<Constraint>();
operatorIdCounter = 0;
@@ -112,6 +111,7 @@ public class JobSpecification implements Serializable, IOperatorDescriptorRegist
@Override
public OperatorDescriptorId createOperatorDescriptorId(IOperatorDescriptor op) {
OperatorDescriptorId odId = new OperatorDescriptorId(operatorIdCounter++);
+ op.setOperatorId(odId);
opMap.put(odId, op);
return odId;
}
@@ -119,6 +119,7 @@ public class JobSpecification implements Serializable, IOperatorDescriptorRegist
@Override
public ConnectorDescriptorId createConnectorDescriptor(IConnectorDescriptor conn) {
ConnectorDescriptorId cdId = new ConnectorDescriptorId(connectorIdCounter++);
+ conn.setConnectorId(cdId);
connMap.put(cdId, conn);
return cdId;
}
@@ -135,8 +136,7 @@ public class JobSpecification implements Serializable, IOperatorDescriptorRegist
IOperatorDescriptor consumerOp, int consumerPort) {
insertIntoIndexedMap(opInputMap, consumerOp.getOperatorId(), consumerPort, conn);
insertIntoIndexedMap(opOutputMap, producerOp.getOperatorId(), producerPort, conn);
- connectorOpMap.put(
- conn.getConnectorId(),
+ connectorOpMap.put(conn.getConnectorId(),
Pair.<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> of(
Pair.<IOperatorDescriptor, Integer> of(producerOp, producerPort),
Pair.<IOperatorDescriptor, Integer> of(consumerOp, consumerPort)));
@@ -166,20 +166,20 @@ public class JobSpecification implements Serializable, IOperatorDescriptorRegist
}
public RecordDescriptor getConnectorRecordDescriptor(IConnectorDescriptor conn) {
- Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
- .getConnectorId());
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap
+ .get(conn.getConnectorId());
return connInfo.getLeft().getLeft().getOutputRecordDescriptors()[connInfo.getLeft().getRight()];
}
public IOperatorDescriptor getConsumer(IConnectorDescriptor conn) {
- Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
- .getConnectorId());
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap
+ .get(conn.getConnectorId());
return connInfo.getRight().getLeft();
}
public int getConsumerInputIndex(IConnectorDescriptor conn) {
- Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
- .getConnectorId());
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap
+ .get(conn.getConnectorId());
return connInfo.getRight().getRight();
}
@@ -220,14 +220,14 @@ public class JobSpecification implements Serializable, IOperatorDescriptorRegist
}
public IOperatorDescriptor getProducer(IConnectorDescriptor conn) {
- Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
- .getConnectorId());
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap
+ .get(conn.getConnectorId());
return connInfo.getLeft().getLeft();
}
public int getProducerOutputIndex(IConnectorDescriptor conn) {
- Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
- .getConnectorId());
+ Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap
+ .get(conn.getConnectorId());
return connInfo.getLeft().getRight();
}
@@ -313,6 +313,7 @@ public class JobSpecification implements Serializable, IOperatorDescriptorRegist
vList.set(index, value);
}
+ @Override
public String toString() {
StringBuilder buffer = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java
index 22fe318..53db11c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/application/IniApplicationConfig.java
@@ -18,11 +18,15 @@
*/
package org.apache.hyracks.control.common.application;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hyracks.api.application.IApplicationConfig;
import org.apache.hyracks.control.common.controllers.IniUtils;
import org.ini4j.Ini;
-
-import java.util.Set;
+import org.ini4j.Profile.Section;
/**
* An implementation of IApplicationConfig which is backed by Ini4j.
@@ -49,6 +53,11 @@ public class IniApplicationConfig implements IApplicationConfig {
}
@Override
+ public String[] getStringArray(String section, String key) {
+ return IniUtils.getStringArray(ini, section, key);
+ }
+
+ @Override
public int getInt(String section, String key) {
return IniUtils.getInt(ini, section, key, 0);
}
@@ -60,7 +69,7 @@ public class IniApplicationConfig implements IApplicationConfig {
@Override
public long getLong(String section, String key) {
- return IniUtils.getLong(ini, section, key, (long) 0);
+ return IniUtils.getLong(ini, section, key, 0);
}
@Override
@@ -77,4 +86,20 @@ public class IniApplicationConfig implements IApplicationConfig {
public Set<String> getKeys(String section) {
return ini.get(section).keySet();
}
+
+ @Override
+ public List<Set<Map.Entry<String, String>>> getMultiSections(String section) {
+ List<Set<Map.Entry<String, String>>> list = new ArrayList<>();
+ List<Section> secs = getMulti(section);
+ if (secs != null) {
+ for (Section sec : secs) {
+ list.add(sec.entrySet());
+ }
+ }
+ return list;
+ }
+
+ private List<Section> getMulti(String section) {
+ return ini.getAll(section);
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java
index 538bb0b..c6c3e73 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/IniUtils.java
@@ -18,11 +18,13 @@
*/
package org.apache.hyracks.control.common.controllers;
-import org.ini4j.Ini;
-
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.Array;
+
+import org.ini4j.Ini;
+import org.ini4j.Profile.Section;
/**
* Some utility functions for reading Ini4j objects with default values.
@@ -49,10 +51,27 @@ public class IniUtils {
return (value != null) ? value : default_value;
}
+ @SuppressWarnings("unchecked")
+ private static <T> T getIniArray(Ini ini, String section, String key, Class<T> clazz) {
+ Section sec = ini.get(section);
+ if (clazz.getComponentType() == null) {
+ return null;
+ }
+ if (sec == null) {
+ return (T) Array.newInstance(clazz.getComponentType(), 0);
+ } else {
+ return sec.getAll(key, clazz);
+ }
+ }
+
public static String getString(Ini ini, String section, String key, String defaultValue) {
return getIniValue(ini, section, key, defaultValue, String.class);
}
+ public static String[] getStringArray(Ini ini, String section, String key) {
+ return getIniArray(ini, section, key, String[].class);
+ }
+
public static int getInt(Ini ini, String section, String key, int defaultValue) {
return getIniValue(ini, section, key, defaultValue, Integer.class);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
index 5caf477..e780ea0 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
@@ -18,19 +18,18 @@
*/
package org.apache.hyracks.dataflow.std.base;
-import org.json.JSONException;
-import org.json.JSONObject;
-
import org.apache.hyracks.api.application.ICCApplicationContext;
import org.apache.hyracks.api.constraints.IConstraintAcceptor;
import org.apache.hyracks.api.dataflow.ConnectorDescriptorId;
import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.job.ActivityCluster;
import org.apache.hyracks.api.job.IConnectorDescriptorRegistry;
+import org.json.JSONException;
+import org.json.JSONObject;
public abstract class AbstractConnectorDescriptor implements IConnectorDescriptor {
private static final long serialVersionUID = 1L;
- protected final ConnectorDescriptorId id;
+ protected ConnectorDescriptorId id;
protected String displayName;
@@ -39,14 +38,17 @@ public abstract class AbstractConnectorDescriptor implements IConnectorDescripto
displayName = getClass().getName() + "[" + id + "]";
}
+ @Override
public ConnectorDescriptorId getConnectorId() {
return id;
}
+ @Override
public String getDisplayName() {
return displayName;
}
+ @Override
public void setDisplayName(String displayName) {
this.displayName = displayName;
}
@@ -67,4 +69,9 @@ public abstract class AbstractConnectorDescriptor implements IConnectorDescripto
ICCApplicationContext appCtx) {
// do nothing
}
+
+ @Override
+ public void setConnectorId(ConnectorDescriptorId id) {
+ this.id = id;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
index e930a2b..4f22a17 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
@@ -18,20 +18,19 @@
*/
package org.apache.hyracks.dataflow.std.base;
-import org.json.JSONException;
-import org.json.JSONObject;
-
import org.apache.hyracks.api.application.ICCApplicationContext;
import org.apache.hyracks.api.constraints.IConstraintAcceptor;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.json.JSONException;
+import org.json.JSONObject;
public abstract class AbstractOperatorDescriptor implements IOperatorDescriptor {
private static final long serialVersionUID = 1L;
- protected final OperatorDescriptorId odId;
+ protected OperatorDescriptorId odId;
protected String[] partitions;
@@ -57,6 +56,11 @@ public abstract class AbstractOperatorDescriptor implements IOperatorDescriptor
}
@Override
+ public void setOperatorId(OperatorDescriptorId id) {
+ this.odId = id;
+ }
+
+ @Override
public int getInputArity() {
return inputArity;
}
@@ -71,10 +75,12 @@ public abstract class AbstractOperatorDescriptor implements IOperatorDescriptor
return recordDescriptors;
}
+ @Override
public String getDisplayName() {
return displayName;
}
+ @Override
public void setDisplayName(String displayName) {
this.displayName = displayName;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java
index 775c367..9437b00 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractSingleActivityOperatorDescriptor.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.dataflow.std.base;
import org.apache.hyracks.api.dataflow.ActivityId;
import org.apache.hyracks.api.dataflow.IActivity;
import org.apache.hyracks.api.dataflow.IActivityGraphBuilder;
+import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
public abstract class AbstractSingleActivityOperatorDescriptor extends AbstractOperatorDescriptor implements IActivity {
@@ -39,6 +40,14 @@ public abstract class AbstractSingleActivityOperatorDescriptor extends AbstractO
}
@Override
+ public final void setOperatorId(OperatorDescriptorId id) {
+ super.setOperatorId(id);
+ if (activityNodeId != null && !activityNodeId.getOperatorDescriptorId().equals(odId)) {
+ activityNodeId.setOperatorDescriptorId(odId);
+ }
+ }
+
+ @Override
public final void contributeActivities(IActivityGraphBuilder builder) {
builder.addActivity(this, this);
for (int i = 0; i < getInputArity(); ++i) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
index 44d77ac..b1cd83e 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
@@ -60,7 +60,7 @@ public class LocalityAwareMToNPartitioningConnectorDescriptor extends AbstractMT
@Override
public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
return new LocalityAwarePartitionDataWriter(ctx, edwFactory, recordDesc, tpcf.createPartitioner(),
nConsumerPartitions, localityMap, index);
}
@@ -78,8 +78,9 @@ public class LocalityAwareMToNPartitioningConnectorDescriptor extends AbstractMT
int receiverIndex, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
BitSet expectedPartitions = new BitSet(nProducerPartitions);
for (int i = 0; i < nProducerPartitions; i++) {
- if (localityMap.isConnected(i, receiverIndex, nConsumerPartitions))
+ if (localityMap.isConnected(i, receiverIndex, nConsumerPartitions)) {
expectedPartitions.set(i);
+ }
}
NonDeterministicChannelReader channelReader = new NonDeterministicChannelReader(nProducerPartitions,
expectedPartitions);
@@ -87,5 +88,4 @@ public class LocalityAwareMToNPartitioningConnectorDescriptor extends AbstractMT
return new PartitionCollector(ctx, getConnectorId(), receiverIndex, expectedPartitions, frameReader,
channelReader);
}
-
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java
index 2ca70da..d748c8e 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java
@@ -34,16 +34,17 @@ import org.apache.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
import org.apache.hyracks.dataflow.std.collectors.PartitionCollector;
public class MToNBroadcastConnectorDescriptor extends AbstractMToNConnectorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
public MToNBroadcastConnectorDescriptor(IConnectorDescriptorRegistry spec) {
super(spec);
}
- private static final long serialVersionUID = 1L;
-
@Override
public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
final IFrameWriter[] epWriters = new IFrameWriter[nConsumerPartitions];
final boolean[] isOpen = new boolean[nConsumerPartitions];
for (int i = 0; i < nConsumerPartitions; ++i) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
index 4872b95..d26b9ef 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
@@ -45,7 +45,7 @@ public class MToNPartitioningConnectorDescriptor extends AbstractMToNConnectorDe
@Override
public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
return new PartitionDataWriter(ctx, nConsumerPartitions, edwFactory, recordDesc, tpcf.createPartitioner());
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
index 04de894..edcad42 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
@@ -69,14 +69,14 @@ public class MToNPartitioningMergingConnectorDescriptor extends AbstractMToNConn
public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
throws HyracksDataException {
- final PartitionDataWriter hashWriter = new PartitionDataWriter(ctx, nConsumerPartitions, edwFactory,
- recordDesc, tpcf.createPartitioner());
+ final PartitionDataWriter hashWriter = new PartitionDataWriter(ctx, nConsumerPartitions, edwFactory, recordDesc,
+ tpcf.createPartitioner());
return hashWriter;
}
@Override
- public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
- int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
+ public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc, int index,
+ int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
index e0c886f..f773918 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
@@ -48,7 +48,7 @@ public class OneToOneConnectorDescriptor extends AbstractConnectorDescriptor {
@Override
public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
- throws HyracksDataException {
+ throws HyracksDataException {
return edwFactory.createFrameWriter(index);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
index 56af78e..67af861 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
@@ -80,16 +80,16 @@ public class SplitOperatorDescriptor extends AbstractOperatorDescriptor {
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
- SplitterMaterializerActivityNode sma = new SplitterMaterializerActivityNode(
- new ActivityId(odId, SPLITTER_MATERIALIZER_ACTIVITY_ID));
+ SplitterMaterializerActivityNode sma =
+ new SplitterMaterializerActivityNode(new ActivityId(odId, SPLITTER_MATERIALIZER_ACTIVITY_ID));
builder.addActivity(this, sma);
builder.addSourceEdge(0, sma, 0);
int pipelineOutputIndex = 0;
int activityId = MATERIALIZE_READER_ACTIVITY_ID;
for (int i = 0; i < outputArity; i++) {
if (outputMaterializationFlags[i]) {
- MaterializeReaderActivityNode mra = new MaterializeReaderActivityNode(
- new ActivityId(odId, activityId++));
+ MaterializeReaderActivityNode mra =
+ new MaterializeReaderActivityNode(new ActivityId(odId, activityId++));
builder.addActivity(this, mra);
builder.addBlockingEdge(sma, mra);
builder.addTargetEdge(i, mra, 0);
@@ -139,6 +139,15 @@ public class SplitOperatorDescriptor extends AbstractOperatorDescriptor {
}
@Override
+ public void flush() throws HyracksDataException {
+ if (!requiresMaterialization) {
+ for (IFrameWriter writer : writers) {
+ writer.flush();
+ }
+ }
+ }
+
+ @Override
public void close() throws HyracksDataException {
HyracksDataException hde = null;
try {
@@ -218,4 +227,5 @@ public class SplitOperatorDescriptor extends AbstractOperatorDescriptor {
};
}
}
+
}
[06/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index 9786ebf..70da097 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -31,6 +31,8 @@ import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.metadata.api.IAsterixStateProxy;
+import org.apache.asterix.metadata.api.IExtensionMetadataEntity;
+import org.apache.asterix.metadata.api.IExtensionMetadataSearchKey;
import org.apache.asterix.metadata.api.IMetadataManager;
import org.apache.asterix.metadata.api.IMetadataNode;
import org.apache.asterix.metadata.entities.CompactionPolicy;
@@ -944,4 +946,38 @@ public class MetadataManager implements IMetadataManager {
}
return dataset;
}
+
+ @Override
+ public <T extends IExtensionMetadataEntity> void addEntity(MetadataTransactionContext mdTxnCtx, T entity)
+ throws MetadataException {
+ try {
+ metadataNode.addEntity(mdTxnCtx.getJobId(), entity);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
+ }
+
+ @Override
+ public <T extends IExtensionMetadataEntity> void deleteEntity(MetadataTransactionContext mdTxnCtx, T entity)
+ throws MetadataException {
+ try {
+ metadataNode.deleteEntity(mdTxnCtx.getJobId(), entity);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
+ }
+
+ @Override
+ public <T extends IExtensionMetadataEntity> List<T> getEntities(MetadataTransactionContext mdTxnCtx,
+ IExtensionMetadataSearchKey searchKey) throws MetadataException {
+ try {
+ return metadataNode.getEntities(mdTxnCtx.getJobId(), searchKey);
+ } catch (RemoteException e) {
+ throw new MetadataException(e);
+ }
+ }
+
+ public static synchronized void instantiate(MetadataManager metadataManager) {
+ MetadataManager.INSTANCE = metadataManager;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 2cd8c46..383039e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -23,7 +23,9 @@ import java.io.IOException;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
@@ -41,6 +43,12 @@ import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.asterix.common.transactions.JobId;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.metadata.api.ExtensionMetadataDataset;
+import org.apache.asterix.metadata.api.ExtensionMetadataDatasetId;
+import org.apache.asterix.metadata.api.IExtensionMetadataEntity;
+import org.apache.asterix.metadata.api.IExtensionMetadataSearchKey;
+import org.apache.asterix.metadata.api.IMetadataEntityTupleTranslator;
+import org.apache.asterix.metadata.api.IMetadataExtension;
import org.apache.asterix.metadata.api.IMetadataIndex;
import org.apache.asterix.metadata.api.IMetadataNode;
import org.apache.asterix.metadata.api.IValueExtractor;
@@ -70,6 +78,7 @@ import org.apache.asterix.metadata.entitytupletranslators.FeedTupleTranslator;
import org.apache.asterix.metadata.entitytupletranslators.FunctionTupleTranslator;
import org.apache.asterix.metadata.entitytupletranslators.IndexTupleTranslator;
import org.apache.asterix.metadata.entitytupletranslators.LibraryTupleTranslator;
+import org.apache.asterix.metadata.entitytupletranslators.MetadataTupleTranslatorProvider;
import org.apache.asterix.metadata.entitytupletranslators.NodeGroupTupleTranslator;
import org.apache.asterix.metadata.entitytupletranslators.NodeTupleTranslator;
import org.apache.asterix.metadata.valueextractors.MetadataEntityValueExtractor;
@@ -113,11 +122,16 @@ public class MetadataNode implements IMetadataNode {
private static final long serialVersionUID = 1L;
private static final DatasetId METADATA_DATASET_ID = new DatasetId(
- MetadataIndexImmutableProperties.METADATA.getDatasetId());
+ MetadataPrimaryIndexes.PROPERTIES_METADATA.getDatasetId());
+ // shared between core and extension
private IDatasetLifecycleManager datasetLifecycleManager;
private ITransactionSubsystem transactionSubsystem;
private int metadataStoragePartition;
+ // core only
+ private transient MetadataTupleTranslatorProvider tupleTranslatorProvider;
+ // extension only
+ private Map<ExtensionMetadataDatasetId, ExtensionMetadataDataset<?>> extensionDatasets;
public static final MetadataNode INSTANCE = new MetadataNode();
@@ -125,11 +139,21 @@ public class MetadataNode implements IMetadataNode {
super();
}
- public void initialize(IAsterixAppRuntimeContext runtimeContext) {
+ public void initialize(IAsterixAppRuntimeContext runtimeContext,
+ MetadataTupleTranslatorProvider tupleTranslatorProvider, List<IMetadataExtension> metadataExtensions) {
+ this.tupleTranslatorProvider = tupleTranslatorProvider;
this.transactionSubsystem = runtimeContext.getTransactionSubsystem();
this.datasetLifecycleManager = runtimeContext.getDatasetLifecycleManager();
this.metadataStoragePartition = ((IAsterixPropertiesProvider) runtimeContext).getMetadataProperties()
.getMetadataPartition().getPartitionId();
+ if (metadataExtensions != null) {
+ extensionDatasets = new HashMap<>();
+ for (IMetadataExtension metadataExtension : metadataExtensions) {
+ for (ExtensionMetadataDataset<?> extensionIndex : metadataExtension.getExtensionIndexes()) {
+ extensionDatasets.put(extensionIndex.getId(), extensionIndex);
+ }
+ }
+ }
}
@Override
@@ -168,10 +192,112 @@ public class MetadataNode implements IMetadataNode {
transactionSubsystem.getLockManager().unlock(METADATA_DATASET_ID, -1, lockMode, txnCtx);
}
+ // TODO(amoudi): make all metadata operations go through the generic methods
+ /**
+ * Add entity to index
+ *
+ * @param jobId
+ * @param entity
+ * @param tupleTranslator
+ * @param index
+ * @throws MetadataException
+ */
+ private <T> void addEntity(JobId jobId, T entity, IMetadataEntityTupleTranslator<T> tupleTranslator,
+ IMetadataIndex index) throws MetadataException {
+ try {
+ ITupleReference tuple = tupleTranslator.getTupleFromMetadataEntity(entity);
+ insertTupleIntoIndex(jobId, index, tuple);
+ } catch (TreeIndexDuplicateKeyException e) {
+ throw new MetadataException(entity.toString() + " already exists.", e);
+ } catch (ACIDException | IndexException | IOException e) {
+ throw new MetadataException(e);
+ }
+ }
+
+ /**
+ * Delete entity from index
+ *
+ * @param jobId
+ * @param entity
+ * @param tupleTranslator
+ * @param index
+ * @throws MetadataException
+ */
+ private <T> void deleteEntity(JobId jobId, T entity, IMetadataEntityTupleTranslator<T> tupleTranslator,
+ IMetadataIndex index) throws MetadataException {
+ try {
+ ITupleReference tuple = tupleTranslator.getTupleFromMetadataEntity(entity);
+ deleteTupleFromIndex(jobId, index, tuple);
+ } catch (TreeIndexDuplicateKeyException e) {
+ throw new MetadataException(entity.toString() + " already exists.", e);
+ } catch (ACIDException | IndexException | IOException e) {
+ throw new MetadataException(e);
+ }
+ }
+
+ /**
+ * retrieve all entities that matches the searchKey
+ *
+ * @param jobId
+ * @param searchKey
+ * @param tupleTranslator
+ * @param index
+ * @return
+ * @throws MetadataException
+ */
+ private <T> List<T> getEntities(JobId jobId, ITupleReference searchKey,
+ IMetadataEntityTupleTranslator<T> tupleTranslator, IMetadataIndex index) throws MetadataException {
+ try {
+ IValueExtractor<T> valueExtractor = new MetadataEntityValueExtractor<>(tupleTranslator);
+ List<T> results = new ArrayList<>();
+ searchIndex(jobId, index, searchKey, valueExtractor, results);
+ return results;
+ } catch (IndexException | IOException e) {
+ throw new MetadataException(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T extends IExtensionMetadataEntity> void addEntity(JobId jobId, T entity)
+ throws MetadataException, RemoteException {
+ ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(entity.getDatasetId());
+ if (index == null) {
+ throw new MetadataException("Metadata Extension Index: " + entity.getDatasetId() + " was not found");
+ }
+ IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator();
+ addEntity(jobId, entity, tupleTranslator, index);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T extends IExtensionMetadataEntity> void deleteEntity(JobId jobId, T entity)
+ throws MetadataException, RemoteException {
+ ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(entity.getDatasetId());
+ if (index == null) {
+ throw new MetadataException("Metadata Extension Index: " + entity.getDatasetId() + " was not found");
+ }
+ IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator();
+ deleteEntity(jobId, entity, tupleTranslator, index);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T extends IExtensionMetadataEntity> List<T> getEntities(JobId jobId, IExtensionMetadataSearchKey searchKey)
+ throws MetadataException, RemoteException {
+ ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets
+ .get(searchKey.getDatasetId());
+ if (index == null) {
+ throw new MetadataException("Metadata Extension Index: " + searchKey.getDatasetId() + " was not found");
+ }
+ IMetadataEntityTupleTranslator<T> tupleTranslator = index.getTupleTranslator();
+ return getEntities(jobId, searchKey.getSearchKey(), tupleTranslator, index);
+ }
+
@Override
public void addDataverse(JobId jobId, Dataverse dataverse) throws MetadataException, RemoteException {
try {
- DataverseTupleTranslator tupleReaderWriter = new DataverseTupleTranslator(true);
+ DataverseTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDataverseTupleTranslator(true);
ITupleReference tuple = tupleReaderWriter.getTupleFromMetadataEntity(dataverse);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATAVERSE_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
@@ -186,7 +312,7 @@ public class MetadataNode implements IMetadataNode {
public void addDataset(JobId jobId, Dataset dataset) throws MetadataException, RemoteException {
try {
// Insert into the 'dataset' dataset.
- DatasetTupleTranslator tupleReaderWriter = new DatasetTupleTranslator(true);
+ DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(true);
ITupleReference datasetTuple = tupleReaderWriter.getTupleFromMetadataEntity(dataset);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple);
@@ -210,7 +336,7 @@ public class MetadataNode implements IMetadataNode {
@Override
public void addIndex(JobId jobId, Index index) throws MetadataException, RemoteException {
try {
- IndexTupleTranslator tupleWriter = new IndexTupleTranslator(jobId, this, true);
+ IndexTupleTranslator tupleWriter = tupleTranslatorProvider.getIndexTupleTranslator(jobId, this, true);
ITupleReference tuple = tupleWriter.getTupleFromMetadataEntity(index);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.INDEX_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
@@ -223,7 +349,7 @@ public class MetadataNode implements IMetadataNode {
@Override
public void addNode(JobId jobId, Node node) throws MetadataException, RemoteException {
try {
- NodeTupleTranslator tupleReaderWriter = new NodeTupleTranslator(true);
+ NodeTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getNodeTupleTranslator(true);
ITupleReference tuple = tupleReaderWriter.getTupleFromMetadataEntity(node);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.NODE_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
@@ -236,7 +362,7 @@ public class MetadataNode implements IMetadataNode {
@Override
public void addNodeGroup(JobId jobId, NodeGroup nodeGroup) throws MetadataException, RemoteException {
try {
- NodeGroupTupleTranslator tupleReaderWriter = new NodeGroupTupleTranslator(true);
+ NodeGroupTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getNodeGroupTupleTranslator(true);
ITupleReference tuple = tupleReaderWriter.getTupleFromMetadataEntity(nodeGroup);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.NODEGROUP_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
@@ -250,7 +376,8 @@ public class MetadataNode implements IMetadataNode {
@Override
public void addDatatype(JobId jobId, Datatype datatype) throws MetadataException, RemoteException {
try {
- DatatypeTupleTranslator tupleReaderWriter = new DatatypeTupleTranslator(jobId, this, true);
+ DatatypeTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDataTypeTupleTranslator(jobId, this,
+ true);
ITupleReference tuple = tupleReaderWriter.getTupleFromMetadataEntity(datatype);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATATYPE_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
@@ -264,7 +391,7 @@ public class MetadataNode implements IMetadataNode {
public void addFunction(JobId jobId, Function function) throws MetadataException, RemoteException {
try {
// Insert into the 'function' dataset.
- FunctionTupleTranslator tupleReaderWriter = new FunctionTupleTranslator(true);
+ FunctionTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFunctionTupleTranslator(true);
ITupleReference functionTuple = tupleReaderWriter.getTupleFromMetadataEntity(function);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, functionTuple);
@@ -572,9 +699,9 @@ public class MetadataNode implements IMetadataNode {
@Override
public List<Dataverse> getDataverses(JobId jobId) throws MetadataException, RemoteException {
try {
- DataverseTupleTranslator tupleReaderWriter = new DataverseTupleTranslator(false);
- IValueExtractor<Dataverse> valueExtractor = new MetadataEntityValueExtractor<Dataverse>(tupleReaderWriter);
- List<Dataverse> results = new ArrayList<Dataverse>();
+ DataverseTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDataverseTupleTranslator(false);
+ IValueExtractor<Dataverse> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Dataverse> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATAVERSE_DATASET, null, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -586,9 +713,9 @@ public class MetadataNode implements IMetadataNode {
public Dataverse getDataverse(JobId jobId, String dataverseName) throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName);
- DataverseTupleTranslator tupleReaderWriter = new DataverseTupleTranslator(false);
- IValueExtractor<Dataverse> valueExtractor = new MetadataEntityValueExtractor<Dataverse>(tupleReaderWriter);
- List<Dataverse> results = new ArrayList<Dataverse>();
+ DataverseTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDataverseTupleTranslator(false);
+ IValueExtractor<Dataverse> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Dataverse> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATAVERSE_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -604,9 +731,9 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName);
- DatasetTupleTranslator tupleReaderWriter = new DatasetTupleTranslator(false);
- IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<Dataset>(tupleReaderWriter);
- List<Dataset> results = new ArrayList<Dataset>();
+ DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(false);
+ IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Dataset> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -618,9 +745,9 @@ public class MetadataNode implements IMetadataNode {
public List<Feed> getDataverseFeeds(JobId jobId, String dataverseName) throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName);
- FeedTupleTranslator tupleReaderWriter = new FeedTupleTranslator(false);
- IValueExtractor<Feed> valueExtractor = new MetadataEntityValueExtractor<Feed>(tupleReaderWriter);
- List<Feed> results = new ArrayList<Feed>();
+ FeedTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedTupleTranslator(false);
+ IValueExtractor<Feed> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Feed> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.FEED_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -633,9 +760,9 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName);
- LibraryTupleTranslator tupleReaderWriter = new LibraryTupleTranslator(false);
- IValueExtractor<Library> valueExtractor = new MetadataEntityValueExtractor<Library>(tupleReaderWriter);
- List<Library> results = new ArrayList<Library>();
+ LibraryTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getLibraryTupleTranslator(false);
+ IValueExtractor<Library> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Library> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.LIBRARY_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -646,9 +773,10 @@ public class MetadataNode implements IMetadataNode {
private List<Datatype> getDataverseDatatypes(JobId jobId, String dataverseName) throws MetadataException {
try {
ITupleReference searchKey = createTuple(dataverseName);
- DatatypeTupleTranslator tupleReaderWriter = new DatatypeTupleTranslator(jobId, this, false);
- IValueExtractor<Datatype> valueExtractor = new MetadataEntityValueExtractor<Datatype>(tupleReaderWriter);
- List<Datatype> results = new ArrayList<Datatype>();
+ DatatypeTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDataTypeTupleTranslator(jobId, this,
+ false);
+ IValueExtractor<Datatype> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Datatype> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATATYPE_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -661,9 +789,9 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName, datasetName);
- DatasetTupleTranslator tupleReaderWriter = new DatasetTupleTranslator(false);
- List<Dataset> results = new ArrayList<Dataset>();
- IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<Dataset>(tupleReaderWriter);
+ DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(false);
+ List<Dataset> results = new ArrayList<>();
+ IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -677,9 +805,9 @@ public class MetadataNode implements IMetadataNode {
public List<Dataset> getAllDatasets(JobId jobId) throws MetadataException {
try {
ITupleReference searchKey = null;
- DatasetTupleTranslator tupleReaderWriter = new DatasetTupleTranslator(false);
- IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<Dataset>(tupleReaderWriter);
- List<Dataset> results = new ArrayList<Dataset>();
+ DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(false);
+ IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Dataset> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -690,9 +818,10 @@ public class MetadataNode implements IMetadataNode {
public List<Datatype> getAllDatatypes(JobId jobId) throws MetadataException {
try {
ITupleReference searchKey = null;
- DatatypeTupleTranslator tupleReaderWriter = new DatatypeTupleTranslator(jobId, this, false);
- IValueExtractor<Datatype> valueExtractor = new MetadataEntityValueExtractor<Datatype>(tupleReaderWriter);
- List<Datatype> results = new ArrayList<Datatype>();
+ DatatypeTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDataTypeTupleTranslator(jobId, this,
+ false);
+ IValueExtractor<Datatype> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Datatype> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATATYPE_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -785,7 +914,7 @@ public class MetadataNode implements IMetadataNode {
public List<String> getDatasetNamesPartitionedOnThisNodeGroup(JobId jobId, String nodegroup)
throws MetadataException {
//this needs to scan the datasets and return the datasets that use this nodegroup
- List<String> nodeGroupDatasets = new ArrayList<String>();
+ List<String> nodeGroupDatasets = new ArrayList<>();
List<Dataset> datasets = getAllDatasets(jobId);
for (Dataset set : datasets) {
if (set.getNodeGroupName().equals(nodegroup)) {
@@ -801,9 +930,10 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName, datasetName, indexName);
- IndexTupleTranslator tupleReaderWriter = new IndexTupleTranslator(jobId, this, false);
- IValueExtractor<Index> valueExtractor = new MetadataEntityValueExtractor<Index>(tupleReaderWriter);
- List<Index> results = new ArrayList<Index>();
+ IndexTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getIndexTupleTranslator(jobId, this,
+ false);
+ IValueExtractor<Index> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Index> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.INDEX_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -819,9 +949,10 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName, datasetName);
- IndexTupleTranslator tupleReaderWriter = new IndexTupleTranslator(jobId, this, false);
- IValueExtractor<Index> valueExtractor = new MetadataEntityValueExtractor<Index>(tupleReaderWriter);
- List<Index> results = new ArrayList<Index>();
+ IndexTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getIndexTupleTranslator(jobId, this,
+ false);
+ IValueExtractor<Index> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Index> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.INDEX_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -834,9 +965,10 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName, datatypeName);
- DatatypeTupleTranslator tupleReaderWriter = new DatatypeTupleTranslator(jobId, this, false);
- IValueExtractor<Datatype> valueExtractor = new MetadataEntityValueExtractor<Datatype>(tupleReaderWriter);
- List<Datatype> results = new ArrayList<Datatype>();
+ DatatypeTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDataTypeTupleTranslator(jobId, this,
+ false);
+ IValueExtractor<Datatype> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Datatype> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATATYPE_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -851,9 +983,9 @@ public class MetadataNode implements IMetadataNode {
public NodeGroup getNodeGroup(JobId jobId, String nodeGroupName) throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(nodeGroupName);
- NodeGroupTupleTranslator tupleReaderWriter = new NodeGroupTupleTranslator(false);
- IValueExtractor<NodeGroup> valueExtractor = new MetadataEntityValueExtractor<NodeGroup>(tupleReaderWriter);
- List<NodeGroup> results = new ArrayList<NodeGroup>();
+ NodeGroupTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getNodeGroupTupleTranslator(false);
+ IValueExtractor<NodeGroup> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<NodeGroup> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.NODEGROUP_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -870,9 +1002,9 @@ public class MetadataNode implements IMetadataNode {
try {
ITupleReference searchKey = createTuple(functionSignature.getNamespace(), functionSignature.getName(),
"" + functionSignature.getArity());
- FunctionTupleTranslator tupleReaderWriter = new FunctionTupleTranslator(false);
- List<Function> results = new ArrayList<Function>();
- IValueExtractor<Function> valueExtractor = new MetadataEntityValueExtractor<Function>(tupleReaderWriter);
+ FunctionTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFunctionTupleTranslator(false);
+ List<Function> results = new ArrayList<>();
+ IValueExtractor<Function> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -916,7 +1048,7 @@ public class MetadataNode implements IMetadataNode {
private ITupleReference getTupleToBeDeleted(JobId jobId, IMetadataIndex metadataIndex, ITupleReference searchKey)
throws MetadataException, IndexException, IOException {
IValueExtractor<ITupleReference> valueExtractor = new TupleCopyValueExtractor(metadataIndex.getTypeTraits());
- List<ITupleReference> results = new ArrayList<ITupleReference>();
+ List<ITupleReference> results = new ArrayList<>();
searchIndex(jobId, metadataIndex, searchKey, valueExtractor, results);
if (results.isEmpty()) {
// TODO: Temporarily a TreeIndexException to make it get caught by
@@ -1059,8 +1191,8 @@ public class MetadataNode implements IMetadataNode {
NoOpOperationCallback.INSTANCE);
IIndexCursor rangeCursor = indexAccessor.createSearchCursor(false);
- DatasetTupleTranslator tupleReaderWriter = new DatasetTupleTranslator(false);
- IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<Dataset>(tupleReaderWriter);
+ DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(false);
+ IValueExtractor<Dataset> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
indexAccessor.search(rangeCursor, rangePred);
@@ -1092,7 +1224,7 @@ public class MetadataNode implements IMetadataNode {
// TODO: Can use Hyrack's TupleUtils for this, once we switch to a newer
// Hyracks version.
- public ITupleReference createTuple(String... fields) throws HyracksDataException {
+ public static ITupleReference createTuple(String... fields) {
@SuppressWarnings("unchecked")
ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.ASTRING);
@@ -1100,7 +1232,12 @@ public class MetadataNode implements IMetadataNode {
ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fields.length);
for (String s : fields) {
aString.setValue(s);
- stringSerde.serialize(aString, tupleBuilder.getDataOutput());
+ try {
+ stringSerde.serialize(aString, tupleBuilder.getDataOutput());
+ } catch (HyracksDataException e) {
+ // This should never happen
+ throw new IllegalStateException("Failed to create search tuple!!!! This should never happen", e);
+ }
tupleBuilder.addFieldEndOffset();
}
ArrayTupleReference tuple = new ArrayTupleReference();
@@ -1113,9 +1250,9 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName);
- FunctionTupleTranslator tupleReaderWriter = new FunctionTupleTranslator(false);
- IValueExtractor<Function> valueExtractor = new MetadataEntityValueExtractor<Function>(tupleReaderWriter);
- List<Function> results = new ArrayList<Function>();
+ FunctionTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFunctionTupleTranslator(false);
+ IValueExtractor<Function> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<Function> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.FUNCTION_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -1127,7 +1264,8 @@ public class MetadataNode implements IMetadataNode {
public void addAdapter(JobId jobId, DatasourceAdapter adapter) throws MetadataException, RemoteException {
try {
// Insert into the 'Adapter' dataset.
- DatasourceAdapterTupleTranslator tupleReaderWriter = new DatasourceAdapterTupleTranslator(true);
+ DatasourceAdapterTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getAdapterTupleTranslator(true);
ITupleReference adapterTuple = tupleReaderWriter.getTupleFromMetadataEntity(adapter);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET, adapterTuple);
@@ -1172,10 +1310,10 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName, adapterName);
- DatasourceAdapterTupleTranslator tupleReaderWriter = new DatasourceAdapterTupleTranslator(false);
- List<DatasourceAdapter> results = new ArrayList<DatasourceAdapter>();
- IValueExtractor<DatasourceAdapter> valueExtractor = new MetadataEntityValueExtractor<DatasourceAdapter>(
- tupleReaderWriter);
+ DatasourceAdapterTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getAdapterTupleTranslator(false);
+ List<DatasourceAdapter> results = new ArrayList<>();
+ IValueExtractor<DatasourceAdapter> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -1191,7 +1329,8 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
// Insert into the 'CompactionPolicy' dataset.
- CompactionPolicyTupleTranslator tupleReaderWriter = new CompactionPolicyTupleTranslator(true);
+ CompactionPolicyTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getCompactionPolicyTupleTranslator(true);
ITupleReference compactionPolicyTuple = tupleReaderWriter.getTupleFromMetadataEntity(compactionPolicy);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET, compactionPolicyTuple);
@@ -1208,10 +1347,10 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverse, policyName);
- CompactionPolicyTupleTranslator tupleReaderWriter = new CompactionPolicyTupleTranslator(false);
- List<CompactionPolicy> results = new ArrayList<CompactionPolicy>();
- IValueExtractor<CompactionPolicy> valueExtractor = new MetadataEntityValueExtractor<CompactionPolicy>(
- tupleReaderWriter);
+ CompactionPolicyTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getCompactionPolicyTupleTranslator(false);
+ List<CompactionPolicy> results = new ArrayList<>();
+ IValueExtractor<CompactionPolicy> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET, searchKey, valueExtractor, results);
if (!results.isEmpty()) {
return results.get(0);
@@ -1227,10 +1366,10 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName);
- DatasourceAdapterTupleTranslator tupleReaderWriter = new DatasourceAdapterTupleTranslator(false);
- IValueExtractor<DatasourceAdapter> valueExtractor = new MetadataEntityValueExtractor<DatasourceAdapter>(
- tupleReaderWriter);
- List<DatasourceAdapter> results = new ArrayList<DatasourceAdapter>();
+ DatasourceAdapterTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getAdapterTupleTranslator(false);
+ IValueExtractor<DatasourceAdapter> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<DatasourceAdapter> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -1242,7 +1381,7 @@ public class MetadataNode implements IMetadataNode {
public void addLibrary(JobId jobId, Library library) throws MetadataException, RemoteException {
try {
// Insert into the 'Library' dataset.
- LibraryTupleTranslator tupleReaderWriter = new LibraryTupleTranslator(true);
+ LibraryTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getLibraryTupleTranslator(true);
ITupleReference libraryTuple = tupleReaderWriter.getTupleFromMetadataEntity(library);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.LIBRARY_DATASET, libraryTuple);
@@ -1285,9 +1424,9 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverseName, libraryName);
- LibraryTupleTranslator tupleReaderWriter = new LibraryTupleTranslator(false);
- List<Library> results = new ArrayList<Library>();
- IValueExtractor<Library> valueExtractor = new MetadataEntityValueExtractor<Library>(tupleReaderWriter);
+ LibraryTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getLibraryTupleTranslator(false);
+ List<Library> results = new ArrayList<>();
+ IValueExtractor<Library> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.LIBRARY_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -1307,7 +1446,7 @@ public class MetadataNode implements IMetadataNode {
public void addFeedPolicy(JobId jobId, FeedPolicyEntity feedPolicy) throws MetadataException, RemoteException {
try {
// Insert into the 'FeedPolicy' dataset.
- FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(true);
+ FeedPolicyTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedPolicyTupleTranslator(true);
ITupleReference feedPolicyTuple = tupleReaderWriter.getTupleFromMetadataEntity(feedPolicy);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, feedPolicyTuple);
@@ -1324,10 +1463,9 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverse, policyName);
- FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(false);
- List<FeedPolicyEntity> results = new ArrayList<FeedPolicyEntity>();
- IValueExtractor<FeedPolicyEntity> valueExtractor = new MetadataEntityValueExtractor<FeedPolicyEntity>(
- tupleReaderWriter);
+ FeedPolicyTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedPolicyTupleTranslator(false);
+ List<FeedPolicyEntity> results = new ArrayList<>();
+ IValueExtractor<FeedPolicyEntity> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, searchKey, valueExtractor, results);
if (!results.isEmpty()) {
return results.get(0);
@@ -1342,7 +1480,7 @@ public class MetadataNode implements IMetadataNode {
public void addFeed(JobId jobId, Feed feed) throws MetadataException, RemoteException {
try {
// Insert into the 'Feed' dataset.
- FeedTupleTranslator tupleReaderWriter = new FeedTupleTranslator(true);
+ FeedTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedTupleTranslator(true);
ITupleReference feedTuple = tupleReaderWriter.getTupleFromMetadataEntity(feed);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.FEED_DATASET, feedTuple);
@@ -1358,9 +1496,9 @@ public class MetadataNode implements IMetadataNode {
public Feed getFeed(JobId jobId, String dataverse, String feedName) throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverse, feedName);
- FeedTupleTranslator tupleReaderWriter = new FeedTupleTranslator(false);
- List<Feed> results = new ArrayList<Feed>();
- IValueExtractor<Feed> valueExtractor = new MetadataEntityValueExtractor<Feed>(tupleReaderWriter);
+ FeedTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedTupleTranslator(false);
+ List<Feed> results = new ArrayList<>();
+ IValueExtractor<Feed> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
searchIndex(jobId, MetadataPrimaryIndexes.FEED_DATASET, searchKey, valueExtractor, results);
if (!results.isEmpty()) {
return results.get(0);
@@ -1407,10 +1545,9 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataverse);
- FeedPolicyTupleTranslator tupleReaderWriter = new FeedPolicyTupleTranslator(false);
- IValueExtractor<FeedPolicyEntity> valueExtractor = new MetadataEntityValueExtractor<FeedPolicyEntity>(
- tupleReaderWriter);
- List<FeedPolicyEntity> results = new ArrayList<FeedPolicyEntity>();
+ FeedPolicyTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getFeedPolicyTupleTranslator(false);
+ IValueExtractor<FeedPolicyEntity> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<FeedPolicyEntity> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.FEED_POLICY_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -1422,7 +1559,8 @@ public class MetadataNode implements IMetadataNode {
public void addExternalFile(JobId jobId, ExternalFile externalFile) throws MetadataException, RemoteException {
try {
// Insert into the 'externalFiles' dataset.
- ExternalFileTupleTranslator tupleReaderWriter = new ExternalFileTupleTranslator(true);
+ ExternalFileTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getExternalFileTupleTranslator(true);
ITupleReference externalFileTuple = tupleReaderWriter.getTupleFromMetadataEntity(externalFile);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET, externalFileTuple);
} catch (TreeIndexDuplicateKeyException e) {
@@ -1438,10 +1576,10 @@ public class MetadataNode implements IMetadataNode {
public List<ExternalFile> getExternalFiles(JobId jobId, Dataset dataset) throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createTuple(dataset.getDataverseName(), dataset.getDatasetName());
- ExternalFileTupleTranslator tupleReaderWriter = new ExternalFileTupleTranslator(false);
- IValueExtractor<ExternalFile> valueExtractor = new MetadataEntityValueExtractor<ExternalFile>(
- tupleReaderWriter);
- List<ExternalFile> results = new ArrayList<ExternalFile>();
+ ExternalFileTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getExternalFileTupleTranslator(false);
+ IValueExtractor<ExternalFile> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<ExternalFile> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET, searchKey, valueExtractor, results);
return results;
} catch (IndexException | IOException e) {
@@ -1513,10 +1651,10 @@ public class MetadataNode implements IMetadataNode {
throws MetadataException, RemoteException {
try {
ITupleReference searchKey = createExternalFileSearchTuple(dataverseName, datasetName, fileNumber);
- ExternalFileTupleTranslator tupleReaderWriter = new ExternalFileTupleTranslator(false);
- IValueExtractor<ExternalFile> valueExtractor = new MetadataEntityValueExtractor<ExternalFile>(
- tupleReaderWriter);
- List<ExternalFile> results = new ArrayList<ExternalFile>();
+ ExternalFileTupleTranslator tupleReaderWriter = tupleTranslatorProvider
+ .getExternalFileTupleTranslator(false);
+ IValueExtractor<ExternalFile> valueExtractor = new MetadataEntityValueExtractor<>(tupleReaderWriter);
+ List<ExternalFile> results = new ArrayList<>();
searchIndex(jobId, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET, searchKey, valueExtractor, results);
if (results.isEmpty()) {
return null;
@@ -1541,7 +1679,7 @@ public class MetadataNode implements IMetadataNode {
deleteTupleFromIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple);
// Previous tuple was deleted
// Insert into the 'dataset' dataset.
- DatasetTupleTranslator tupleReaderWriter = new DatasetTupleTranslator(true);
+ DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(true);
datasetTuple = tupleReaderWriter.getTupleFromMetadataEntity(dataset);
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple);
} catch (ACIDException | IndexException | IOException e) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
new file mode 100644
index 0000000..d3f7e1c
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDataset.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.api;
+
+import java.util.List;
+
+import org.apache.asterix.metadata.bootstrap.MetadataIndex;
+import org.apache.asterix.metadata.bootstrap.MetadataIndexImmutableProperties;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+
+public class ExtensionMetadataDataset<T> extends MetadataIndex {
+
+ private static final long serialVersionUID = 1L;
+ private final ExtensionMetadataDatasetId indexId;
+ private final transient IMetadataEntityTupleTranslator<T> tupleTranslator;
+
+ public ExtensionMetadataDataset(MetadataIndexImmutableProperties indexProperties, int numFields, IAType[] keyTypes,
+ List<List<String>> keyNames, int numSecondaryIndexKeys, ARecordType payloadType, boolean isPrimaryIndex,
+ int[] primaryKeyIndexes, ExtensionMetadataDatasetId indexId,
+ IMetadataEntityTupleTranslator<T> tupleTranslator) {
+ super(indexProperties, numFields, keyTypes, keyNames, numSecondaryIndexKeys, payloadType, isPrimaryIndex,
+ primaryKeyIndexes);
+ this.indexId = indexId;
+ this.tupleTranslator = tupleTranslator;
+ }
+
+ public ExtensionMetadataDatasetId getId() {
+ return indexId;
+ }
+
+ public IMetadataEntityTupleTranslator<T> getTupleTranslator() {
+ return tupleTranslator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDatasetId.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDatasetId.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDatasetId.java
new file mode 100644
index 0000000..4074cdc
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/ExtensionMetadataDatasetId.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.api;
+
+import java.io.Serializable;
+
+import org.apache.asterix.common.api.ExtensionId;
+import org.apache.commons.lang3.ObjectUtils;
+
+public class ExtensionMetadataDatasetId implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final ExtensionId extensionId;
+ private final String datasetName;
+
+ public ExtensionMetadataDatasetId(ExtensionId extensionId, String datasetName) {
+ this.extensionId = extensionId;
+ this.datasetName = datasetName;
+ }
+
+ public ExtensionId getExtensionId() {
+ return extensionId;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (o instanceof ExtensionMetadataDatasetId) {
+ ExtensionMetadataDatasetId otherId = (ExtensionMetadataDatasetId) o;
+ return ObjectUtils.equals(extensionId, otherId.getExtensionId())
+ && ObjectUtils.equals(datasetName, otherId.getDatasetName());
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return ObjectUtils.hashCodeMulti(datasetName, extensionId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataEntity.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataEntity.java
new file mode 100644
index 0000000..6ae23f0
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataEntity.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.api;
+
+import java.io.Serializable;
+
+/**
+ * Represents an extension metadata entity
+ */
+public interface IExtensionMetadataEntity extends Serializable {
+
+ /**
+ * Get the dataset Id of the dataset of the entity
+ *
+ * @return the dataset Id
+ */
+ ExtensionMetadataDatasetId getDatasetId();
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java
new file mode 100644
index 0000000..e8e39e3
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IExtensionMetadataSearchKey.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.api;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * A search key used to search extension datasets for extension entities
+ */
+public interface IExtensionMetadataSearchKey extends Serializable {
+
+ /**
+ * Gets the dataset id for the extension metadata dataset to be searched
+ *
+ * @return the dataset Id
+ */
+ ExtensionMetadataDatasetId getDatasetId();
+
+ /**
+ * Gets the tuple reference representation of the search key
+ *
+ * @return the search key as a tuple reference
+ */
+ ITupleReference getSearchKey();
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java
index 2fc846b..1759c0f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntity.java
@@ -16,7 +16,6 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.asterix.metadata.api;
import java.io.Serializable;
@@ -24,7 +23,6 @@ import java.io.Serializable;
import org.apache.asterix.metadata.MetadataCache;
public interface IMetadataEntity<T> extends Serializable {
-
public static final int PENDING_NO_OP = 0;
public static final int PENDING_ADD_OP = 1;
public static final int PENDING_DROP_OP = 2;
@@ -32,4 +30,5 @@ public interface IMetadataEntity<T> extends Serializable {
public T addToCache(MetadataCache cache);
public T dropFromCache(MetadataCache cache);
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java
index 7c115b8..d8ccbde 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataEntityTupleTranslator.java
@@ -20,6 +20,7 @@
package org.apache.asterix.metadata.api;
import java.io.IOException;
+import java.io.Serializable;
import org.apache.asterix.metadata.MetadataException;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -30,7 +31,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
* representation in a Hyracks tuple, and vice versa. Implementations of this
* interface are intended to be used within an IMetadataNode.
*/
-public interface IMetadataEntityTupleTranslator<T> {
+public interface IMetadataEntityTupleTranslator<T> extends Serializable {
/**
* Transforms a metadata entity of type T from a given tuple to a Java
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java
new file mode 100644
index 0000000..9b55151
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataExtension.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.metadata.api;
+
+import java.rmi.RemoteException;
+import java.util.List;
+
+import org.apache.asterix.common.api.IExtension;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.metadata.entitytupletranslators.MetadataTupleTranslatorProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * An interface for Metadata Extensions
+ */
+public interface IMetadataExtension extends IExtension {
+
+ @Override
+ default ExtensionKind getExtensionKind() {
+ return ExtensionKind.METADATA;
+ }
+
+ /**
+ * @return The tuple translator provider that must be used by the {@code IMetadataNode } to read and write core
+ * {@code IMetadataEntity} objects
+ */
+ MetadataTupleTranslatorProvider getMetadataTupleTranslatorProvider();
+
+ /**
+ * @return A list of additional extension instances of {@code IMetadataIndex} that are introduced by the extension
+ */
+ @SuppressWarnings("rawtypes")
+ List<ExtensionMetadataDataset> getExtensionIndexes();
+
+ /**
+ * Called when booting the {@code IMetadataNode}
+ *
+ * @throws HyracksDataException
+ * @throws RemoteException
+ * @throws ACIDException
+ */
+ void initializeMetadata() throws HyracksDataException, RemoteException, ACIDException;
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
index 9b8bcfb..9eec64c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataIndex.java
@@ -19,6 +19,7 @@
package org.apache.asterix.metadata.api;
+import java.io.Serializable;
import java.util.List;
import org.apache.asterix.common.transactions.DatasetId;
@@ -33,7 +34,7 @@ import org.apache.hyracks.api.io.FileReference;
/**
* Descriptor interface for a primary or secondary index on metadata datasets.
*/
-public interface IMetadataIndex {
+public interface IMetadataIndex extends Serializable {
public String getDataverseName();
public String getNodeGroupName();
@@ -83,5 +84,4 @@ public interface IMetadataIndex {
boolean isPrimaryIndex();
int[] getPrimaryKeyIndexes();
-
}
[16/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
Add Asterix Extension Manager
More extension support is added. A user can now provide implementations
for the IExtension interface which will give them more control over
the behavior of the system and give them the ability to add custom
features.
Initial customizations include:
1. Metadata Tuple Translators
2. Metadata Datasets
3. Query Translators
4. Statement Handlers
5. Lang Compilation Provider
Change-Id: I280268495cc3aad00f898cba21f7299f7120ce5c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1017
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <bu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/ab81748a
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/ab81748a
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/ab81748a
Branch: refs/heads/master
Commit: ab81748abce39d8a368d5069d96d96357bedfe6e
Parents: a4815d3
Author: Yingyi Bu <yi...@couchbase.com>
Authored: Fri Aug 19 21:29:31 2016 -0700
Committer: Yingyi Bu <bu...@gmail.com>
Committed: Fri Aug 19 23:15:04 2016 -0700
----------------------------------------------------------------------
.../org/apache/asterix/active/ActiveEvent.java | 32 +-
.../active/ActiveJobNotificationHandler.java | 51 +-
.../apache/asterix/active/ActiveManager.java | 41 +-
.../asterix/active/ActivePartitionMessage.java | 55 -
.../apache/asterix/active/ActiveRuntime.java | 39 -
.../apache/asterix/active/ActiveRuntimeId.java | 20 +-
.../asterix/active/ActiveRuntimeManager.java | 6 +-
.../asterix/active/ActiveRuntimeRegistry.java | 79 -
.../ActiveSourceOperatorNodePushable.java | 125 +
.../apache/asterix/active/IActiveRuntime.java | 15 +-
.../asterix/active/IActiveRuntimeRegistry.java | 49 -
.../active/message/ActiveManagerMessage.java | 55 +
.../active/message/ActivePartitionMessage.java | 68 +
.../asterix/algebra/base/ILangExtension.java | 64 +
.../extension/ExtensionFunctionIdentifier.java | 78 +
.../extension/IAlgebraExtensionManager.java | 34 +
.../algebra/extension/IExtensionStatement.java | 48 +
.../asterix/optimizer/base/RuleCollections.java | 7 +-
.../rules/MetaFunctionToMetaVariableRule.java | 14 +-
.../optimizer/rules/UnnestToDataScanRule.java | 35 +-
.../am/AbstractIntroduceAccessMethodRule.java | 129 +-
.../optimizer/rules/am/BTreeAccessMethod.java | 120 +-
.../rules/am/IntroduceJoinAccessMethodRule.java | 8 +-
.../am/IntroduceLSMComponentFilterRule.java | 34 +-
.../am/IntroduceSelectAccessMethodRule.java | 4 +-
.../rules/am/InvertedIndexAccessMethod.java | 85 +-
.../rules/am/OptimizableOperatorSubTree.java | 286 +-
.../optimizer/rules/am/RTreeAccessMethod.java | 62 +-
.../translator/AbstractLangTranslator.java | 6 +-
.../AqlExpressionToPlanTranslator.java | 2 +-
.../AqlPlusExpressionToPlanTranslator.java | 4 +-
.../asterix/translator/IStatementExecutor.java | 129 +
.../translator/IStatementExecutorFactory.java | 46 +
.../LangExpressionToPlanTranslator.java | 250 +-
.../translator/util/PlanTranslationUtil.java | 1 -
.../asterix-algebra/src/main/javacc/AQLPlus.jj | 2 +-
.../apache/asterix/api/common/APIFramework.java | 23 +-
.../api/common/AsterixAppRuntimeContext.java | 465 ---
...rixAppRuntimeContextProviderForRecovery.java | 5 +-
.../common/AsterixHyracksIntegrationUtil.java | 6 +-
.../asterix/api/common/SessionConfig.java | 263 --
.../asterix/api/http/servlet/APIServlet.java | 42 +-
.../asterix/api/http/servlet/AQLAPIServlet.java | 16 +-
.../api/http/servlet/ClusterAPIServlet.java | 4 +-
.../api/http/servlet/ConnectorAPIServlet.java | 8 +-
.../asterix/api/http/servlet/DDLAPIServlet.java | 25 +-
.../api/http/servlet/QueryAPIServlet.java | 19 +-
.../api/http/servlet/QueryResultAPIServlet.java | 17 +-
.../api/http/servlet/QueryServiceServlet.java | 39 +-
.../api/http/servlet/QueryStatusAPIServlet.java | 15 +-
.../api/http/servlet/RESTAPIServlet.java | 36 +-
.../api/http/servlet/UpdateAPIServlet.java | 20 +-
.../asterix/api/java/AsterixJavaClient.java | 22 +-
.../app/cc/CompilerExtensionManager.java | 156 +
.../app/cc/IStatementExecutorExtension.java | 37 +
.../app/external/ActiveLifecycleListener.java | 15 +-
.../app/external/ExternalLibraryUtils.java | 8 +-
.../app/external/FeedWorkCollection.java | 15 +-
.../app/nc/AsterixNCAppRuntimeContext.java | 475 +++
.../asterix/app/nc/NCExtensionManager.java | 110 +
.../asterix/app/result/ResultPrinter.java | 190 ++
.../apache/asterix/app/result/ResultReader.java | 64 +
.../apache/asterix/app/result/ResultUtil.java | 254 ++
.../DefaultStatementExecutorFactory.java | 43 +
.../asterix/app/translator/QueryTranslator.java | 3122 ++++++++++++++++++
.../asterix/aql/translator/QueryTranslator.java | 3107 -----------------
.../org/apache/asterix/drivers/AsterixCLI.java | 8 +-
.../asterix/drivers/AsterixClientDriver.java | 4 +-
.../asterix/drivers/AsterixWebServer.java | 8 +-
.../bootstrap/CCApplicationEntryPoint.java | 60 +-
.../bootstrap/GlobalRecoveryManager.java | 28 +-
.../bootstrap/NCApplicationEntryPoint.java | 5 +-
.../asterix/messaging/CCMessageBroker.java | 4 +-
.../asterix/messaging/NCMessageBroker.java | 21 +-
.../apache/asterix/result/ResultPrinter.java | 188 --
.../org/apache/asterix/result/ResultReader.java | 65 -
.../org/apache/asterix/result/ResultUtils.java | 243 --
.../app/bootstrap/TestNodeController.java | 92 +-
.../aql/translator/QueryTranslatorTest.java | 6 +-
.../org/apache/asterix/test/dml/DmlTest.java | 6 +-
.../asterix/test/optimizer/OptimizerTest.java | 4 +-
.../asterix/test/runtime/ExecutionTestUtil.java | 10 +-
.../asterix/test/runtime/HDFSCluster.java | 18 +-
.../apache/asterix/common/api/ExtensionId.java | 64 +
.../common/api/IAsterixAppRuntimeContext.java | 2 +-
.../apache/asterix/common/api/IExtension.java | 63 +
.../asterix/common/app/SessionConfig.java | 264 ++
.../asterix/common/config/AsterixExtension.java | 41 +
.../config/AsterixExtensionProperties.java | 32 +
.../common/config/AsterixProperties.java | 54 +
.../config/AsterixPropertiesAccessor.java | 192 +-
.../common/config/MetadataConstants.java | 33 -
.../asterix/common/exceptions/ErrorCode.java | 2 +
.../messaging/api/IApplicationMessage.java | 3 +-
.../apache/asterix/common/utils/ConfigUtil.java | 41 +
.../src/main/resources/schema/asterix-conf.xsd | 29 +
.../external/feed/api/ISubscriberRuntime.java | 27 -
.../feed/dataflow/FeedRuntimeInputHandler.java | 2 +-
.../feed/management/FeedEventsListener.java | 4 +-
.../external/feed/runtime/AdapterExecutor.java | 9 +-
.../feed/runtime/AdapterRuntimeManager.java | 22 +-
.../feed/runtime/CollectionRuntime.java | 27 +-
.../external/feed/runtime/IngestionRuntime.java | 20 +-
.../feed/runtime/SubscribableRuntime.java | 17 +-
.../asterix/external/indexing/ExternalFile.java | 23 +-
.../FeedCollectOperatorDescriptor.java | 4 +-
.../FeedCollectOperatorNodePushable.java | 19 +-
.../FeedIntakeOperatorNodePushable.java | 8 +-
.../FeedMessageOperatorNodePushable.java | 13 +-
.../operators/FeedMetaComputeNodePushable.java | 23 +-
.../operators/FeedMetaStoreNodePushable.java | 33 +-
.../lang/aql/rewrites/AqlQueryRewriter.java | 10 +-
.../aql/statement/SubscribeFeedStatement.java | 5 +
.../asterix-lang-aql/src/main/javacc/AQL.jj | 6 +-
.../asterix/lang/common/base/Expression.java | 1 -
.../asterix/lang/common/base/Statement.java | 23 +
.../lang/common/statement/CompactStatement.java | 5 +
.../common/statement/ConnectFeedStatement.java | 5 +
.../statement/CreateDataverseStatement.java | 5 +
.../statement/CreateFeedPolicyStatement.java | 5 +
.../common/statement/CreateFeedStatement.java | 4 +
.../statement/CreateFunctionStatement.java | 5 +
.../common/statement/CreateIndexStatement.java | 5 +
.../lang/common/statement/DatasetDecl.java | 7 +-
.../lang/common/statement/DataverseDecl.java | 5 +
.../statement/DataverseDropStatement.java | 5 +
.../lang/common/statement/DeleteStatement.java | 5 +
.../statement/DisconnectFeedStatement.java | 5 +
.../common/statement/DropDatasetStatement.java | 65 +
.../lang/common/statement/DropStatement.java | 60 -
.../common/statement/FeedDropStatement.java | 5 +
.../statement/FeedPolicyDropStatement.java | 5 +
.../lang/common/statement/FunctionDecl.java | 5 +
.../common/statement/FunctionDropStatement.java | 5 +
.../common/statement/IndexDropStatement.java | 5 +
.../lang/common/statement/InsertStatement.java | 5 +
.../lang/common/statement/LoadStatement.java | 5 +
.../statement/NodeGroupDropStatement.java | 5 +
.../lang/common/statement/NodegroupDecl.java | 5 +
.../asterix/lang/common/statement/Query.java | 5 +
.../RefreshExternalDatasetStatement.java | 5 +
.../lang/common/statement/RunStatement.java | 5 +
.../lang/common/statement/SetStatement.java | 5 +
.../asterix/lang/common/statement/TypeDecl.java | 5 +
.../common/statement/TypeDropStatement.java | 5 +
.../lang/common/statement/UpdateStatement.java | 5 +
.../lang/common/statement/WriteStatement.java | 5 +
.../lang/common/util/LangRecordParseUtil.java | 168 +
.../lang/common/visitor/FormatPrintVisitor.java | 6 +-
.../base/AbstractQueryExpressionVisitor.java | 4 +-
.../lang/common/visitor/base/ILangVisitor.java | 4 +-
.../SqlppBuiltinFunctionRewriteVisitor.java | 2 +-
.../visitor/VariableCheckAndRewriteVisitor.java | 2 +-
.../AbstractSqlppExpressionScopingVisitor.java | 2 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 87 +-
.../pom.xml | 208 +-
.../extension/grammar/GrammarExtensionMojo.java | 139 +-
.../asterix/lang/extension/EchoStatement.java | 5 +
.../src/test/resources/lang/extension.jj | 5 +
.../basic-test/basic-test-plugin-config.xml | 158 +-
.../asterix/metadata/MetadataManager.java | 36 +
.../apache/asterix/metadata/MetadataNode.java | 340 +-
.../metadata/api/ExtensionMetadataDataset.java | 51 +
.../api/ExtensionMetadataDatasetId.java | 62 +
.../metadata/api/IExtensionMetadataEntity.java | 34 +
.../api/IExtensionMetadataSearchKey.java | 43 +
.../asterix/metadata/api/IMetadataEntity.java | 3 +-
.../api/IMetadataEntityTupleTranslator.java | 3 +-
.../metadata/api/IMetadataExtension.java | 61 +
.../asterix/metadata/api/IMetadataIndex.java | 4 +-
.../asterix/metadata/api/IMetadataManager.java | 183 +-
.../asterix/metadata/api/IMetadataNode.java | 155 +-
.../metadata/bootstrap/MetadataBootstrap.java | 148 +-
.../metadata/bootstrap/MetadataIndex.java | 18 +-
.../MetadataIndexImmutableProperties.java | 44 +-
.../bootstrap/MetadataPrimaryIndexes.java | 195 +-
.../metadata/bootstrap/MetadataRecordTypes.java | 4 +-
.../metadata/declared/AqlDataSource.java | 13 +
.../metadata/declared/AqlMetadataProvider.java | 467 ++-
.../metadata/declared/DatasetDataSource.java | 72 +-
.../metadata/declared/FeedDataSource.java | 65 +-
.../metadata/declared/IMutationDataSource.java | 34 +
.../metadata/declared/LoadableDataSource.java | 32 +
.../asterix/metadata/entities/Dataset.java | 21 +-
.../asterix/metadata/entities/Dataverse.java | 5 +
.../entities/InternalDatasetDetails.java | 16 +-
.../AbstractTupleTranslator.java | 9 +-
.../CompactionPolicyTupleTranslator.java | 4 +-
.../DatasetTupleTranslator.java | 41 +-
.../DatasourceAdapterTupleTranslator.java | 9 +-
.../DatatypeTupleTranslator.java | 24 +-
.../DataverseTupleTranslator.java | 4 +-
.../ExternalFileTupleTranslator.java | 10 +-
.../FeedPolicyTupleTranslator.java | 6 +-
.../FeedTupleTranslator.java | 19 +-
.../FunctionTupleTranslator.java | 19 +-
.../IndexTupleTranslator.java | 20 +-
.../LibraryTupleTranslator.java | 14 +-
.../MetadataTupleTranslatorProvider.java | 78 +
.../NodeGroupTupleTranslator.java | 9 +-
.../NodeTupleTranslator.java | 4 +-
.../metadata/feeds/BuiltinFeedPolicies.java | 5 +-
.../metadata/feeds/FeedMetadataUtil.java | 161 +-
.../metadata/feeds/LocationConstraint.java | 29 +
.../asterix/metadata/utils/DatasetUtils.java | 5 +-
.../metadata/utils/MetadataConstants.java | 35 +
.../utils/SplitsAndConstraintsUtil.java | 42 +-
.../DatasetTupleTranslatorTest.java | 9 +-
.../IndexTupleTranslatorTest.java | 9 +-
.../org/apache/asterix/om/base/AString.java | 10 +-
.../apache/asterix/om/types/ARecordType.java | 14 +-
.../asterix/om/util/AsterixAppContextInfo.java | 24 +-
...tiTransactionJobletEventListenerFactory.java | 84 +
.../server/test/NCServiceExecutionIT.java | 14 +-
asterixdb/pom.xml | 13 +
.../algebra/functions/FunctionIdentifier.java | 3 +-
.../api/application/IApplicationConfig.java | 13 +
.../apache/hyracks/api/dataflow/ActivityId.java | 11 +-
.../api/dataflow/IConnectorDescriptor.java | 7 +-
.../api/dataflow/IOperatorDescriptor.java | 12 +-
.../hyracks/api/exceptions/ErrorCode.java | 1 +
.../hyracks/api/job/JobSpecification.java | 35 +-
.../application/IniApplicationConfig.java | 31 +-
.../control/common/controllers/IniUtils.java | 23 +-
.../std/base/AbstractConnectorDescriptor.java | 15 +-
.../std/base/AbstractOperatorDescriptor.java | 14 +-
...bstractSingleActivityOperatorDescriptor.java | 9 +
...wareMToNPartitioningConnectorDescriptor.java | 6 +-
.../MToNBroadcastConnectorDescriptor.java | 7 +-
.../MToNPartitioningConnectorDescriptor.java | 2 +-
...NPartitioningMergingConnectorDescriptor.java | 8 +-
.../connectors/OneToOneConnectorDescriptor.java | 2 +-
.../std/misc/SplitOperatorDescriptor.java | 18 +-
233 files changed, 9719 insertions(+), 6982 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveEvent.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveEvent.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveEvent.java
index e5ccd05..c907a36 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveEvent.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveEvent.java
@@ -18,16 +18,14 @@
*/
package org.apache.asterix.active;
-import java.io.Serializable;
-
import org.apache.hyracks.api.job.JobId;
public class ActiveEvent {
private final JobId jobId;
private final EntityId entityId;
- private final Serializable payload;
private final EventKind eventKind;
+ private final Object eventObject;
public enum EventKind {
JOB_START,
@@ -35,34 +33,30 @@ public class ActiveEvent {
PARTITION_EVENT
}
- public ActiveEvent(JobId jobId, ActiveEvent.EventKind eventKind) {
- this(jobId, eventKind, null, null);
- }
-
- public ActiveEvent(JobId jobId, ActiveEvent.EventKind eventKind, EntityId feedId) {
- this(jobId, eventKind, feedId, null);
- }
-
- public ActiveEvent(JobId jobId, ActiveEvent.EventKind eventKind, EntityId feedId, Serializable payload) {
+ public ActiveEvent(JobId jobId, ActiveEvent.EventKind eventKind, EntityId entityId, Object eventObject) {
this.jobId = jobId;
+ this.entityId = entityId;
this.eventKind = eventKind;
- this.entityId = feedId;
- this.payload = payload;
+ this.eventObject = eventObject;
+ }
+
+ public ActiveEvent(JobId jobId, ActiveEvent.EventKind eventKind, EntityId entityId) {
+ this(jobId, eventKind, entityId, null);
}
public JobId getJobId() {
return jobId;
}
- public EntityId getFeedId() {
+ public EntityId getEntityId() {
return entityId;
}
- public Serializable getPayload() {
- return payload;
- }
-
public EventKind getEventKind() {
return eventKind;
}
+
+ public Object getEventObject() {
+ return eventObject;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveJobNotificationHandler.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveJobNotificationHandler.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveJobNotificationHandler.java
index 9c69aca..30a2eb6 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveJobNotificationHandler.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveJobNotificationHandler.java
@@ -55,8 +55,8 @@ public class ActiveJobNotificationHandler implements Runnable {
EntityId entityId = jobInfo.getEntityId();
IActiveEntityEventsListener listener = entityEventListener.get(entityId);
if (DEBUG) {
- LOGGER.log(Level.INFO, "Next event is of type " + event.getEventKind());
- LOGGER.log(Level.INFO, "Notifying the listener");
+ LOGGER.log(Level.WARNING, "Next event is of type " + event.getEventKind());
+ LOGGER.log(Level.WARNING, "Notifying the listener");
}
listener.notify(event);
if (event.getEventKind() == EventKind.JOB_FINISH) {
@@ -74,7 +74,7 @@ public class ActiveJobNotificationHandler implements Runnable {
private void removeFinishedJob(JobId jobId) {
if (DEBUG) {
- LOGGER.log(Level.INFO, "Removing the job");
+ LOGGER.log(Level.WARNING, "Removing the job");
}
jobId2ActiveJobInfos.remove(jobId);
}
@@ -82,7 +82,7 @@ public class ActiveJobNotificationHandler implements Runnable {
private void removeInactiveListener(IActiveEntityEventsListener listener) {
if (!listener.isEntityActive()) {
if (DEBUG) {
- LOGGER.log(Level.INFO, "Removing the listener since it is not active anymore");
+ LOGGER.log(Level.WARNING, "Removing the listener since it is not active anymore");
}
entityEventListener.remove(listener.getEntityId());
}
@@ -90,40 +90,45 @@ public class ActiveJobNotificationHandler implements Runnable {
public IActiveEntityEventsListener getActiveEntityListener(EntityId entityId) {
if (DEBUG) {
- LOGGER.log(Level.INFO, "getActiveEntityListener(EntityId entityId) was called with entity " + entityId);
+ LOGGER.log(Level.WARNING, "getActiveEntityListener(EntityId entityId) was called with entity " + entityId);
IActiveEntityEventsListener listener = entityEventListener.get(entityId);
- LOGGER.log(Level.INFO, "Listener found: " + listener);
+ LOGGER.log(Level.WARNING, "Listener found: " + listener);
}
return entityEventListener.get(entityId);
}
public synchronized ActiveJob[] getActiveJobs() {
if (DEBUG) {
- LOGGER.log(Level.INFO, "getActiveJobs() was called");
- LOGGER.log(Level.INFO, "Number of jobs found: " + jobId2ActiveJobInfos.size());
+ LOGGER.log(Level.WARNING, "getActiveJobs() was called");
+ LOGGER.log(Level.WARNING, "Number of jobs found: " + jobId2ActiveJobInfos.size());
}
return jobId2ActiveJobInfos.values().toArray(new ActiveJob[jobId2ActiveJobInfos.size()]);
}
public boolean isActiveJob(JobId jobId) {
if (DEBUG) {
- LOGGER.log(Level.INFO, "isActiveJob(JobId jobId) called with jobId: " + jobId);
+ LOGGER.log(Level.WARNING, "isActiveJob(JobId jobId) called with jobId: " + jobId);
boolean found = jobId2ActiveJobInfos.get(jobId) != null;
- LOGGER.log(Level.INFO, "Job was found to be: " + (found ? "Active" : "Inactive"));
+ LOGGER.log(Level.WARNING, "Job was found to be: " + (found ? "Active" : "Inactive"));
}
return jobId2ActiveJobInfos.get(jobId) != null;
}
+ public EntityId getEntity(JobId jobId) {
+ ActiveJob jobInfo = jobId2ActiveJobInfos.get(jobId);
+ return jobInfo == null ? null : jobInfo.getEntityId();
+ }
+
public void notifyJobCreation(JobId jobId, JobSpecification jobSpecification) {
if (DEBUG) {
- LOGGER.log(Level.INFO,
+ LOGGER.log(Level.WARNING,
"notifyJobCreation(JobId jobId, JobSpecification jobSpecification) was called with jobId = "
+ jobId);
}
Object property = jobSpecification.getProperty(ACTIVE_ENTITY_PROPERTY_NAME);
if (property == null || !(property instanceof ActiveJob)) {
if (DEBUG) {
- LOGGER.log(Level.INFO, "Job was is not active. property found to be: " + property);
+ LOGGER.log(Level.WARNING, "Job was is not active. property found to be: " + property);
}
return;
} else {
@@ -131,7 +136,7 @@ public class ActiveJobNotificationHandler implements Runnable {
}
if (DEBUG) {
boolean found = jobId2ActiveJobInfos.get(jobId) != null;
- LOGGER.log(Level.INFO, "Job was found to be: " + (found ? "Active" : "Inactive"));
+ LOGGER.log(Level.WARNING, "Job was found to be: " + (found ? "Active" : "Inactive"));
}
ActiveJob jobInfo = jobId2ActiveJobInfos.get(jobId);
if (jobInfo != null) {
@@ -139,11 +144,12 @@ public class ActiveJobNotificationHandler implements Runnable {
IActiveEntityEventsListener listener = entityEventListener.get(entityId);
listener.notifyJobCreation(jobId, jobSpecification);
if (DEBUG) {
- LOGGER.log(Level.INFO, "Listener was notified" + jobId);
+ LOGGER.log(Level.WARNING, "Listener was notified" + jobId);
}
} else {
if (DEBUG) {
- LOGGER.log(Level.INFO, "Listener was not notified since it was not registered for the job " + jobId);
+ LOGGER.log(Level.WARNING,
+ "Listener was not notified since it was not registered for the job " + jobId);
}
}
}
@@ -154,16 +160,17 @@ public class ActiveJobNotificationHandler implements Runnable {
public synchronized IActiveEntityEventsListener[] getEventListeners() {
if (DEBUG) {
- LOGGER.log(Level.INFO, "getEventListeners() was called");
- LOGGER.log(Level.INFO, "returning " + entityEventListener.size() + " Listeners");
+ LOGGER.log(Level.WARNING, "getEventListeners() was called");
+ LOGGER.log(Level.WARNING, "returning " + entityEventListener.size() + " Listeners");
}
return entityEventListener.values().toArray(new IActiveEntityEventsListener[entityEventListener.size()]);
}
public synchronized void registerListener(IActiveEntityEventsListener listener) throws HyracksDataException {
if (DEBUG) {
- LOGGER.log(Level.INFO, "registerListener(IActiveEntityEventsListener listener) was called for the entity "
- + listener.getEntityId());
+ LOGGER.log(Level.WARNING,
+ "registerListener(IActiveEntityEventsListener listener) was called for the entity "
+ + listener.getEntityId());
}
if (entityEventListener.containsKey(listener.getEntityId())) {
throw new HyracksDataException(
@@ -174,9 +181,9 @@ public class ActiveJobNotificationHandler implements Runnable {
public synchronized void monitorJob(JobId jobId, ActiveJob activeJob) {
if (DEBUG) {
- LOGGER.log(Level.INFO, "monitorJob(JobId jobId, ActiveJob activeJob) called with job id: " + jobId);
+ LOGGER.log(Level.WARNING, "monitorJob(JobId jobId, ActiveJob activeJob) called with job id: " + jobId);
boolean found = jobId2ActiveJobInfos.get(jobId) != null;
- LOGGER.log(Level.INFO, "Job was found to be: " + (found ? "Active" : "Inactive"));
+ LOGGER.log(Level.WARNING, "Job was found to be: " + (found ? "Active" : "Inactive"));
}
if (entityEventListener.containsKey(activeJob.getEntityId())) {
if (jobId2ActiveJobInfos.containsKey(jobId)) {
@@ -184,7 +191,7 @@ public class ActiveJobNotificationHandler implements Runnable {
return;
}
if (DEBUG) {
- LOGGER.log(Level.INFO, "monitoring started for job id: " + jobId);
+ LOGGER.log(Level.WARNING, "monitoring started for job id: " + jobId);
}
jobId2ActiveJobInfos.put(jobId, activeJob);
} else {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index 5992294..2f34465 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -21,29 +21,23 @@ package org.apache.asterix.active;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.log4j.Logger;
public class ActiveManager {
+ private static final Logger LOGGER = Logger.getLogger(ActiveManager.class.getName());
private final Map<ActiveRuntimeId, IActiveRuntime> runtimes;
-
- private final IActiveRuntimeRegistry activeRuntimeRegistry;
-
private final ConcurrentFramePool activeFramePool;
-
private final String nodeId;
public ActiveManager(String nodeId, long activeMemoryBudget, int frameSize) throws HyracksDataException {
this.nodeId = nodeId;
- this.activeRuntimeRegistry = new ActiveRuntimeRegistry(nodeId);
this.activeFramePool = new ConcurrentFramePool(nodeId, activeMemoryBudget, frameSize);
this.runtimes = new ConcurrentHashMap<>();
}
- public IActiveRuntimeRegistry getActiveRuntimeRegistry() {
- return activeRuntimeRegistry;
- }
-
public ConcurrentFramePool getFramePool() {
return activeFramePool;
}
@@ -59,12 +53,37 @@ public class ActiveManager {
runtimes.remove(id);
}
- public IActiveRuntime getSubscribableRuntime(ActiveRuntimeId subscribableRuntimeId) {
- return runtimes.get(subscribableRuntimeId);
+ public IActiveRuntime getRuntime(ActiveRuntimeId runtimeId) {
+ return runtimes.get(runtimeId);
}
@Override
public String toString() {
return ActiveManager.class.getSimpleName() + "[" + nodeId + "]";
}
+
+ public void submit(ActiveManagerMessage message) {
+ switch (message.getKind()) {
+ case ActiveManagerMessage.STOP_ACTIVITY:
+ stopRuntime(message);
+ break;
+ default:
+ LOGGER.warn("Unknown message type received");
+ }
+ }
+
+ private void stopRuntime(ActiveManagerMessage message) {
+ ActiveRuntimeId runtimeId = (ActiveRuntimeId) message.getPayload();
+ IActiveRuntime runtime = runtimes.get(runtimeId);
+ if (runtime == null) {
+ LOGGER.warn("Request to stop a runtime that is not registered " + runtimeId);
+ } else {
+ try {
+ runtime.stop();
+ } catch (HyracksDataException | InterruptedException e) {
+ // TODO(till) Figure out a better way to handle failure to stop a runtime
+ LOGGER.warn("Failed to stop runtime: " + runtimeId, e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivePartitionMessage.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivePartitionMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivePartitionMessage.java
deleted file mode 100644
index 8875647..0000000
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActivePartitionMessage.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.active;
-
-import java.io.Serializable;
-
-import org.apache.asterix.common.messaging.AbstractApplicationMessage;
-import org.apache.hyracks.api.job.JobId;
-
-public class ActivePartitionMessage extends AbstractApplicationMessage {
-
- private static final long serialVersionUID = 1L;
- private final EntityId feedId;
- private final JobId jobId;
- private final Serializable payload;
-
- public ActivePartitionMessage(EntityId feedId, JobId jobId, Serializable payload) {
- this.feedId = feedId;
- this.jobId = jobId;
- this.payload = payload;
- }
-
- @Override
- public ApplicationMessageType getMessageType() {
- return ApplicationMessageType.ACTIVE_ENTITY_MESSAGE;
- }
-
- public EntityId getFeedId() {
- return feedId;
- }
-
- public JobId getJobId() {
- return jobId;
- }
-
- public Serializable getPayload() {
- return payload;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntime.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntime.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntime.java
deleted file mode 100644
index 8b0914a..0000000
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntime.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.active;
-
-public class ActiveRuntime implements IActiveRuntime {
-
- /** A unique identifier for the runtime **/
- protected final ActiveRuntimeId runtimeId;
-
- public ActiveRuntime(ActiveRuntimeId runtimeId) {
- this.runtimeId = runtimeId;;
- }
-
- @Override
- public ActiveRuntimeId getRuntimeId() {
- return runtimeId;
- }
-
- @Override
- public String toString() {
- return runtimeId.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeId.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeId.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeId.java
index 64926fd..f1f5876 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeId.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeId.java
@@ -25,20 +25,20 @@ public class ActiveRuntimeId implements Serializable {
private static final long serialVersionUID = 1L;
private final EntityId entityId;
- private final String runtimeId;
+ private final String runtimeName;
private final int partition;
private final int hashCode;
- public ActiveRuntimeId(EntityId entityId, String runtimeId, int partition) {
+ public ActiveRuntimeId(EntityId entityId, String runtimeName, int partition) {
this.entityId = entityId;
- this.runtimeId = runtimeId;
+ this.runtimeName = runtimeName;
this.partition = partition;
this.hashCode = toString().hashCode();
}
@Override
public String toString() {
- return "(" + entityId + ")" + "[" + partition + "]:" + runtimeId;
+ return "(" + entityId + ")" + "[" + partition + "]:" + runtimeName;
}
@Override
@@ -50,7 +50,7 @@ public class ActiveRuntimeId implements Serializable {
return false;
}
ActiveRuntimeId other = (ActiveRuntimeId) o;
- return other.entityId.equals(entityId) && other.getFeedRuntimeType().equals(runtimeId)
+ return other.entityId.equals(entityId) && other.getRuntimeName().equals(runtimeName)
&& other.getPartition() == partition;
}
@@ -59,19 +59,15 @@ public class ActiveRuntimeId implements Serializable {
return hashCode;
}
- public String getFeedRuntimeType() {
- return runtimeId;
+ public String getRuntimeName() {
+ return runtimeName;
}
public int getPartition() {
return partition;
}
- public String getRuntimeType() {
- return runtimeId;
- }
-
- public EntityId getFeedId() {
+ public EntityId getEntityId() {
return entityId;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeManager.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeManager.java
index 9743856..e71367a 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeManager.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeManager.java
@@ -31,7 +31,7 @@ import java.util.logging.Logger;
public class ActiveRuntimeManager {
private static final Logger LOGGER = Logger.getLogger(ActiveRuntimeManager.class.getName());
- private final Map<ActiveRuntimeId, ActiveRuntime> activeRuntimes;
+ private final Map<ActiveRuntimeId, ActiveSourceOperatorNodePushable> activeRuntimes;
private final ExecutorService executorService;
@@ -61,11 +61,11 @@ public class ActiveRuntimeManager {
}
}
- public ActiveRuntime getFeedRuntime(ActiveRuntimeId runtimeId) {
+ public ActiveSourceOperatorNodePushable getRuntime(ActiveRuntimeId runtimeId) {
return activeRuntimes.get(runtimeId);
}
- public void registerRuntime(ActiveRuntimeId runtimeId, ActiveRuntime feedRuntime) {
+ public void registerRuntime(ActiveRuntimeId runtimeId, ActiveSourceOperatorNodePushable feedRuntime) {
activeRuntimes.put(runtimeId, feedRuntime);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeRegistry.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeRegistry.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeRegistry.java
deleted file mode 100644
index 050426c..0000000
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveRuntimeRegistry.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.active;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * An implementation of the {@code IActiveRuntimeRegistry} interface.
- * Provider necessary central repository for registering/retrieving
- * artifacts/services associated with an active entity.
- */
-public class ActiveRuntimeRegistry implements IActiveRuntimeRegistry {
-
- private static final Logger LOGGER = Logger.getLogger(ActiveRuntimeRegistry.class.getName());
-
- private Map<ActiveRuntimeId, ActiveRuntimeManager> activeRuntimeManagers = new HashMap<>();
- private final String nodeId;
-
- public ActiveRuntimeRegistry(String nodeId) {
- this.nodeId = nodeId;
- }
-
- @Override
- public void deregisterRuntime(ActiveRuntimeId runtimeId) {
- try {
- ActiveRuntimeManager mgr = activeRuntimeManagers.get(runtimeId);
- if (mgr != null) {
- mgr.deregisterRuntime(runtimeId);
- mgr.close();
- activeRuntimeManagers.remove(runtimeId);
- }
- } catch (Exception e) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.log(Level.WARNING, "Exception in closing feed runtime" + e.getMessage(), e);
- }
- }
-
- }
-
- @Override
- public synchronized void registerRuntime(ActiveRuntime runtime) {
- ActiveRuntimeManager runtimeMgr = activeRuntimeManagers.get(runtime.getRuntimeId());
- if (runtimeMgr == null) {
- runtimeMgr = new ActiveRuntimeManager();
- activeRuntimeManagers.put(runtime.getRuntimeId(), runtimeMgr);
- }
- runtimeMgr.registerRuntime(runtime.getRuntimeId(), runtime);
- }
-
- @Override
- public ActiveRuntime getRuntime(ActiveRuntimeId runtimeId) {
- ActiveRuntimeManager runtimeMgr = activeRuntimeManagers.get(runtimeId);
- return runtimeMgr != null ? runtimeMgr.getFeedRuntime(runtimeId) : null;
- }
-
- @Override
- public String toString() {
- return ActiveRuntimeRegistry.class.getSimpleName() + "[" + nodeId + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
new file mode 100644
index 0000000..1cda298
--- /dev/null
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveSourceOperatorNodePushable.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.active;
+
+import org.apache.asterix.active.message.ActivePartitionMessage;
+import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
+
+public abstract class ActiveSourceOperatorNodePushable extends AbstractOperatorNodePushable implements IActiveRuntime {
+
+ protected final IHyracksTaskContext ctx;
+ protected final ActiveManager activeManager;
+ /** A unique identifier for the runtime **/
+ protected final ActiveRuntimeId runtimeId;
+ private volatile boolean done = false;
+
+ public ActiveSourceOperatorNodePushable(IHyracksTaskContext ctx, ActiveRuntimeId runtimeId) {
+ this.ctx = ctx;
+ activeManager = (ActiveManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
+ .getApplicationObject()).getActiveManager();
+ this.runtimeId = runtimeId;
+ }
+
+ @Override
+ public ActiveRuntimeId getRuntimeId() {
+ return runtimeId;
+ }
+
+ /**
+ * Starts the active job. This method must not return until the job has finished
+ *
+ * @throws HyracksDataException
+ * @throws InterruptedException
+ */
+ protected abstract void start() throws HyracksDataException, InterruptedException;
+
+ @Override
+ public final void stop() throws HyracksDataException, InterruptedException {
+ try {
+ abort();
+ } finally {
+ if (!done) {
+ synchronized (this) {
+ while (!done) {
+ wait();
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * called from a different thread. This method stops the active node and force the start() call to return
+ *
+ * @throws HyracksDataException
+ * @throws InterruptedException
+ */
+ protected abstract void abort() throws HyracksDataException, InterruptedException;
+
+ @Override
+ public String toString() {
+ return runtimeId.toString();
+ }
+
+ @Override
+ public final void initialize() throws HyracksDataException {
+ activeManager.registerRuntime(this);
+ try {
+ // notify cc that runtime has been registered
+ ctx.sendApplicationMessageToCC(new ActivePartitionMessage(runtimeId, ctx.getJobletContext().getJobId(),
+ ActivePartitionMessage.ACTIVE_RUNTIME_REGISTERED), null);
+ start();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new HyracksDataException(e);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ } finally {
+ synchronized (this) {
+ done = true;
+ notifyAll();
+ }
+ }
+ }
+
+ @Override
+ public final void deinitialize() throws HyracksDataException {
+ activeManager.deregisterRuntime(runtimeId);
+ try {
+ ctx.sendApplicationMessageToCC(new ActivePartitionMessage(runtimeId, ctx.getJobletContext().getJobId(),
+ ActivePartitionMessage.ACTIVE_RUNTIME_DEREGISTERED), null);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public final int getInputArity() {
+ return 0;
+ }
+
+ @Override
+ public final IFrameWriter getInputFrameWriter(int index) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
index 32c5c50..528c220 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntime.java
@@ -18,11 +18,20 @@
*/
package org.apache.asterix.active;
-@FunctionalInterface
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public interface IActiveRuntime {
/**
- * @return the unique runtime id associated with the feedRuntime
+ * @return the unique runtime id associated with the active runtime
+ */
+ ActiveRuntimeId getRuntimeId();
+
+ /**
+ * Stops the running activity
+ *
+ * @throws HyracksDataException
+ * @throws InterruptedException
*/
- public ActiveRuntimeId getRuntimeId();
+ void stop() throws HyracksDataException, InterruptedException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntimeRegistry.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntimeRegistry.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntimeRegistry.java
deleted file mode 100644
index b2c6f8e..0000000
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/IActiveRuntimeRegistry.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.active;
-
-import java.io.IOException;
-
-public interface IActiveRuntimeRegistry {
-
- /**
- * Allows registration of a runtime.
- *
- * @param runtime
- * @throws Exception
- */
- public void registerRuntime(ActiveRuntime runtime);
-
- /**
- * Obtain runtime corresponding to a feedRuntimeId
- *
- * @param runtimeId
- * @return
- */
- public ActiveRuntime getRuntime(ActiveRuntimeId runtimeId);
-
- /**
- * De-register a feed
- *
- * @param feedConnection
- * @throws IOException
- */
- void deregisterRuntime(ActiveRuntimeId runtimeId);
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
new file mode 100644
index 0000000..a6e1788
--- /dev/null
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActiveManagerMessage.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.active.message;
+
+import java.io.Serializable;
+
+import org.apache.asterix.common.messaging.AbstractApplicationMessage;
+
+public class ActiveManagerMessage extends AbstractApplicationMessage {
+ public static final byte STOP_ACTIVITY = 0x00;
+
+ private static final long serialVersionUID = 1L;
+ private final byte kind;
+ private final String src;
+ private final Serializable payload;
+
+ public ActiveManagerMessage(byte kind, String src, Serializable payload) {
+ this.kind = kind;
+ this.src = src;
+ this.payload = payload;
+ }
+
+ @Override
+ public ApplicationMessageType getMessageType() {
+ return ApplicationMessageType.ACTIVE_MANAGER_MESSAGE;
+ }
+
+ public Serializable getPayload() {
+ return payload;
+ }
+
+ public byte getKind() {
+ return kind;
+ }
+
+ public String getSrc() {
+ return src;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActivePartitionMessage.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActivePartitionMessage.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActivePartitionMessage.java
new file mode 100644
index 0000000..f5bdf39
--- /dev/null
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/message/ActivePartitionMessage.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.active.message;
+
+import java.io.Serializable;
+
+import org.apache.asterix.active.ActiveRuntimeId;
+import org.apache.asterix.common.messaging.AbstractApplicationMessage;
+import org.apache.hyracks.api.job.JobId;
+
+public class ActivePartitionMessage extends AbstractApplicationMessage {
+
+ public static final byte ACTIVE_RUNTIME_REGISTERED = 0x00;
+ public static final byte ACTIVE_RUNTIME_DEREGISTERED = 0x01;
+ private static final long serialVersionUID = 1L;
+ private final ActiveRuntimeId activeRuntimeId;
+ private final JobId jobId;
+ private final Serializable payload;
+ private final byte event;
+
+ public ActivePartitionMessage(ActiveRuntimeId activeRuntimeId, JobId jobId, byte event) {
+ this(activeRuntimeId, jobId, event, null);
+ }
+
+ public ActivePartitionMessage(ActiveRuntimeId activeRuntimeId, JobId jobId, byte event, Serializable payload) {
+ this.activeRuntimeId = activeRuntimeId;
+ this.jobId = jobId;
+ this.event = event;
+ this.payload = payload;
+ }
+
+ @Override
+ public ApplicationMessageType getMessageType() {
+ return ApplicationMessageType.ACTIVE_ENTITY_TO_CC_MESSAGE;
+ }
+
+ public ActiveRuntimeId getActiveRuntimeId() {
+ return activeRuntimeId;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public Serializable getPayload() {
+ return payload;
+ }
+
+ public byte getEvent() {
+ return event;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
new file mode 100644
index 0000000..ac3bc03
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.algebra.base;
+
+import org.apache.asterix.common.api.IExtension;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+
+/**
+ * An interface for language extensions
+ */
+public interface ILangExtension extends IExtension {
+
+ public enum Language {
+ AQL,
+ SQLPP
+ }
+
+ @Override
+ default ExtensionKind getExtensionKind() {
+ return ExtensionKind.LANG;
+ }
+
+ ILangCompilationProvider getLangCompilationProvider(Language lang);
+
+ //TODO(amoudi/yingyi) this is not a good way to extend re-write rules. introduce rewrite-rule-provider.
+ /**
+ * Called by the compiler when the unnest function is an extension function.
+ * Provides a way to add additional types of datasources
+ *
+ * @param opRef
+ * @param context
+ * @param unnestOp
+ * @param unnestExpr
+ * @param functionCallExpr
+ * @return true if transformation was successful, false otherwise
+ * @throws AlgebricksException
+ */
+ boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnestOp,
+ ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr) throws AlgebricksException;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java
new file mode 100644
index 0000000..ae67337
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.algebra.extension;
+
+import org.apache.asterix.common.api.ExtensionId;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+
+/**
+ * an Identifier for an extension function
+ */
+public class ExtensionFunctionIdentifier extends FunctionIdentifier {
+ //TODO (till): remove this classs
+
+ private static final long serialVersionUID = 1L;
+ private final ExtensionId extensionId;
+
+ /**
+ * Create an identifier for an external function
+ *
+ * @param namespace
+ * @param name
+ * @param extensionId
+ */
+ public ExtensionFunctionIdentifier(String namespace, String name, ExtensionId extensionId) {
+ super(namespace, name);
+ this.extensionId = extensionId;
+ }
+
+ /**
+ * Create an identifier for an external function
+ *
+ * @param namespace
+ * @param name
+ * @param arity
+ * @param extensionId
+ */
+ public ExtensionFunctionIdentifier(String namespace, String name, int arity, ExtensionId extensionId) {
+ super(namespace, name, arity);
+ this.extensionId = extensionId;
+ }
+
+ public ExtensionId getExtensionId() {
+ return extensionId;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode() + extensionId.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o instanceof ExtensionFunctionIdentifier) {
+ ExtensionFunctionIdentifier oId = (ExtensionFunctionIdentifier) o;
+ return super.equals(o) && extensionId.equals(oId.getExtensionId());
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
new file mode 100644
index 0000000..0a8402c
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.algebra.extension;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+
+public interface IAlgebraExtensionManager {
+
+ boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnestOp,
+ ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr) throws AlgebricksException;
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IExtensionStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IExtensionStatement.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IExtensionStatement.java
new file mode 100644
index 0000000..9e2241f
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IExtensionStatement.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.algebra.extension;
+
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+
+/**
+ * An interface that provides an extension mechanism to extend a language with additional statements
+ */
+public interface IExtensionStatement extends Statement {
+
+ @Override
+ default byte getKind() {
+ return Kind.EXTENSION;
+ }
+
+ /**
+ * Called when the {@code IQueryTranslator} encounters an extension statement.
+ * An implementation class should implement the actual processing of the statement in this method.
+ *
+ * @param queryTranslator
+ * @param metadataProvider
+ * @param statementExecutor
+ * @param hcc
+ * @throws Exception
+ */
+ void handle(IStatementExecutor statementExecutor, AqlMetadataProvider metadataProvider,
+ IHyracksClientConnection hcc) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 720de76..b29f743 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -22,6 +22,7 @@ package org.apache.asterix.optimizer.base;
import java.util.LinkedList;
import java.util.List;
+import org.apache.asterix.algebra.extension.IAlgebraExtensionManager;
import org.apache.asterix.optimizer.rules.AddEquivalenceClassForRecordConstructorRule;
import org.apache.asterix.optimizer.rules.AsterixExtractFunctionsFromJoinConditionRule;
import org.apache.asterix.optimizer.rules.AsterixInlineVariablesRule;
@@ -152,7 +153,9 @@ public final class RuleCollections {
return autogen;
}
- public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() {
+ //TODO(amoudi/yingyi): refactor this to use a provider instead of passing the extensionManager
+ public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection(
+ IAlgebraExtensionManager algebraExtensionManager) {
List<IAlgebraicRewriteRule> normalization = new LinkedList<>();
normalization.add(new ResolveVariableRule());
normalization.add(new IntroduceUnnestForCollectionToSequenceRule());
@@ -174,7 +177,7 @@ public final class RuleCollections {
normalization.add(new ExtractCommonExpressionsRule());
normalization.add(new ConstantFoldingRule());
normalization.add(new RemoveRedundantSelectRule());
- normalization.add(new UnnestToDataScanRule());
+ normalization.add(new UnnestToDataScanRule(algebraExtensionManager));
normalization.add(new MetaFunctionToMetaVariableRule());
normalization.add(new FuzzyEqRule());
normalization.add(new SimilarityCheckRule());
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
index 605ddb4..1fa7730 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.metadata.declared.AqlDataSource;
import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
-import org.apache.asterix.metadata.declared.FeedDataSource;
+import org.apache.asterix.metadata.declared.IMutationDataSource;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
import org.apache.asterix.om.types.IAType;
@@ -86,11 +86,11 @@ public class MetaFunctionToMetaVariableRule implements IAlgebraicRewriteRule {
LogicalVariable metaVar = dataSource.getMetaVariable(allVars);
LogicalExpressionReferenceTransform currentTransformer = null;
if (dataSource.getDatasourceType() == AqlDataSourceType.FEED) {
- FeedDataSource fds = (FeedDataSource) dataSource;
- if (fds.isChange()) {
+ IMutationDataSource mds = (IMutationDataSource) dataSource;
+ if (mds.isChange()) {
transformers = new ArrayList<>();
- transformers.add(new MetaKeyExpressionReferenceTransform(fds.getPkVars(allVars),
- fds.getKeyAccessExpression()));
+ transformers.add(new MetaKeyExpressionReferenceTransform(mds.getPkVars(allVars),
+ mds.getKeyAccessExpression()));
} else if (metaVar != null) {
transformers = new ArrayList<>();
transformers.add(new MetaKeyToFieldAccessTransform(metaVar));
@@ -98,7 +98,7 @@ public class MetaFunctionToMetaVariableRule implements IAlgebraicRewriteRule {
}
if (!dataSource.hasMeta() && transformers == null) {
return inputTransfomer;
- };
+ }
if (metaVar != null) {
currentTransformer = new LogicalExpressionReferenceTransform(dataVar, metaVar);
}
@@ -286,7 +286,7 @@ class MetaKeyToFieldAccessTransform implements ILogicalExpressionReferenceTransf
throw new AlgebricksException("Unsupported field name type " + fieldNameType.getTypeTag());
}
IFunctionInfo finfoAccess = FunctionUtil.getFunctionInfo(functionIdentifier);
- ArrayList<Mutable<ILogicalExpression>> argExprs = new ArrayList<Mutable<ILogicalExpression>>(2);
+ ArrayList<Mutable<ILogicalExpression>> argExprs = new ArrayList<>(2);
argExprs.add(new MutableObject<>(new VariableReferenceExpression(metaVar)));
argExprs.add(new MutableObject<>(fieldNameExpression));
exprRef.setValue(new ScalarFunctionCallExpression(finfoAccess, argExprs));
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
index bb382f0..17dec7c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -18,11 +18,10 @@
*/
package org.apache.asterix.optimizer.rules;
-import static org.apache.asterix.om.util.ConstantExpressionUtil.getStringArgument;
-
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.algebra.extension.IAlgebraExtensionManager;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
import org.apache.asterix.external.util.ExternalDataUtils;
@@ -45,6 +44,7 @@ import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.util.ConstantExpressionUtil;
import org.apache.asterix.optimizer.rules.util.EquivalenceClassUtils;
import org.apache.asterix.translator.util.PlanTranslationUtil;
import org.apache.commons.lang3.mutable.Mutable;
@@ -68,6 +68,11 @@ import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependenc
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
+ private final IAlgebraExtensionManager algebraExtensionManager;
+
+ public UnnestToDataScanRule(IAlgebraExtensionManager algebraExtensionManager) {
+ this.algebraExtensionManager = algebraExtensionManager;
+ }
@Override
public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
@@ -150,19 +155,17 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType,
metaRecordType, dataset, context);
return true;
- }
-
- if (fid.equals(AsterixBuiltinFunctions.FEED_COLLECT)) {
+ } else if (fid.equals(AsterixBuiltinFunctions.FEED_COLLECT)) {
if (unnest.getPositionalVariable() != null) {
throw new AlgebricksException("No positional variables are allowed over feeds.");
}
- String dataverse = getStringArgument(f, 0);
- String sourceFeedName = getStringArgument(f, 1);
- String getTargetFeed = getStringArgument(f, 2);
- String subscriptionLocation = getStringArgument(f, 3);
- String targetDataset = getStringArgument(f, 4);
- String outputType = getStringArgument(f, 5);
+ String dataverse = ConstantExpressionUtil.getStringArgument(f, 0);
+ String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1);
+ String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2);
+ String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3);
+ String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4);
+ String outputType = ConstantExpressionUtil.getStringArgument(f, 5);
AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
@@ -190,10 +193,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
}
// Does it produce pk?
if (ds.isChange()) {
- int numOfPKs = ds.getPkTypes().size();
- for (int i = 0; i < numOfPKs; i++) {
- feedDataScanOutputVariables.addAll(pkVars);
- }
+ feedDataScanOutputVariables.addAll(pkVars);
}
DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds);
@@ -202,8 +202,9 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
opRef.setValue(scan);
context.computeAndSetTypeEnvironmentForOperator(scan);
return true;
+ } else if (algebraExtensionManager != null) {
+ return algebraExtensionManager.unnestToDataScan(opRef, context, unnest, unnestExpr, f);
}
-
}
return false;
@@ -297,6 +298,6 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
dataverseName = datasetNameComponents[0];
datasetName = datasetNameComponents[1];
}
- return new Pair<String, String>(dataverseName, datasetName);
+ return new Pair<>(dataverseName, datasetName);
}
}
\ No newline at end of file
[13/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index cd088c1..5728947 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -27,6 +27,9 @@ import java.util.List;
import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
import org.apache.asterix.api.common.Job.SubmissionMode;
+import org.apache.asterix.app.cc.CompilerExtensionManager;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.common.app.SessionConfig;
import org.apache.asterix.common.config.AsterixCompilerProperties;
import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.common.config.OptimizationConfUtil;
@@ -50,10 +53,10 @@ import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.optimizer.base.RuleCollections;
-import org.apache.asterix.result.ResultUtils;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.transaction.management.service.transaction.JobIdFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -89,19 +92,21 @@ import org.json.JSONException;
* to Hyracks through the Hyracks client interface.
*/
public class APIFramework {
- public static final String HTML_STATEMENT_SEPARATOR = "<!-- BEGIN -->";
private final IRewriterFactory rewriterFactory;
private final IAstPrintVisitorFactory astPrintVisitorFactory;
private final ILangExpressionToPlanTranslatorFactory translatorFactory;
+ private final CompilerExtensionManager cExtensionManager;
- public APIFramework(ILangCompilationProvider compilationProvider) {
+ public APIFramework(ILangCompilationProvider compilationProvider, CompilerExtensionManager cExtensionManager) {
this.rewriterFactory = compilationProvider.getRewriterFactory();
this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory();
this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory();
+ this.cExtensionManager = cExtensionManager;
}
- private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultLogicalRewrites() {
+ private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>
+ buildDefaultLogicalRewrites(CompilerExtensionManager ccExtensionManager) {
List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<>();
SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
@@ -109,14 +114,16 @@ public class APIFramework {
defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildInitialTranslationRuleCollection()));
defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildTypeInferenceRuleCollection()));
defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildAutogenerateIDRuleCollection()));
- defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection()));
+ defaultLogicalRewrites
+ .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager)));
defaultLogicalRewrites
.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
// fj
defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildFuzzyJoinRuleCollection()));
//
- defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection()));
+ defaultLogicalRewrites
+ .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager)));
defaultLogicalRewrites
.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
@@ -251,7 +258,7 @@ public class APIFramework {
HeuristicCompilerFactoryBuilder builder =
new HeuristicCompilerFactoryBuilder(AqlOptimizationContextFactory.INSTANCE);
builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
- builder.setLogicalRewrites(buildDefaultLogicalRewrites());
+ builder.setLogicalRewrites(buildDefaultLogicalRewrites(cExtensionManager));
builder.setPhysicalRewrites(buildDefaultPhysicalRewrites());
IDataFormat format = queryMetadataProvider.getFormat();
ICompilerFactory compilerFactory = builder.create();
@@ -289,7 +296,7 @@ public class APIFramework {
try {
LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
- ResultUtils.displayResults(pvisitor.get().toString(), conf, new ResultUtils.Stats(), null);
+ ResultUtil.displayResults(pvisitor.get().toString(), conf, new Stats(), null);
return null;
} catch (IOException e) {
throw new AlgebricksException(e);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
deleted file mode 100644
index 4bb09d4..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++ /dev/null
@@ -1,465 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.api.common;
-
-import java.io.IOException;
-import java.rmi.RemoteException;
-import java.rmi.server.UnicastRemoteObject;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.active.ActiveManager;
-import org.apache.asterix.common.api.AsterixThreadExecutor;
-import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.cluster.ClusterPartition;
-import org.apache.asterix.common.config.AsterixBuildProperties;
-import org.apache.asterix.common.config.AsterixCompilerProperties;
-import org.apache.asterix.common.config.AsterixExternalProperties;
-import org.apache.asterix.common.config.AsterixFeedProperties;
-import org.apache.asterix.common.config.AsterixMetadataProperties;
-import org.apache.asterix.common.config.AsterixPropertiesAccessor;
-import org.apache.asterix.common.config.AsterixReplicationProperties;
-import org.apache.asterix.common.config.AsterixStorageProperties;
-import org.apache.asterix.common.config.AsterixTransactionProperties;
-import org.apache.asterix.common.config.IAsterixPropertiesProvider;
-import org.apache.asterix.common.context.AsterixFileMapManager;
-import org.apache.asterix.common.context.DatasetLifecycleManager;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.replication.IRemoteRecoveryManager;
-import org.apache.asterix.common.replication.IReplicaResourcesManager;
-import org.apache.asterix.common.replication.IReplicationChannel;
-import org.apache.asterix.common.replication.IReplicationManager;
-import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
-import org.apache.asterix.common.transactions.IRecoveryManager;
-import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
-import org.apache.asterix.common.transactions.ITransactionSubsystem;
-import org.apache.asterix.external.library.ExternalLibraryManager;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataNode;
-import org.apache.asterix.metadata.api.IAsterixStateProxy;
-import org.apache.asterix.metadata.api.IMetadataNode;
-import org.apache.asterix.metadata.bootstrap.MetadataBootstrap;
-import org.apache.asterix.metadata.bootstrap.MetadataIndexImmutableProperties;
-import org.apache.asterix.om.util.AsterixClusterProperties;
-import org.apache.asterix.replication.management.ReplicationChannel;
-import org.apache.asterix.replication.management.ReplicationManager;
-import org.apache.asterix.replication.recovery.RemoteRecoveryManager;
-import org.apache.asterix.replication.storage.ReplicaResourcesManager;
-import org.apache.asterix.transaction.management.resource.GlobalResourceIdFactoryProvider;
-import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
-import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepositoryFactory;
-import org.apache.asterix.transaction.management.service.transaction.TransactionSubsystem;
-import org.apache.hyracks.api.application.IApplicationConfig;
-import org.apache.hyracks.api.application.INCApplicationContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
-import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
-import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory;
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
-import org.apache.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
-import org.apache.hyracks.storage.common.buffercache.DelayPageCleanerPolicy;
-import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
-import org.apache.hyracks.storage.common.buffercache.IPageCleanerPolicy;
-import org.apache.hyracks.storage.common.buffercache.IPageReplacementStrategy;
-import org.apache.hyracks.storage.common.file.IFileMapManager;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
-import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
-import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
-import org.apache.hyracks.storage.common.file.IResourceIdFactory;
-
-public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAsterixPropertiesProvider {
- private static final Logger LOGGER = Logger.getLogger(AsterixAppRuntimeContext.class.getName());
-
- private ILSMMergePolicyFactory metadataMergePolicyFactory;
- private final INCApplicationContext ncApplicationContext;
-
- private AsterixCompilerProperties compilerProperties;
- private AsterixExternalProperties externalProperties;
- private AsterixMetadataProperties metadataProperties;
- private AsterixStorageProperties storageProperties;
- private AsterixTransactionProperties txnProperties;
- private AsterixFeedProperties feedProperties;
- private AsterixBuildProperties buildProperties;
- private AsterixReplicationProperties replicationProperties;
-
- private AsterixThreadExecutor threadExecutor;
- private IDatasetLifecycleManager datasetLifecycleManager;
- private IFileMapManager fileMapManager;
- private IBufferCache bufferCache;
- private ITransactionSubsystem txnSubsystem;
-
- private ILSMIOOperationScheduler lsmIOScheduler;
- private PersistentLocalResourceRepository localResourceRepository;
- private IResourceIdFactory resourceIdFactory;
- private IIOManager ioManager;
- private boolean isShuttingdown;
-
- private ActiveManager activeManager;
-
- private IReplicationChannel replicationChannel;
- private IReplicationManager replicationManager;
- private IRemoteRecoveryManager remoteRecoveryManager;
- private IReplicaResourcesManager replicaResourcesManager;
- private final int metadataRmiPort;
-
- private ILibraryManager libraryManager;
-
- public AsterixAppRuntimeContext(INCApplicationContext ncApplicationContext, int metadataRmiPort)
- throws AsterixException {
- this.ncApplicationContext = ncApplicationContext;
- // Determine whether to use old-style asterix-configuration.xml or new-style configuration.
- // QQQ strip this out eventually
- AsterixPropertiesAccessor propertiesAccessor;
- IApplicationConfig cfg = ncApplicationContext.getAppConfig();
- // QQQ this is NOT a good way to determine whether the config is valid
- if (cfg.getString("cc", "cluster.address") != null) {
- propertiesAccessor = new AsterixPropertiesAccessor(cfg);
- } else {
- propertiesAccessor = new AsterixPropertiesAccessor();
- }
- compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
- externalProperties = new AsterixExternalProperties(propertiesAccessor);
- metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
- storageProperties = new AsterixStorageProperties(propertiesAccessor);
- txnProperties = new AsterixTransactionProperties(propertiesAccessor);
- feedProperties = new AsterixFeedProperties(propertiesAccessor);
- buildProperties = new AsterixBuildProperties(propertiesAccessor);
- replicationProperties =
- new AsterixReplicationProperties(propertiesAccessor, AsterixClusterProperties.INSTANCE.getCluster());
- this.metadataRmiPort = metadataRmiPort;
- }
-
- @Override
- public void initialize(boolean initialRun) throws IOException, ACIDException {
- Logger.getLogger("org.apache").setLevel(externalProperties.getLogLevel());
-
- threadExecutor = new AsterixThreadExecutor(ncApplicationContext.getThreadFactory());
- fileMapManager = new AsterixFileMapManager();
- ICacheMemoryAllocator allocator = new HeapBufferAllocator();
- IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
- ioManager = ncApplicationContext.getRootContext().getIOManager();
- IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator,
- storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
-
- AsynchronousScheduler.INSTANCE.init(ncApplicationContext.getThreadFactory());
- lsmIOScheduler = AsynchronousScheduler.INSTANCE;
-
- metadataMergePolicyFactory = new PrefixMergePolicyFactory();
-
- ILocalResourceRepositoryFactory persistentLocalResourceRepositoryFactory =
- new PersistentLocalResourceRepositoryFactory(ioManager, ncApplicationContext.getNodeId(),
- metadataProperties);
-
- localResourceRepository =
- (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory.createRepository();
-
- IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider =
- new AsterixAppRuntimeContextProviderForRecovery(this);
- txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider,
- txnProperties);
-
- IRecoveryManager recoveryMgr = txnSubsystem.getRecoveryManager();
- SystemState systemState = recoveryMgr.getSystemState();
- if (initialRun || systemState == SystemState.NEW_UNIVERSE) {
- //delete any storage data before the resource factory is initialized
- localResourceRepository.deleteStorageData(true);
- }
- initializeResourceIdFactory();
-
- datasetLifecycleManager = new DatasetLifecycleManager(storageProperties, localResourceRepository,
- MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager(),
- ioManager.getIODevices().size());
-
- isShuttingdown = false;
-
- activeManager = new ActiveManager(ncApplicationContext.getNodeId(),
- feedProperties.getMemoryComponentGlobalBudget(), compilerProperties.getFrameSize());
-
- if (replicationProperties.isReplicationEnabled()) {
- String nodeId = ncApplicationContext.getNodeId();
-
- replicaResourcesManager = new ReplicaResourcesManager(localResourceRepository, metadataProperties);
-
- replicationManager = new ReplicationManager(nodeId, replicationProperties, replicaResourcesManager,
- txnSubsystem.getLogManager(), asterixAppRuntimeContextProvider);
-
- //pass replication manager to replication required object
- //LogManager to replicate logs
- txnSubsystem.getLogManager().setReplicationManager(replicationManager);
-
- //PersistentLocalResourceRepository to replicate metadata files and delete backups on drop index
- localResourceRepository.setReplicationManager(replicationManager);
-
- /**
- * add the partitions that will be replicated in this node as inactive partitions
- */
- //get nodes which replicate to this node
- Set<String> replicationClients = replicationProperties.getNodeReplicationClients(nodeId);
- //remove the node itself
- replicationClients.remove(nodeId);
- for (String clientId : replicationClients) {
- //get the partitions of each client
- ClusterPartition[] clientPartitions = metadataProperties.getNodePartitions().get(clientId);
- for (ClusterPartition partition : clientPartitions) {
- localResourceRepository.addInactivePartition(partition.getPartitionId());
- }
- }
-
- //initialize replication channel
- replicationChannel = new ReplicationChannel(nodeId, replicationProperties, txnSubsystem.getLogManager(),
- replicaResourcesManager, replicationManager, ncApplicationContext,
- asterixAppRuntimeContextProvider);
-
- remoteRecoveryManager = new RemoteRecoveryManager(replicationManager, this, replicationProperties);
-
- bufferCache =
- new BufferCache(ioManager, prs, pcp, fileMapManager, storageProperties.getBufferCacheMaxOpenFiles(),
- ncApplicationContext.getThreadFactory(), replicationManager);
- } else {
- bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager,
- storageProperties.getBufferCacheMaxOpenFiles(), ncApplicationContext.getThreadFactory());
- }
-
- // The order of registration is important. The buffer cache must registered before recovery and transaction managers.
- //Notes: registered components are stopped in reversed order
- ILifeCycleComponentManager lccm = ncApplicationContext.getLifeCycleComponentManager();
- lccm.register((ILifeCycleComponent) bufferCache);
- /**
- * LogManager must be stopped after RecoveryManager, DatasetLifeCycleManager, and ReplicationManager
- * to process any logs that might be generated during stopping these components
- */
- lccm.register((ILifeCycleComponent) txnSubsystem.getLogManager());
- /**
- * ReplicationManager must be stopped after indexLifecycleManager and recovery manager
- * so that any logs/files generated during closing datasets or checkpoints are sent to remote replicas
- */
- if (replicationManager != null) {
- lccm.register(replicationManager);
- }
- lccm.register((ILifeCycleComponent) txnSubsystem.getRecoveryManager());
- /**
- * Stopping indexLifecycleManager will flush and close all datasets.
- */
- lccm.register((ILifeCycleComponent) datasetLifecycleManager);
- lccm.register((ILifeCycleComponent) txnSubsystem.getTransactionManager());
- lccm.register((ILifeCycleComponent) txnSubsystem.getLockManager());
-
- /**
- * Initializes the library manager.
- */
- libraryManager = new ExternalLibraryManager();
- }
-
- @Override
- public boolean isShuttingdown() {
- return isShuttingdown;
- }
-
- @Override
- public void setShuttingdown(boolean isShuttingdown) {
- this.isShuttingdown = isShuttingdown;
- }
-
- @Override
- public void deinitialize() throws HyracksDataException {
- }
-
- @Override
- public IBufferCache getBufferCache() {
- return bufferCache;
- }
-
- @Override
- public IFileMapProvider getFileMapManager() {
- return fileMapManager;
- }
-
- @Override
- public ITransactionSubsystem getTransactionSubsystem() {
- return txnSubsystem;
- }
-
- @Override
- public IDatasetLifecycleManager getDatasetLifecycleManager() {
- return datasetLifecycleManager;
- }
-
- @Override
- public double getBloomFilterFalsePositiveRate() {
- return storageProperties.getBloomFilterFalsePositiveRate();
- }
-
- @Override
- public ILSMIOOperationScheduler getLSMIOScheduler() {
- return lsmIOScheduler;
- }
-
- @Override
- public ILocalResourceRepository getLocalResourceRepository() {
- return localResourceRepository;
- }
-
- @Override
- public IResourceIdFactory getResourceIdFactory() {
- return resourceIdFactory;
- }
-
- @Override
- public IIOManager getIOManager() {
- return ioManager;
- }
-
- @Override
- public AsterixStorageProperties getStorageProperties() {
- return storageProperties;
- }
-
- @Override
- public AsterixTransactionProperties getTransactionProperties() {
- return txnProperties;
- }
-
- @Override
- public AsterixCompilerProperties getCompilerProperties() {
- return compilerProperties;
- }
-
- @Override
- public AsterixMetadataProperties getMetadataProperties() {
- return metadataProperties;
- }
-
- @Override
- public AsterixExternalProperties getExternalProperties() {
- return externalProperties;
- }
-
- @Override
- public AsterixFeedProperties getFeedProperties() {
- return feedProperties;
- }
-
- @Override
- public AsterixBuildProperties getBuildProperties() {
- return buildProperties;
- }
-
- @Override
- public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID) {
- return datasetLifecycleManager.getOperationTracker(datasetID);
- }
-
- @Override
- public AsterixThreadExecutor getThreadExecutor() {
- return threadExecutor;
- }
-
- @Override
- public ILSMMergePolicyFactory getMetadataMergePolicyFactory() {
- return metadataMergePolicyFactory;
- }
-
- @Override
- public ActiveManager getFeedManager() {
- return activeManager;
- }
-
- @Override
- public AsterixReplicationProperties getReplicationProperties() {
- return replicationProperties;
- }
-
- @Override
- public IReplicationChannel getReplicationChannel() {
- return replicationChannel;
- }
-
- @Override
- public IReplicaResourcesManager getReplicaResourcesManager() {
- return replicaResourcesManager;
- }
-
- @Override
- public IRemoteRecoveryManager getRemoteRecoveryManager() {
- return remoteRecoveryManager;
- }
-
- @Override
- public IReplicationManager getReplicationManager() {
- return replicationManager;
- }
-
- @Override
- public ILibraryManager getLibraryManager() {
- return libraryManager;
- }
-
- @Override
- public void initializeResourceIdFactory() throws HyracksDataException {
- resourceIdFactory = new GlobalResourceIdFactoryProvider(ncApplicationContext).createResourceIdFactory();
- }
-
- @Override
- public void initializeMetadata(boolean newUniverse) throws Exception {
- IAsterixStateProxy proxy;
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Bootstrapping metadata");
- }
- MetadataNode.INSTANCE.initialize(this);
-
- proxy = (IAsterixStateProxy) ncApplicationContext.getDistributedState();
- if (proxy == null) {
- throw new IllegalStateException("Metadata node cannot access distributed state");
- }
-
- // This is a special case, we just give the metadataNode directly.
- // This way we can delay the registration of the metadataNode until
- // it is completely initialized.
- MetadataManager.INSTANCE = new MetadataManager(proxy, MetadataNode.INSTANCE);
- MetadataBootstrap.startUniverse(this, ncApplicationContext, newUniverse);
- MetadataBootstrap.startDDLRecovery();
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Metadata node bound");
- }
- }
-
- @Override
- public void exportMetadataNodeStub() throws RemoteException {
- IMetadataNode stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, metadataRmiPort);
- ((IAsterixStateProxy) ncApplicationContext.getDistributedState()).setMetadataNode(stub);
- }
-
- @Override
- public void unexportMetadataNodeStub() throws RemoteException {
- UnicastRemoteObject.unexportObject(MetadataNode.INSTANCE, false);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
index 7ac5036..265025f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.api.common;
+import org.apache.asterix.app.nc.AsterixNCAppRuntimeContext;
import org.apache.asterix.common.api.AsterixThreadExecutor;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.api.IDatasetLifecycleManager;
@@ -33,9 +34,9 @@ import org.apache.hyracks.storage.common.file.IResourceIdFactory;
public class AsterixAppRuntimeContextProviderForRecovery implements IAsterixAppRuntimeContextProvider {
- private final AsterixAppRuntimeContext asterixAppRuntimeContext;
+ private final AsterixNCAppRuntimeContext asterixAppRuntimeContext;
- public AsterixAppRuntimeContextProviderForRecovery(AsterixAppRuntimeContext asterixAppRuntimeContext) {
+ public AsterixAppRuntimeContextProviderForRecovery(AsterixNCAppRuntimeContext asterixAppRuntimeContext) {
this.asterixAppRuntimeContext = asterixAppRuntimeContext;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 4b00195..b1bf418 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -24,6 +24,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import org.apache.asterix.common.config.AsterixPropertiesAccessor;
import org.apache.asterix.common.config.GlobalConfig;
@@ -42,7 +44,7 @@ import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.NodeControllerService;
public class AsterixHyracksIntegrationUtil {
-
+ private static final Logger LOGGER = Logger.getLogger(AsterixHyracksIntegrationUtil.class.getName());
private static final String IO_DIR_KEY = "java.io.tmpdir";
public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 1098;
public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 1099;
@@ -75,7 +77,7 @@ public class AsterixHyracksIntegrationUtil {
try {
nodeControllerService.start();
} catch (Exception e) {
- e.printStackTrace();
+ LOGGER.log(Level.SEVERE, e.getMessage(), e);
}
}
};
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
deleted file mode 100644
index ba205d1..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.api.common;
-
-import java.io.PrintWriter;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
-
-/**
- * SessionConfig captures several different parameters for controlling
- * the execution of an APIFramework call.
- * <li>It specifies how the execution will proceed (for instance,
- * whether to optimize, or whether to execute at all).
- * <li>It allows you specify where the primary execution output will
- * be sent.
- * <li>It also allows you to request additional output for optional
- * out-of-band data about the execution (query plan, etc).
- * <li>It allows you to specify the output format for the primary
- * execution output - LOSSLESS_JSON, CSV, etc.
- * <li>It allows you to specify output format-specific parameters.
- */
-
-public class SessionConfig {
- /**
- * Used to specify the output format for the primary execution.
- */
- public enum OutputFormat {
- ADM,
- CSV,
- CLEAN_JSON,
- LOSSLESS_JSON
- };
-
- /**
- * Produce out-of-band output for Hyracks Job.
- */
- public static final String OOB_HYRACKS_JOB = "oob-hyracks-job";
-
- /**
- * Produce out-of-band output for Expression Tree.
- */
- public static final String OOB_EXPR_TREE = "oob-expr-tree";
-
- /**
- * Produce out-of-band output for Rewritten Expression Tree.
- */
- public static final String OOB_REWRITTEN_EXPR_TREE = "oob-rewritten-expr-tree";
-
- /**
- * Produce out-of-band output for Logical Plan.
- */
- public static final String OOB_LOGICAL_PLAN = "oob-logical-plan";
-
- /**
- * Produce out-of-band output for Optimized Logical Plan.
- */
- public static final String OOB_OPTIMIZED_LOGICAL_PLAN = "oob-optimized-logical-plan";
-
- /**
- * Format flag: print only physical ops (for optimizer tests).
- */
- public static final String FORMAT_ONLY_PHYSICAL_OPS = "format-only-physical-ops";
-
- /**
- * Format flag: wrap out-of-band data in HTML.
- */
- public static final String FORMAT_HTML = "format-html";
-
- /**
- * Format flag: print CSV header line.
- */
- public static final String FORMAT_CSV_HEADER = "format-csv-header";
-
- /**
- * Format flag: wrap results in outer array brackets (JSON or ADM).
- */
- public static final String FORMAT_WRAPPER_ARRAY = "format-wrapper-array";
-
- /**
- * Format flag: indent JSON results.
- */
- public static final String FORMAT_INDENT_JSON = "indent-json";
-
- /**
- * Format flag: quote records in the results array.
- */
- public static final String FORMAT_QUOTE_RECORD = "quote-record";
-
- public interface ResultDecorator {
- AlgebricksAppendable append(AlgebricksAppendable app) throws AlgebricksException;
- }
-
- // Standard execution flags.
- private final boolean executeQuery;
- private final boolean generateJobSpec;
- private final boolean optimize;
-
- // Output path for primary execution.
- private final PrintWriter out;
-
- // Output format.
- private final OutputFormat fmt;
-
- private final ResultDecorator preResultDecorator;
- private final ResultDecorator postResultDecorator;
-
- // Flags.
- private final Map<String, Boolean> flags;
-
- /**
- * Create a SessionConfig object with all default values:
- * - All format flags set to "false".
- * - All out-of-band outputs set to "null".
- * - "Optimize" set to "true".
- * - "Execute Query" set to "true".
- * - "Generate Job Spec" set to "true".
- *
- * @param out
- * PrintWriter for execution output.
- * @param fmt
- * Output format for execution output.
- */
- public SessionConfig(PrintWriter out, OutputFormat fmt) {
- this(out, fmt, null, null, true, true, true);
- }
-
- public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
- ResultDecorator postResultDecorator) {
- this(out, fmt, preResultDecorator, postResultDecorator, true, true, true);
- }
-
- public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery,
- boolean generateJobSpec) {
- this(out, fmt, null, null, optimize, executeQuery, generateJobSpec);
- }
-
- /**
- * Create a SessionConfig object with all optional values set to defaults:
- * - All format flags set to "false".
- * - All out-of-band outputs set to "false".
- *
- * @param out
- * PrintWriter for execution output.
- * @param fmt
- * Output format for execution output.
- * @param optimize
- * Whether to optimize the execution.
- * @param executeQuery
- * Whether to execute the query or not.
- * @param generateJobSpec
- * Whether to generate the Hyracks job specification (if
- * false, job cannot be executed).
- */
- public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
- ResultDecorator postResultDecorator, boolean optimize, boolean executeQuery, boolean generateJobSpec) {
- this.out = out;
- this.fmt = fmt;
- this.preResultDecorator = preResultDecorator;
- this.postResultDecorator = postResultDecorator;
- this.optimize = optimize;
- this.executeQuery = executeQuery;
- this.generateJobSpec = generateJobSpec;
- this.flags = new HashMap<String, Boolean>();
- }
-
- /**
- * Retrieve the PrintWriter to produce output to.
- */
- public PrintWriter out() {
- return this.out;
- }
-
- /**
- * Retrieve the OutputFormat for this execution.
- */
- public OutputFormat fmt() {
- return this.fmt;
- }
-
- public AlgebricksAppendable resultPrefix(AlgebricksAppendable app) throws AlgebricksException {
- return this.preResultDecorator != null ? this.preResultDecorator.append(app) : app;
- };
-
- public AlgebricksAppendable resultPostfix(AlgebricksAppendable app) throws AlgebricksException {
- return this.postResultDecorator != null ? this.postResultDecorator.append(app) : app;
- };
-
- /**
- * Retrieve the value of the "execute query" flag.
- */
- public boolean isExecuteQuery() {
- return executeQuery;
- }
-
- /**
- * Retrieve the value of the "optimize" flag.
- */
- public boolean isOptimize() {
- return optimize;
- }
-
- /**
- * Retrieve the value of the "generate job spec" flag.
- */
- public boolean isGenerateJobSpec() {
- return generateJobSpec;
- }
-
- /**
- * Specify all out-of-band settings at once. For convenience of older code.
- */
- public void setOOBData(boolean expr_tree, boolean rewritten_expr_tree, boolean logical_plan,
- boolean optimized_logical_plan, boolean hyracks_job) {
- this.set(OOB_EXPR_TREE, expr_tree);
- this.set(OOB_REWRITTEN_EXPR_TREE, rewritten_expr_tree);
- this.set(OOB_LOGICAL_PLAN, logical_plan);
- this.set(OOB_OPTIMIZED_LOGICAL_PLAN, optimized_logical_plan);
- this.set(OOB_HYRACKS_JOB, hyracks_job);
- }
-
- /**
- * Specify a flag.
- *
- * @param flag
- * One of the OOB_ or FORMAT_ constants from this class.
- * @param value
- * Value for the flag (all flags default to "false").
- */
- public void set(String flag, boolean value) {
- flags.put(flag, Boolean.valueOf(value));
- }
-
- /**
- * Retrieve the setting of a format-specific flag.
- *
- * @param flag
- * One of the FORMAT_ constants from this class.
- * @returns true or false (all flags default to "false").
- */
- public boolean is(String flag) {
- Boolean value = flags.get(flag);
- return value == null ? false : value.booleanValue();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
index 204ff5c..b811172 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.api.http.servlet;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
import java.awt.image.BufferedImage;
import java.io.BufferedReader;
import java.io.IOException;
@@ -35,40 +38,38 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.api.common.APIFramework;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.app.SessionConfig.OutputFormat;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.lang.aql.parser.TokenMgrError;
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.client.dataset.HyracksDataset;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
public class APIServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
-
private static final Logger LOGGER = Logger.getLogger(APIServlet.class.getName());
+ public static final String HTML_STATEMENT_SEPARATOR = "<!-- BEGIN -->";
private final ILangCompilationProvider aqlCompilationProvider;
private final ILangCompilationProvider sqlppCompilationProvider;
+ private final transient IStatementExecutorFactory statementExectorFactory;
- public APIServlet() {
- this.aqlCompilationProvider = new AqlCompilationProvider();
- this.sqlppCompilationProvider = new SqlppCompilationProvider();
+ public APIServlet(ILangCompilationProvider aqlCompilationProvider,
+ ILangCompilationProvider sqlppCompilationProvider, IStatementExecutorFactory statementExecutorFactory) {
+ this.aqlCompilationProvider = aqlCompilationProvider;
+ this.sqlppCompilationProvider = sqlppCompilationProvider;
+ this.statementExectorFactory = statementExecutorFactory;
}
@Override
@@ -124,20 +125,21 @@ public class APIServlet extends HttpServlet {
sessionConfig.setOOBData(isSet(printExprParam), isSet(printRewrittenExprParam),
isSet(printLogicalPlanParam), isSet(printOptimizedLogicalPlanParam), isSet(printJob));
MetadataManager.INSTANCE.init();
- QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+ IStatementExecutor translator =
+ statementExectorFactory.create(aqlStatements, sessionConfig, compilationProvider);
double duration = 0;
long startTime = System.currentTimeMillis();
- translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC);
+ translator.compileAndExecute(hcc, hds, IStatementExecutor.ResultDelivery.SYNC);
long endTime = System.currentTimeMillis();
duration = (endTime - startTime) / 1000.00;
- out.println(APIFramework.HTML_STATEMENT_SEPARATOR);
+ out.println(HTML_STATEMENT_SEPARATOR);
out.println("<PRE>Duration of all jobs: " + duration + " sec</PRE>");
} catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
GlobalConfig.ASTERIX_LOGGER.log(Level.INFO, pe.toString(), pe);
- ResultUtils.webUIParseExceptionHandler(out, pe, query);
+ ResultUtil.webUIParseExceptionHandler(out, pe, query);
} catch (Exception e) {
GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
- ResultUtils.webUIErrorHandler(out, e);
+ ResultUtil.webUIErrorHandler(out, e);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
index de19715..bbb9dde 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
@@ -18,21 +18,22 @@
*/
package org.apache.asterix.api.http.servlet;
-import java.util.List;
-
import javax.servlet.http.HttpServletRequest;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
public class AQLAPIServlet extends RESTAPIServlet {
private static final long serialVersionUID = 1L;
private static final String AQL_STMT_PARAM_NAME = "aql";
- private static final List<Byte> ALLOWED_STATEMENTS = Statement.KINDS;
+ private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY | Statement.Category.UPDATE
+ | Statement.Category.DDL | Statement.Category.PROCEDURE;
- public AQLAPIServlet(ILangCompilationProvider compilationProvider) {
- super(compilationProvider);
+ public AQLAPIServlet(ILangCompilationProvider compilationProvider,
+ IStatementExecutorFactory statementExecutorFactory) {
+ super(compilationProvider, statementExecutorFactory);
}
@Override
@@ -41,13 +42,12 @@ public class AQLAPIServlet extends RESTAPIServlet {
}
@Override
- protected List<Byte> getAllowedStatements() {
- return ALLOWED_STATEMENTS;
+ protected byte getAllowedCategories() {
+ return ALLOWED_CATEGORIES;
}
@Override
protected String getErrorMessage() {
throw new IllegalStateException();
}
-
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
index 8f3b14b..eb23902 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
@@ -25,8 +25,8 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.asterix.app.result.ResultUtil;
import org.apache.asterix.om.util.AsterixClusterProperties;
-import org.apache.asterix.result.ResultUtils;
import org.json.JSONException;
import org.json.JSONObject;
@@ -43,7 +43,7 @@ public class ClusterAPIServlet extends HttpServlet {
responseWriter.write(responseObject.toString());
response.setStatus(HttpServletResponse.SC_OK);
} catch (JSONException e) {
- ResultUtils.apiErrorHandler(responseWriter, e);
+ ResultUtil.apiErrorHandler(responseWriter, e);
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
responseWriter.flush();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
index bca6d35..c43377a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
@@ -19,6 +19,8 @@
package org.apache.asterix.api.http.servlet;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;
@@ -43,8 +45,6 @@ import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-
/***
* The REST API that takes a dataverse name and a dataset name as the input
* and returns an array of file splits (IP, file-path) of the dataset in LOSSLESS_JSON.
@@ -94,8 +94,8 @@ public class ConnectorAPIServlet extends HttpServlet {
return;
}
boolean temp = dataset.getDatasetDetails().isTemp();
- FileSplit[] fileSplits = metadataProvider.splitsForDataset(mdTxnCtx, dataverseName, datasetName,
- datasetName, temp);
+ FileSplit[] fileSplits =
+ metadataProvider.splitsForDataset(mdTxnCtx, dataverseName, datasetName, datasetName, temp);
ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(),
dataset.getItemTypeName());
List<List<String>> primaryKeys = DatasetUtils.getPartitioningKeys(dataset);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
index 943691d..19a7609 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
@@ -18,29 +18,20 @@
*/
package org.apache.asterix.api.http.servlet;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
import javax.servlet.http.HttpServletRequest;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
public class DDLAPIServlet extends RESTAPIServlet {
private static final long serialVersionUID = 1L;
+ private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY | Statement.Category.UPDATE
+ | Statement.Category.DDL;
- private static final List<Byte> allowedStatements = Collections.unmodifiableList(Arrays.asList(
- Statement.Kind.DATAVERSE_DECL, Statement.Kind.DATAVERSE_DROP, Statement.Kind.DATASET_DECL,
- Statement.Kind.NODEGROUP_DECL, Statement.Kind.NODEGROUP_DROP, Statement.Kind.TYPE_DECL,
- Statement.Kind.TYPE_DROP, Statement.Kind.CREATE_INDEX, Statement.Kind.INDEX_DECL,
- Statement.Kind.CREATE_DATAVERSE, Statement.Kind.DATASET_DROP, Statement.Kind.INDEX_DROP,
- Statement.Kind.CREATE_FUNCTION, Statement.Kind.FUNCTION_DROP, Statement.Kind.CREATE_PRIMARY_FEED,
- Statement.Kind.CREATE_SECONDARY_FEED, Statement.Kind.DROP_FEED, Statement.Kind.CREATE_FEED_POLICY,
- Statement.Kind.DROP_FEED_POLICY));
-
- public DDLAPIServlet(ILangCompilationProvider compilationProvider) {
- super(compilationProvider);
+ public DDLAPIServlet(ILangCompilationProvider compilationProvider,
+ IStatementExecutorFactory statementExecutorFactory) {
+ super(compilationProvider, statementExecutorFactory);
}
@Override
@@ -49,8 +40,8 @@ public class DDLAPIServlet extends RESTAPIServlet {
}
@Override
- protected List<Byte> getAllowedStatements() {
- return allowedStatements;
+ protected byte getAllowedCategories() {
+ return ALLOWED_CATEGORIES;
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
index 63aa2df..cc70324 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
@@ -18,24 +18,19 @@
*/
package org.apache.asterix.api.http.servlet;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
import javax.servlet.http.HttpServletRequest;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
public class QueryAPIServlet extends RESTAPIServlet {
private static final long serialVersionUID = 1L;
+ private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY;
- private static final List<Byte> allowedStatements = Collections.unmodifiableList(Arrays.asList(
- Statement.Kind.DATAVERSE_DECL, Statement.Kind.FUNCTION_DECL, Statement.Kind.QUERY, Statement.Kind.SET,
- Statement.Kind.WRITE, Statement.Kind.RUN));
-
- public QueryAPIServlet(ILangCompilationProvider compilationProvider) {
- super(compilationProvider);
+ public QueryAPIServlet(ILangCompilationProvider compilationProvider,
+ IStatementExecutorFactory queryTranslatorFactory) {
+ super(compilationProvider, queryTranslatorFactory);
}
@Override
@@ -44,8 +39,8 @@ public class QueryAPIServlet extends RESTAPIServlet {
}
@Override
- protected List<Byte> getAllowedStatements() {
- return allowedStatements;
+ protected byte getAllowedCategories() {
+ return ALLOWED_CATEGORIES;
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
index 9d29adf..fdbf253 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.api.http.servlet;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
import java.io.IOException;
import java.io.PrintWriter;
@@ -26,9 +29,10 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataset.IHyracksDataset;
@@ -38,9 +42,6 @@ import org.apache.hyracks.client.dataset.HyracksDataset;
import org.json.JSONArray;
import org.json.JSONObject;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
public class QueryResultAPIServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
@@ -77,7 +78,7 @@ public class QueryResultAPIServlet extends HttpServlet {
JobId jobId = new JobId(handle.getLong(0));
ResultSetId rsId = new ResultSetId(handle.getLong(1));
- ResultReader resultReader = new ResultReader(hcc, hds);
+ ResultReader resultReader = new ResultReader(hds);
resultReader.open(jobId, rsId);
// QQQ The output format is determined by the initial
@@ -87,7 +88,7 @@ public class QueryResultAPIServlet extends HttpServlet {
// originally determined there. Need to save this value on
// some object that we can obtain here.
SessionConfig sessionConfig = RESTAPIServlet.initResponse(request, response);
- ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats(), null);
+ ResultUtil.displayResults(resultReader, sessionConfig, new Stats(), null);
} catch (Exception e) {
out.println(e.getMessage());
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
index 90e8adf..5b91f2f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.api.http.servlet;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -33,19 +36,20 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.translator.QueryTranslator;
+import org.apache.asterix.common.app.SessionConfig;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.utils.JSONUtil;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.lang.aql.parser.TokenMgrError;
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
+import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
@@ -53,15 +57,18 @@ import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.client.dataset.HyracksDataset;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
public class QueryServiceServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private static final Logger LOGGER = Logger.getLogger(QueryServiceServlet.class.getName());
+ private final transient ILangCompilationProvider compilationProvider;
+ private final transient IStatementExecutorFactory statementExecutorFactory;
- private transient final ILangCompilationProvider compilationProvider;
+ public QueryServiceServlet(ILangCompilationProvider compilationProvider,
+ IStatementExecutorFactory statementExecutorFactory) {
+ this.compilationProvider = compilationProvider;
+ this.statementExecutorFactory = statementExecutorFactory;
+ }
public enum Parameter {
// Standard
@@ -210,10 +217,6 @@ public class QueryServiceServlet extends HttpServlet {
}
}
- public QueryServiceServlet(final ILangCompilationProvider compilationProvider) {
- this.compilationProvider = compilationProvider;
- }
-
private static String getParameterValue(String content, String attribute) {
if (content == null || attribute == null) {
return null;
@@ -396,12 +399,12 @@ public class QueryServiceServlet extends HttpServlet {
response.setContentType(MediaType.JSON.str());
int respCode = HttpServletResponse.SC_OK;
- ResultUtils.Stats stats = new ResultUtils.Stats();
+ Stats stats = new Stats();
long execStart = 0;
long execEnd = -1;
resultWriter.print("{\n");
- UUID requestId = printRequestId(resultWriter);
+ printRequestId(resultWriter);
printSignature(resultWriter);
printType(resultWriter, sessionConfig);
try {
@@ -419,7 +422,8 @@ public class QueryServiceServlet extends HttpServlet {
IParser parser = compilationProvider.getParserFactory().createParser(query);
List<Statement> aqlStatements = parser.parse();
MetadataManager.INSTANCE.init();
- QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+ IStatementExecutor translator =
+ statementExecutorFactory.create(aqlStatements, sessionConfig, compilationProvider);
execStart = System.nanoTime();
translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC, stats);
execEnd = System.nanoTime();
@@ -439,7 +443,8 @@ public class QueryServiceServlet extends HttpServlet {
execEnd = System.nanoTime();
}
}
- printMetrics(resultWriter, System.nanoTime() - elapsedStart, execEnd - execStart, stats.count, stats.size);
+ printMetrics(resultWriter, System.nanoTime() - elapsedStart, execEnd - execStart, stats.getCount(),
+ stats.getSize());
resultWriter.print("}\n");
resultWriter.flush();
String result = stringWriter.toString();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
index 2eb5e81..9555e11 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.api.http.servlet;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
import java.io.IOException;
import java.io.PrintWriter;
@@ -26,18 +29,14 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-import org.apache.asterix.result.ResultReader;
+import org.apache.asterix.app.result.ResultReader;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.client.dataset.HyracksDataset;
-
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+import org.json.JSONArray;
+import org.json.JSONObject;
public class QueryStatusAPIServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
@@ -70,7 +69,7 @@ public class QueryStatusAPIServlet extends HttpServlet {
/* TODO(madhusudancs): We need to find a way to LOSSLESS_JSON serialize default format obtained from
* metadataProvider in the AQLTranslator and store it as part of the result handle.
*/
- ResultReader resultReader = new ResultReader(hcc, hds);
+ ResultReader resultReader = new ResultReader(hds);
resultReader.open(jobId, rsId);
JSONObject jsonResponse = new JSONObject();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
index 81bf21d..100efed 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.api.http.servlet;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
import java.io.IOException;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
@@ -30,9 +33,11 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.app.translator.QueryTranslator;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.app.SessionConfig.OutputFormat;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
@@ -41,26 +46,26 @@ import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.commons.io.IOUtils;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.client.dataset.HyracksDataset;
import org.json.JSONObject;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
abstract class RESTAPIServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private final ILangCompilationProvider compilationProvider;
private final IParserFactory parserFactory;
+ private final transient IStatementExecutorFactory statementExecutorFactory;
- public RESTAPIServlet(ILangCompilationProvider compilationProvider) {
+ public RESTAPIServlet(ILangCompilationProvider compilationProvider,
+ IStatementExecutorFactory statementExecutorFactory) {
this.compilationProvider = compilationProvider;
this.parserFactory = compilationProvider.getParserFactory();
+ this.statementExecutorFactory = statementExecutorFactory;
}
/**
@@ -193,26 +198,27 @@ abstract class RESTAPIServlet extends HttpServlet {
List<Statement> aqlStatements = parser.parse();
if (!containsForbiddenStatements(aqlStatements)) {
MetadataManager.INSTANCE.init();
- QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+ IStatementExecutor translator =
+ statementExecutorFactory.create(aqlStatements, sessionConfig, compilationProvider);
translator.compileAndExecute(hcc, hds, resultDelivery);
}
} catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe);
- String errorMessage = ResultUtils.buildParseExceptionMessage(pe, query);
+ String errorMessage = ResultUtil.buildParseExceptionMessage(pe, query);
JSONObject errorResp =
- ResultUtils.getErrorResponse(2, errorMessage, "", ResultUtils.extractFullStackTrace(pe));
+ ResultUtil.getErrorResponse(2, errorMessage, "", ResultUtil.extractFullStackTrace(pe));
sessionConfig.out().write(errorResp.toString());
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
} catch (Exception e) {
GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
- ResultUtils.apiErrorHandler(sessionConfig.out(), e);
+ ResultUtil.apiErrorHandler(sessionConfig.out(), e);
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
}
private boolean containsForbiddenStatements(List<Statement> aqlStatements) throws AsterixException {
for (Statement st : aqlStatements) {
- if (!getAllowedStatements().contains(st.getKind())) {
+ if ((st.getCategory() & getAllowedCategories()) == 0) {
throw new AsterixException(String.format(getErrorMessage(), st.getKind()));
}
}
@@ -233,7 +239,7 @@ abstract class RESTAPIServlet extends HttpServlet {
protected abstract String getQueryParameter(HttpServletRequest request);
- protected abstract List<Byte> getAllowedStatements();
+ protected abstract byte getAllowedCategories();
protected abstract String getErrorMessage();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
index afa202b..1c62cdb 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
@@ -18,25 +18,19 @@
*/
package org.apache.asterix.api.http.servlet;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
import javax.servlet.http.HttpServletRequest;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
public class UpdateAPIServlet extends RESTAPIServlet {
private static final long serialVersionUID = 1L;
- private static final List<Byte> allowedStatements = Collections.unmodifiableList(Arrays.asList(
- Statement.Kind.DATAVERSE_DECL, Statement.Kind.DELETE, Statement.Kind.INSERT,
- Statement.Kind.UPSERT, Statement.Kind.UPDATE, Statement.Kind.DML_CMD_LIST, Statement.Kind.LOAD,
- Statement.Kind.CONNECT_FEED, Statement.Kind.DISCONNECT_FEED, Statement.Kind.SET,
- Statement.Kind.COMPACT, Statement.Kind.EXTERNAL_DATASET_REFRESH, Statement.Kind.RUN));
+ private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY | Statement.Category.UPDATE;
- public UpdateAPIServlet(ILangCompilationProvider compilationProvider) {
- super(compilationProvider);
+ public UpdateAPIServlet(ILangCompilationProvider compilationProvider,
+ IStatementExecutorFactory statementExecutorFactory) {
+ super(compilationProvider, statementExecutorFactory);
}
@Override
@@ -45,8 +39,8 @@ public class UpdateAPIServlet extends RESTAPIServlet {
}
@Override
- protected List<Byte> getAllowedStatements() {
- return allowedStatements;
+ protected byte getAllowedCategories() {
+ return ALLOWED_CATEGORIES;
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
index 964893b..e358963 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
@@ -24,14 +24,16 @@ import java.util.List;
import org.apache.asterix.api.common.APIFramework;
import org.apache.asterix.api.common.Job;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.translator.QueryTranslator;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.app.SessionConfig.OutputFormat;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutorFactory;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.job.JobSpecification;
@@ -46,20 +48,22 @@ public class AsterixJavaClient {
private final ILangCompilationProvider compilationProvider;
private final IParserFactory parserFactory;
private final APIFramework apiFramework;
+ private final IStatementExecutorFactory statementExecutorFactory;
public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText, PrintWriter writer,
- ILangCompilationProvider compilationProvider) {
+ ILangCompilationProvider compilationProvider, IStatementExecutorFactory statementExecutorFactory) {
this.hcc = hcc;
this.queryText = queryText;
this.writer = writer;
this.compilationProvider = compilationProvider;
+ this.apiFramework = new APIFramework(compilationProvider, null);
+ this.statementExecutorFactory = statementExecutorFactory;
parserFactory = compilationProvider.getParserFactory();
- this.apiFramework = new APIFramework(compilationProvider);
}
public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText,
- ILangCompilationProvider compilationProvider) {
- this(hcc, queryText, new PrintWriter(System.out, true), compilationProvider);
+ ILangCompilationProvider compilationProvider, IStatementExecutorFactory statementExecutorFactory) {
+ this(hcc, queryText, new PrintWriter(System.out, true), compilationProvider, statementExecutorFactory);
}
public void compile() throws Exception {
@@ -68,7 +72,7 @@ public class AsterixJavaClient {
public void compile(boolean optimize, boolean printRewrittenExpressions, boolean printLogicalPlan,
boolean printOptimizedPlan, boolean printPhysicalOpsOnly, boolean generateBinaryRuntime, boolean printJob)
- throws Exception {
+ throws Exception {
queryJobSpec = null;
dmlJobs = null;
@@ -90,7 +94,7 @@ public class AsterixJavaClient {
conf.set(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS, true);
}
- QueryTranslator translator = new QueryTranslator(statements, conf, compilationProvider);
+ IStatementExecutor translator = statementExecutorFactory.create(statements, conf, compilationProvider);
translator.compileAndExecute(hcc, null, QueryTranslator.ResultDelivery.SYNC);
writer.flush();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
new file mode 100644
index 0000000..af10394
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.cc;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.algebra.base.ILangExtension;
+import org.apache.asterix.algebra.base.ILangExtension.Language;
+import org.apache.asterix.algebra.extension.ExtensionFunctionIdentifier;
+import org.apache.asterix.algebra.extension.IAlgebraExtensionManager;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
+import org.apache.asterix.common.api.ExtensionId;
+import org.apache.asterix.common.api.IExtension;
+import org.apache.asterix.common.config.AsterixExtension;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.compiler.provider.AqlCompilationProvider;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
+import org.apache.asterix.translator.IStatementExecutorFactory;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * AsterixDB's implementation of {@code IAlgebraExtensionManager} which takes care of
+ * initializing extensions for App and Compilation purposes
+ */
+public class CompilerExtensionManager implements IAlgebraExtensionManager {
+
+ private static final String ERROR_MESSAGE_ID_CONFLICT = "Two Extensions share the same Id: %1$s";
+ public static final String ERROR_MESSAGE_COMPONENT_CONFLICT =
+ "Extension Conflict between %1$s and %2$s both extensions extend %3$s";
+
+ private final Map<ExtensionId, IExtension> extensions = new HashMap<>();
+
+ private final IStatementExecutorExtension statementExecutorExtension;
+ private final ILangCompilationProvider aqlCompilationProvider;
+ private final ILangCompilationProvider sqlppCompilationProvider;
+ private final DefaultStatementExecutorFactory defaultQueryTranslatorFactory;
+
+ /**
+ * Initialize {@code CompilerExtensionManager} from configuration
+ *
+ * @param list
+ * @throws InstantiationException
+ * @throws IllegalAccessException
+ * @throws ClassNotFoundException
+ * @throws HyracksDataException
+ */
+ public CompilerExtensionManager(List<AsterixExtension> list)
+ throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
+ Pair<ExtensionId, ILangCompilationProvider> aqlcp = null;
+ Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
+ IStatementExecutorExtension see = null;
+ defaultQueryTranslatorFactory = new DefaultStatementExecutorFactory(this);
+
+ if (list != null) {
+ for (AsterixExtension extensionConf : list) {
+ IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
+ extension.configure(extensionConf.getArgs());
+ if (extensions.containsKey(extension.getId())) {
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_EXTENSION_CONFLICT,
+ ERROR_MESSAGE_ID_CONFLICT, extension.getId());
+ }
+ extensions.put(extension.getId(), extension);
+ switch (extension.getExtensionKind()) {
+ case STATEMENT_EXECUTOR:
+ see = extendStatementExecutor(see, (IStatementExecutorExtension) extension);
+ break;
+ case LANG:
+ ILangExtension le = (ILangExtension) extension;
+ aqlcp = extendLangCompilationProvider(Language.AQL, aqlcp, le);
+ sqlppcp = extendLangCompilationProvider(Language.SQLPP, sqlppcp, le);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ this.statementExecutorExtension = see;
+ this.aqlCompilationProvider = aqlcp == null ? new AqlCompilationProvider() : aqlcp.second;
+ this.sqlppCompilationProvider = sqlppcp == null ? new SqlppCompilationProvider() : sqlppcp.second;
+ }
+
+ private Pair<ExtensionId, ILangCompilationProvider> extendLangCompilationProvider(Language lang,
+ Pair<ExtensionId, ILangCompilationProvider> cp, ILangExtension le) throws HyracksDataException {
+ if (cp != null && le.getLangCompilationProvider(lang) != null) {
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_EXTENSION_CONFLICT,
+ ERROR_MESSAGE_COMPONENT_CONFLICT, le.getId(), cp.first, lang.toString());
+ }
+ return (le.getLangCompilationProvider(lang) != null)
+ ? new Pair<>(le.getId(), le.getLangCompilationProvider(lang)) : cp;
+ }
+
+ private IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension qte,
+ IStatementExecutorExtension extension) throws HyracksDataException {
+ if (qte != null) {
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_EXTENSION_CONFLICT,
+ ERROR_MESSAGE_COMPONENT_CONFLICT, qte.getId(), extension.getId(),
+ IStatementExecutorFactory.class.getSimpleName());
+ }
+ return extension;
+ }
+
+ public IStatementExecutorFactory getQueryTranslatorFactory() {
+ return statementExecutorExtension == null ? defaultQueryTranslatorFactory
+ : statementExecutorExtension.getQueryTranslatorFactory();
+ }
+
+ public ILangCompilationProvider getAqlCompilationProvider() {
+ return aqlCompilationProvider;
+ }
+
+ public ILangCompilationProvider getSqlppCompilationProvider() {
+ return sqlppCompilationProvider;
+ }
+
+ // TODO(amoudi/yingyi): this is not a good way to extend rewrite rules. introduce re-write rule provider
+ @Override
+ public boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
+ UnnestOperator unnestOp, ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr)
+ throws AlgebricksException {
+ FunctionIdentifier functionId = functionCallExpr.getFunctionIdentifier();
+ if (functionId instanceof ExtensionFunctionIdentifier) {
+ ILangExtension extension =
+ (ILangExtension) extensions.get(((ExtensionFunctionIdentifier) functionId).getExtensionId());
+ return extension.unnestToDataScan(opRef, context, unnestOp, unnestExpr, functionCallExpr);
+ }
+ return false;
+ }
+}
[07/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDecl.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDecl.java
index 89933e5..5067f9f 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDecl.java
@@ -44,4 +44,9 @@ public class DataverseDecl implements Statement {
public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
return visitor.visit(this, arg);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.QUERY;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDropStatement.java
index 0054140..64ba087 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDropStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DataverseDropStatement.java
@@ -51,4 +51,9 @@ public class DataverseDropStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DeleteStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DeleteStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DeleteStatement.java
index ffe2c0d..d779b4f 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DeleteStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DeleteStatement.java
@@ -117,4 +117,9 @@ public class DeleteStatement implements Statement {
return equals && ObjectUtils.equals(dataverses, target.dataverses)
&& ObjectUtils.equals(rewrittenQuery, target.rewrittenQuery) && ObjectUtils.equals(vars, target.vars);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DisconnectFeedStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DisconnectFeedStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DisconnectFeedStatement.java
index 9d8cb9d..af4d1fb 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DisconnectFeedStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DisconnectFeedStatement.java
@@ -75,4 +75,9 @@ public class DisconnectFeedStatement implements Statement {
return "disconnect feed " + feedName + " from " + datasetName;
}
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropDatasetStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropDatasetStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropDatasetStatement.java
new file mode 100644
index 0000000..7dc1ad8
--- /dev/null
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropDatasetStatement.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.statement;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class DropDatasetStatement implements Statement {
+
+ private final Identifier dataverseName;
+ private final Identifier datasetName;
+ private boolean ifExists;
+
+ public DropDatasetStatement(Identifier dataverseName, Identifier datasetName, boolean ifExists) {
+ this.dataverseName = dataverseName;
+ this.datasetName = datasetName;
+ this.ifExists = ifExists;
+ }
+
+ @Override
+ public byte getKind() {
+ return Statement.Kind.DATASET_DROP;
+ }
+
+ public Identifier getDataverseName() {
+ return dataverseName;
+ }
+
+ public Identifier getDatasetName() {
+ return datasetName;
+ }
+
+ public boolean getIfExists() {
+ return ifExists;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropStatement.java
deleted file mode 100644
index f4189dd..0000000
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DropStatement.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.lang.common.statement;
-
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.struct.Identifier;
-import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
-
-public class DropStatement implements Statement {
-
- private final Identifier dataverseName;
- private final Identifier datasetName;
- private boolean ifExists;
-
- public DropStatement(Identifier dataverseName, Identifier datasetName, boolean ifExists) {
- this.dataverseName = dataverseName;
- this.datasetName = datasetName;
- this.ifExists = ifExists;
- }
-
- @Override
- public byte getKind() {
- return Statement.Kind.DATASET_DROP;
- }
-
- public Identifier getDataverseName() {
- return dataverseName;
- }
-
- public Identifier getDatasetName() {
- return datasetName;
- }
-
- public boolean getIfExists() {
- return ifExists;
- }
-
- @Override
- public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
- return visitor.visit(this, arg);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDropStatement.java
index 878fc49..0391a4b 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDropStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDropStatement.java
@@ -56,4 +56,9 @@ public class FeedDropStatement implements Statement {
public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
return visitor.visit(this, arg);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedPolicyDropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedPolicyDropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedPolicyDropStatement.java
index c217690..d08dc31 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedPolicyDropStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedPolicyDropStatement.java
@@ -57,4 +57,9 @@ public class FeedPolicyDropStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java
index c71f7f2..08c90c5 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDecl.java
@@ -81,4 +81,9 @@ public class FunctionDecl implements Statement {
public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
return visitor.visit(this, arg);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.QUERY;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDropStatement.java
index ae1a29b..00ae48c 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDropStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FunctionDropStatement.java
@@ -51,4 +51,9 @@ public class FunctionDropStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IndexDropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IndexDropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IndexDropStatement.java
index 1e3f6b0..1b8d0d5 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IndexDropStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/IndexDropStatement.java
@@ -63,4 +63,9 @@ public class IndexDropStatement implements Statement {
public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
return visitor.visit(this, arg);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InsertStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InsertStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InsertStatement.java
index 1e69a68..b2dc72e 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InsertStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InsertStatement.java
@@ -82,4 +82,9 @@ public class InsertStatement implements Statement {
&& ObjectUtils.equals(dataverseName, target.dataverseName) && ObjectUtils.equals(query, target.query);
}
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/LoadStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/LoadStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/LoadStatement.java
index 849b0bd..145514f 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/LoadStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/LoadStatement.java
@@ -84,4 +84,9 @@ public class LoadStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodeGroupDropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodeGroupDropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodeGroupDropStatement.java
index 710bedd..e27b447 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodeGroupDropStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodeGroupDropStatement.java
@@ -51,4 +51,9 @@ public class NodeGroupDropStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodegroupDecl.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodegroupDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodegroupDecl.java
index 4232584..092e804 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodegroupDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/NodegroupDecl.java
@@ -62,4 +62,9 @@ public class NodegroupDecl implements Statement {
public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
return visitor.visit(this, arg);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/Query.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/Query.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/Query.java
index 9be4830..fd6a088 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/Query.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/Query.java
@@ -121,4 +121,9 @@ public class Query implements Statement {
&& ObjectUtils.equals(datasets, target.datasets) && ObjectUtils.equals(dataverses, target.dataverses)
&& topLevel == target.topLevel;
}
+
+ @Override
+ public byte getCategory() {
+ return Category.QUERY;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RefreshExternalDatasetStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RefreshExternalDatasetStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RefreshExternalDatasetStatement.java
index 277b4bf..ab35b27 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RefreshExternalDatasetStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RefreshExternalDatasetStatement.java
@@ -54,4 +54,9 @@ public class RefreshExternalDatasetStatement implements Statement {
return Statement.Kind.EXTERNAL_DATASET_REFRESH;
}
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RunStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RunStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RunStatement.java
index 6b33078..006799d 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RunStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/RunStatement.java
@@ -78,4 +78,9 @@ public class RunStatement implements Statement {
return Statement.Kind.RUN;
}
+ @Override
+ public byte getCategory() {
+ return Category.QUERY;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/SetStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/SetStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/SetStatement.java
index 54ba599..dfa8406 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/SetStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/SetStatement.java
@@ -50,4 +50,9 @@ public class SetStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.QUERY;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDecl.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDecl.java
index e25688e..a9eb795 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDecl.java
@@ -76,4 +76,9 @@ public class TypeDecl implements Statement {
return datagenAnnotation;
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDropStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDropStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDropStatement.java
index d675054..99b9bb1 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDropStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/TypeDropStatement.java
@@ -57,4 +57,9 @@ public class TypeDropStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/UpdateStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/UpdateStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/UpdateStatement.java
index 8ccc694..09ed242 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/UpdateStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/UpdateStatement.java
@@ -86,4 +86,9 @@ public class UpdateStatement implements Statement {
&& ObjectUtils.equals(ucs, update.ucs) && ObjectUtils.equals(vars, update.vars);
}
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java
index a755a54..311e650 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/WriteStatement.java
@@ -57,4 +57,9 @@ public class WriteStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.QUERY;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java
new file mode 100644
index 0000000..94144c6
--- /dev/null
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/LangRecordParseUtil.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.util;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.asterix.builders.OrderedListBuilder;
+import org.apache.asterix.builders.RecordBuilder;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Literal;
+import org.apache.asterix.lang.common.expression.FieldBinding;
+import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.ADouble;
+import org.apache.asterix.om.base.AInt64;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.asterix.om.base.ANull;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+
+@SuppressWarnings("unchecked")
+public class LangRecordParseUtil {
+ private static final String NOT_ALLOWED_EXPRESSIONS_ERROR_MESSAGE =
+ "JSON record can only have expressions [%1$s, %2$s, %3$s]";
+ private static final ISerializerDeserializer<ADouble> doubleSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ADOUBLE);
+ private static final ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ASTRING);
+ private static final ISerializerDeserializer<AInt64> intSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.AINT64);
+ private static final ISerializerDeserializer<ABoolean> booleanSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ABOOLEAN);
+ private static final ISerializerDeserializer<ANull> nullSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ANULL);
+
+ private LangRecordParseUtil() {
+ }
+
+ private static void parseExpression(Expression expr, ArrayBackedValueStorage serialized)
+ throws HyracksDataException {
+ switch (expr.getKind()) {
+ case LITERAL_EXPRESSION:
+ parseLiteral((LiteralExpr) expr, serialized);
+ break;
+ case RECORD_CONSTRUCTOR_EXPRESSION:
+ parseRecord((RecordConstructor) expr, serialized);
+ break;
+ case LIST_CONSTRUCTOR_EXPRESSION:
+ parseList((ListConstructor) expr, serialized);
+ break;
+ default:
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_PARSE_ERROR,
+ NOT_ALLOWED_EXPRESSIONS_ERROR_MESSAGE,
+ new Serializable[] { Expression.Kind.LITERAL_EXPRESSION.toString(),
+ Expression.Kind.RECORD_CONSTRUCTOR_EXPRESSION.toString(),
+ Expression.Kind.LIST_CONSTRUCTOR_EXPRESSION.toString() });
+ }
+ }
+
+ public static void parseRecord(RecordConstructor recordValue, ArrayBackedValueStorage serialized)
+ throws HyracksDataException {
+ AMutableString fieldNameString = new AMutableString(null);
+ ArrayBackedValueStorage fieldName = new ArrayBackedValueStorage();
+ ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+ RecordBuilder recordBuilder = new RecordBuilder();
+ recordBuilder.reset(ARecordType.FULLY_OPEN_RECORD_TYPE);
+ recordBuilder.init();
+ List<FieldBinding> fbList = recordValue.getFbList();
+ for (FieldBinding fb : fbList) {
+ fieldName.reset();
+ fieldValue.reset();
+ // get key
+ Expression keyExpr = fb.getLeftExpr();
+ if (keyExpr.getKind() != Expression.Kind.LITERAL_EXPRESSION) {
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_PARSE_ERROR,
+ "JSON key can only be of type %1$s", Expression.Kind.LITERAL_EXPRESSION);
+ }
+ LiteralExpr keyLiteralExpr = (LiteralExpr) keyExpr;
+ Literal keyLiteral = keyLiteralExpr.getValue();
+ if (keyLiteral.getLiteralType() != Literal.Type.STRING) {
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_PARSE_ERROR,
+ "JSON key can only be of type %1$s", Literal.Type.STRING);
+ }
+ fieldNameString.setValue(keyLiteral.getStringValue());
+ stringSerde.serialize(fieldNameString, fieldName.getDataOutput());
+ // get value
+ parseExpression(fb.getRightExpr(), fieldValue);
+ recordBuilder.addField(fieldName, fieldValue);
+ }
+ recordBuilder.write(serialized.getDataOutput(), true);
+ }
+
+ private static void parseList(ListConstructor valueExpr, ArrayBackedValueStorage serialized)
+ throws HyracksDataException {
+ if (valueExpr.getType() != ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR) {
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_PARSE_ERROR,
+ "JSON List can't be of type %1$s", valueExpr.getType());
+ }
+ ArrayBackedValueStorage serializedValue = new ArrayBackedValueStorage();
+ OrderedListBuilder listBuilder = new OrderedListBuilder();
+ listBuilder.reset(null);
+ for (Expression expr : valueExpr.getExprList()) {
+ serializedValue.reset();
+ parseExpression(expr, serializedValue);
+ listBuilder.addItem(serializedValue);
+ }
+ listBuilder.write(serialized.getDataOutput(), true);
+ }
+
+ private static void parseLiteral(LiteralExpr objectExpr, ArrayBackedValueStorage serialized)
+ throws HyracksDataException {
+ Literal value = objectExpr.getValue();
+ switch (value.getLiteralType()) {
+ case DOUBLE:
+ doubleSerde.serialize(new ADouble((Double) value.getValue()), serialized.getDataOutput());
+ break;
+ case TRUE:
+ booleanSerde.serialize(ABoolean.TRUE, serialized.getDataOutput());
+ break;
+ case FALSE:
+ booleanSerde.serialize(ABoolean.FALSE, serialized.getDataOutput());
+ break;
+ case FLOAT:
+ doubleSerde.serialize(new ADouble((Float) value.getValue()), serialized.getDataOutput());
+ break;
+ case INTEGER:
+ intSerde.serialize(new AInt64(((Integer) value.getValue()).longValue()), serialized.getDataOutput());
+ break;
+ case LONG:
+ intSerde.serialize(new AInt64((Long) value.getValue()), serialized.getDataOutput());
+ break;
+ case NULL:
+ nullSerde.serialize(ANull.NULL, serialized.getDataOutput());
+ break;
+ case STRING:
+ stringSerde.serialize(new AString((String) value.getValue()), serialized.getDataOutput());
+ break;
+ default:
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_PARSE_ERROR,
+ "Unknown Literal Type %1$s", value.getLiteralType());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 5feb61b..5ff4a03 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -30,7 +30,6 @@ import java.util.Set;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
@@ -74,7 +73,7 @@ import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
@@ -97,6 +96,7 @@ import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.UnaryExprType;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
@@ -566,7 +566,7 @@ public class FormatPrintVisitor implements ILangVisitor<Void, Integer> {
}
@Override
- public Void visit(DropStatement del, Integer step) throws AsterixException {
+ public Void visit(DropDatasetStatement del, Integer step) throws AsterixException {
out.println(
skip(step) + "drop " + datasetSymbol + generateFullName(del.getDataverseName(), del.getDatasetName())
+ generateIfExists(del.getIfExists()) + SEMICOLON);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
index 1e8dfd8..e32615f 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
@@ -37,7 +37,7 @@ import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
import org.apache.asterix.lang.common.statement.FunctionDropStatement;
@@ -70,7 +70,7 @@ public abstract class AbstractQueryExpressionVisitor<R, T> implements ILangVisit
}
@Override
- public R visit(DropStatement del, T arg) throws AsterixException {
+ public R visit(DropDatasetStatement del, T arg) throws AsterixException {
return null;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
index 1545f3e..4a16025 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
@@ -53,7 +53,7 @@ import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
@@ -84,7 +84,7 @@ public interface ILangVisitor<R, T> {
R visit(LoadStatement stmtLoad, T arg) throws AsterixException;
- R visit(DropStatement del, T arg) throws AsterixException;
+ R visit(DropDatasetStatement del, T arg) throws AsterixException;
R visit(InsertStatement insert, T arg) throws AsterixException;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java
index a3346ed..fbf08ee 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppBuiltinFunctionRewriteVisitor.java
@@ -21,7 +21,6 @@ package org.apache.asterix.lang.sqlpp.rewrites.visitor;
import java.util.ArrayList;
import java.util.List;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
@@ -34,6 +33,7 @@ import org.apache.asterix.lang.sqlpp.expression.CaseExpression;
import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppSimpleExpressionVisitor;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
public class SqlppBuiltinFunctionRewriteVisitor extends AbstractSqlppSimpleExpressionVisitor {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
index 9769d4f..e6c95b7 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/VariableCheckAndRewriteVisitor.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
@@ -39,6 +38,7 @@ import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppExpressionScopingVisitor;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
public class VariableCheckAndRewriteVisitor extends AbstractSqlppExpressionScopingVisitor {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
index 9c3e1fb..6ae55dd 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
@@ -59,6 +58,7 @@ import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.core.algebra.base.Counter;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 95f09f5..2378060 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -45,7 +45,6 @@ import org.apache.asterix.common.annotations.TypeDataGen;
import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.Expression;
@@ -102,7 +101,7 @@ import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
@@ -145,6 +144,7 @@ import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
import org.apache.asterix.lang.sqlpp.util.ExpressionToVariableUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
@@ -314,7 +314,7 @@ Statement SingleStatement() throws ParseException:
| stmt = InsertStatement()
| stmt = DeleteStatement()
| stmt = UpdateStatement()
- | stmt = FeedStatement()
+ | stmt = ConnectionStatement()
| stmt = CompactStatement()
| stmt = ExplainStatement()
| stmt = Query(false) <SEMICOLON>
@@ -870,7 +870,7 @@ Statement DropStatement() throws ParseException:
(
Dataset() pairId = QualifiedName() ifExists = IfExists()
{
- stmt = new DropStatement(pairId.first, pairId.second, ifExists);
+ stmt = new DropDatasetStatement(pairId.first, pairId.second, ifExists);
}
| <INDEX> tripleId = DoubleQualifiedName() ifExists = IfExists()
{
@@ -1097,7 +1097,7 @@ Statement CompactStatement() throws ParseException:
}
}
-Statement FeedStatement() throws ParseException:
+Statement ConnectionStatement() throws ParseException:
{
Pair<Identifier,Identifier> feedNameComponents = null;
Pair<Identifier,Identifier> datasetNameComponents = null;
@@ -1108,18 +1108,54 @@ Statement FeedStatement() throws ParseException:
}
{
(
- <CONNECT> <FEED> feedNameComponents = QualifiedName() <TO> Dataset() datasetNameComponents = QualifiedName() (policy = GetPolicy())?
+ <CONNECT> stmt = ConnectStatement()
+ | <DISCONNECT> stmt = DisconnectStatement()
+ )
+ {
+ return stmt;
+ }
+}
+
+Statement DisconnectStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> feedNameComponents = null;
+ Pair<Identifier,Identifier> datasetNameComponents = null;
+
+ Map<String,String> configuration = null;
+ Statement stmt = null;
+ String policy = null;
+}
+{
+ (
+ <FEED> feedNameComponents = QualifiedName() <FROM> Dataset() datasetNameComponents = QualifiedName()
{
- stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, policy, getVarCounter());
+ stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
}
- | <DISCONNECT> <FEED> feedNameComponents = QualifiedName() <FROM> Dataset() datasetNameComponents = QualifiedName()
+ )
+ {
+ return stmt;
+ }
+}
+
+Statement ConnectStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> feedNameComponents = null;
+ Pair<Identifier,Identifier> datasetNameComponents = null;
+
+ Map<String,String> configuration = null;
+ Statement stmt = null;
+ String policy = null;
+}
+{
+ (
+ <FEED> feedNameComponents = QualifiedName() <TO> Dataset() datasetNameComponents = QualifiedName() (policy = GetPolicy())?
{
- stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
+ stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, policy, getVarCounter());
}
)
- {
- return stmt;
- }
+ {
+ return stmt;
+ }
}
Map<String,String> Configuration() throws ParseException :
@@ -2289,13 +2325,15 @@ Expression CaseExpr() throws ParseException:
}
}
-SelectExpression SelectExpression(boolean subquery) throws ParseException: {
+SelectExpression SelectExpression(boolean subquery) throws ParseException:
+{
List<LetClause> letClauses = new ArrayList<LetClause>();
SelectSetOperation selectSetOperation;
OrderbyClause orderbyClause = null;
LimitClause limitClause = null;
createNewScope();
-} {
+}
+{
( letClauses = LetClause() )?
selectSetOperation = SelectSetOperation()
(orderbyClause = OrderbyClause() {})?
@@ -2305,7 +2343,8 @@ SelectExpression SelectExpression(boolean subquery) throws ParseException: {
}
}
-SelectSetOperation SelectSetOperation() throws ParseException: {
+SelectSetOperation SelectSetOperation() throws ParseException:
+{
SetOperationInput setOperationInputLeft;
List<SetOperationRight> setOperationRights = new ArrayList<SetOperationRight>();
}
@@ -2337,7 +2376,8 @@ SelectSetOperation SelectSetOperation() throws ParseException: {
}
}
-SelectExpression Subquery() throws ParseException: {
+SelectExpression Subquery() throws ParseException:
+{
SelectExpression selectExpr = null;
}
{
@@ -2347,7 +2387,8 @@ SelectExpression Subquery() throws ParseException: {
}
}
-SelectBlock SelectBlock() throws ParseException: {
+SelectBlock SelectBlock() throws ParseException:
+{
SelectClause selectClause = null;
FromClause fromClause = null;
List<LetClause> fromLetClauses = null;
@@ -2397,7 +2438,8 @@ SelectBlock SelectBlock() throws ParseException: {
}
}
-SelectClause SelectClause() throws ParseException: {
+SelectClause SelectClause() throws ParseException:
+{
SelectRegular selectRegular = null;
SelectElement selectElement = null;
boolean distinct = false;
@@ -2420,7 +2462,8 @@ SelectClause SelectClause() throws ParseException: {
}
}
-SelectRegular SelectRegular() throws ParseException: {
+SelectRegular SelectRegular() throws ParseException:
+{
List<Projection> projections = new ArrayList<Projection>();
}
{
@@ -2436,7 +2479,8 @@ SelectRegular SelectRegular() throws ParseException: {
}
}
-SelectElement SelectElement() throws ParseException: {
+SelectElement SelectElement() throws ParseException:
+{
Expression expr = null;
String name = null;
}
@@ -2447,7 +2491,8 @@ SelectElement SelectElement() throws ParseException: {
}
}
-Projection Projection() throws ParseException: {
+Projection Projection() throws ParseException :
+{
Expression expr = null;
Identifier identifier = null;
String name = null;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
index d3e4a41..f66c449 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/pom.xml
@@ -1,102 +1,112 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-maven-plugins</artifactId>
- <version>0.8.9-SNAPSHOT</version>
- </parent>
- <artifactId>asterix-grammar-extension-maven-plugin</artifactId>
- <packaging>maven-plugin</packaging>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-plugin-api</artifactId>
- <version>3.2.5</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-core</artifactId>
- <version>3.2.5</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven</groupId>
- <artifactId>maven-compat</artifactId>
- <version>3.2.5</version>
- </dependency>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-maven-plugins</artifactId>
+ <version>0.8.9-SNAPSHOT</version>
+ </parent>
+ <artifactId>asterix-grammar-extension-maven-plugin</artifactId>
+ <packaging>maven-plugin</packaging>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-plugin-api</artifactId>
+ <version>3.2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-core</artifactId>
+ <version>3.2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-compat</artifactId>
+ <version>3.2.5</version>
+ </dependency>
<!-- dependencies to annotations -->
- <dependency>
- <groupId>org.apache.maven.plugin-tools</groupId>
- <artifactId>maven-plugin-annotations</artifactId>
- <version>3.4</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-external-data</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.maven.plugin-testing</groupId>
- <artifactId>maven-plugin-testing-harness</artifactId>
- <version>3.3.0</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-lang-common</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>javacc-maven-plugin</artifactId>
- <version>2.6</version>
- <executions>
- <execution>
- <id>javacc</id>
- <goals>
- <goal>javacc</goal>
- </goals>
- <phase>verify</phase>
- <configuration>
- <isStatic>false</isStatic>
- <javaUnicodeEscape>true</javaUnicodeEscape>
- <sourceDirectory>${project.build.directory}/generated-sources/lang</sourceDirectory>
- <outputDirectory>${project.build.directory}/generated-sources/javacc</outputDirectory>
- </configuration>
- </execution>
- <execution>
- <id>javacc-jjdoc</id>
- <goals>
- <goal>jjdoc</goal>
- </goals>
- <phase>verify</phase>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-plugin-plugin</artifactId>
- <version>3.3</version>
- <configuration>
- <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
- </configuration>
- <executions>
- <execution>
- <id>mojo-descriptor</id>
- <phase>process-classes</phase>
- <goals>
- <goal>descriptor</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
+ <dependency>
+ <groupId>org.apache.maven.plugin-tools</groupId>
+ <artifactId>maven-plugin-annotations</artifactId>
+ <version>3.4</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-project</artifactId>
+ <version>2.0.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-external-data</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.plugin-testing</groupId>
+ <artifactId>maven-plugin-testing-harness</artifactId>
+ <version>3.3.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-lang-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-lang-sqlpp</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>javacc-maven-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>javacc</id>
+ <goals>
+ <goal>javacc</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <isStatic>false</isStatic>
+ <javaUnicodeEscape>true</javaUnicodeEscape>
+ <sourceDirectory>${project.build.directory}/generated-sources/lang</sourceDirectory>
+ <outputDirectory>${project.build.directory}/generated-sources/javacc</outputDirectory>
+ </configuration>
+ </execution>
+ <execution>
+ <id>javacc-jjdoc</id>
+ <goals>
+ <goal>jjdoc</goal>
+ </goals>
+ <phase>verify</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-plugin-plugin</artifactId>
+ <version>3.3</version>
+ <configuration>
+ <skipErrorNoDescriptorsFound>true</skipErrorNoDescriptorsFound>
+ </configuration>
+ <executions>
+ <execution>
+ <id>mojo-descriptor</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>descriptor</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
index 8f263f7..5b44e23 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/main/java/org/apache/asterix/extension/grammar/GrammarExtensionMojo.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -60,20 +61,23 @@ public class GrammarExtensionMojo extends AbstractMojo {
private static final char OPEN_PAREN = '(';
private static final char CLOSE_PAREN = ')';
private static final char SEMICOLON = ';';
- private static final List<Character> SIG_SPECIAL_CHARS =
- Arrays.asList(new Character[] { '(', ')', ':', '<', '>', ';', '.' });
+ private static final List<Character> SIG_SPECIAL_CHARS = Arrays
+ .asList(new Character[] { '(', ')', ':', '<', '>', ';', '.' });
private static final String KWCLASS = "class";
private static final String KWIMPORT = "import";
+ private static final String KWUNIMPORT = "unimport";
private static final String KWPACKAGE = "package";
private static final String NEWPRODUCTION = "@new";
private static final String MERGEPRODUCTION = "@merge";
private static final String OVERRIDEPRODUCTION = "@override";
private static final String BEFORE = "before:";
private static final String AFTER = "after:";
- private static final List<String> KEYWORDS =
- Arrays.asList(new String[] { KWCLASS, KWIMPORT, KWPACKAGE, PARSER_BEGIN, PARSER_END });
- private static final List<String> EXTENSIONKEYWORDS =
- Arrays.asList(new String[] { KWIMPORT, NEWPRODUCTION, OVERRIDEPRODUCTION, MERGEPRODUCTION });
+ private static final String REPLACE = "replace";
+ private static final String WITH = "with";
+ private static final List<String> KEYWORDS = Arrays
+ .asList(new String[] { KWCLASS, KWIMPORT, KWPACKAGE, PARSER_BEGIN, PARSER_END });
+ private static final List<String> EXTENSIONKEYWORDS = Arrays
+ .asList(new String[] { KWIMPORT, KWUNIMPORT, NEWPRODUCTION, OVERRIDEPRODUCTION, MERGEPRODUCTION });
private static final String REGEX_WS_DOT_SEMICOLON = "\\s|[.]|[;]";
private static final String REGEX_WS_PAREN = "\\s|[(]|[)]";
private static final String OPTIONS = "options";
@@ -81,18 +85,25 @@ public class GrammarExtensionMojo extends AbstractMojo {
private Position position = new Position();
private Map<String, Pair<String, String>> extensibles = new HashMap<>();
private Map<String, String[]> mergeElements = new HashMap<>();
- private List<Pair<String, String>> finals = new ArrayList<>();
+ private List<Pair<String, String>> baseFinals = new ArrayList<>();
+ private List<Pair<String, String>> extensionFinals = new ArrayList<>();
private List<List<String>> imports = new ArrayList<>();
private String baseClassName;
private String baseClassDef;
private String optionsBlock;
private boolean read = false;
+ private boolean shouldReplace = false;
+ private String oldWord = null;
+ private String newWord = null;
@Parameter(property = "grammarix.base")
private String base;
- @Parameter(property = "grammarix.extension")
- private String extension;
+ @Parameter(property = "grammarix.gbase")
+ private String gbase;
+
+ @Parameter(property = "grammarix.gextension")
+ private String gextension;
@Parameter(property = "grammarix.output")
private String output;
@@ -106,9 +117,10 @@ public class GrammarExtensionMojo extends AbstractMojo {
@Override
public void execute() throws MojoExecutionException {
- getLog().info("Current dir: " + System.getProperty("user.dir"));
- getLog().info("base: " + base);
- getLog().info("extension: " + extension);
+ base = new File(base).getAbsolutePath();
+ getLog().info("Base dir: " + base);
+ getLog().info("Grammar-base: " + gbase);
+ getLog().info("Grammar-extension: " + gextension);
processBase();
processExtension();
generateOutput();
@@ -187,7 +199,14 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
}
- for (Pair<String, String> element : finals) {
+ for (Pair<String, String> element : extensionFinals) {
+ writer.write(toOutput(element.first));
+ writer.newLine();
+ writer.write(element.second);
+ writer.newLine();
+ }
+
+ for (Pair<String, String> element : baseFinals) {
writer.write(toOutput(element.first));
writer.newLine();
writer.write(element.second);
@@ -247,8 +266,6 @@ public class GrammarExtensionMojo extends AbstractMojo {
// second block
writer.write(OPEN_BRACE);
writer.newLine();
- writer.write(" ");
- writer.write(OPEN_PAREN);
if (extensions[2] != null) {
writer.write(extensions[2]);
}
@@ -258,21 +275,33 @@ public class GrammarExtensionMojo extends AbstractMojo {
Position blockPosition = new Position();
blockPosition.index = 0;
blockPosition.line = blockReader.readLine();
- while (blockPosition.line != null && blockPosition.line.trim().length() == 0) {
+ while (blockPosition.line != null
+ && (blockPosition.line.trim().length() == 0 || blockPosition.line.indexOf(OPEN_BRACE) < 0)) {
+ if (blockPosition.line.trim().length() > 0) {
+ writer.write(blockPosition.line);
+ writer.newLine();
+ }
blockPosition.line = blockReader.readLine();
}
- int block2Open = blockPosition.line.indexOf(OPEN_BRACE);
- if (block2Open < 0) {
+ if (blockPosition.line == null) {
throw new MojoExecutionException(errorMessage);
}
+ int block2Open = blockPosition.line.indexOf(OPEN_BRACE);
blockPosition.line = blockPosition.line.substring(block2Open + 1);
- while (blockPosition.line != null && blockPosition.line.trim().length() == 0) {
+ while (blockPosition.line != null
+ && (blockPosition.line.trim().length() == 0 || blockPosition.line.indexOf(OPEN_PAREN) < 0)) {
+ if (blockPosition.line.trim().length() > 0) {
+ writer.write(blockPosition.line);
+ writer.newLine();
+ }
blockPosition.line = blockReader.readLine();
}
- int innerBlock1Open = blockPosition.line.indexOf(OPEN_PAREN);
- if (innerBlock1Open < 0) {
+ if (blockPosition.line == null) {
throw new MojoExecutionException(errorMessage);
}
+ int innerBlock1Open = blockPosition.line.indexOf(OPEN_PAREN);
+ writer.write(" ");
+ writer.write(OPEN_PAREN);
blockPosition.index = innerBlock1Open;
readBlock(blockReader, OPEN_PAREN, CLOSE_PAREN, blockPosition);
String innerBlock1String = record.toString();
@@ -397,7 +426,7 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
private void processBase() throws MojoExecutionException {
- try (BufferedReader reader = Files.newBufferedReader(Paths.get(base), StandardCharsets.UTF_8)) {
+ try (BufferedReader reader = Files.newBufferedReader(Paths.get(base, gbase), StandardCharsets.UTF_8)) {
StringBuilder identifier = new StringBuilder();
while ((position.line = reader.readLine()) != null) {
if (position.line.trim().startsWith("//")) {
@@ -424,7 +453,7 @@ public class GrammarExtensionMojo extends AbstractMojo {
} else if (openAngularIndex == 0) {
position.index = position.line.indexOf(OPEN_ANGULAR);
readFinalProduction(identifier, reader);
- addFinalProduction(identifier);
+ addFinalProduction(identifier, baseFinals);
} else if (identifier.length() > 0 || position.line.trim().length() > 0) {
identifier.append(position.line);
identifier.append('\n');
@@ -463,7 +492,7 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
}
- private void addFinalProduction(StringBuilder identifier) {
+ private void addFinalProduction(StringBuilder identifier, List<Pair<String, String>> finals) {
String sig = toSignature(identifier.toString());
finals.add(new Pair<String, String>(sig, record.toString()));
record.reset();
@@ -484,6 +513,34 @@ public class GrammarExtensionMojo extends AbstractMojo {
imports.add(importList);
}
+ private void handleUnImport(BufferedReader reader) throws IOException {
+ ArrayList<String> importList = new ArrayList<>();
+ String[] tokens = position.line.split(REGEX_WS_DOT_SEMICOLON);
+ importList.addAll(Arrays.asList(tokens));
+ while (position.line.indexOf(SEMICOLON) < 0) {
+ position.line = reader.readLine();
+ tokens = position.line.split(REGEX_WS_DOT_SEMICOLON);
+ importList.addAll(Arrays.asList(tokens));
+ }
+ // remove from imports
+ Iterator<List<String>> it = imports.iterator();
+ while (it.hasNext()) {
+ List<String> anImport = it.next();
+ if (anImport.size() == importList.size()) {
+ boolean equals = true;
+ for (int i = 1; i < anImport.size(); i++) {
+ if (!anImport.get(i).equals(importList.get(i))) {
+ equals = false;
+ break;
+ }
+ }
+ if (equals) {
+ it.remove();
+ }
+ }
+ }
+ }
+
private String importToString(List<String> importTokens) {
return "import " + StringUtils.join(importTokens.subList(1, importTokens.size()), '.') + ";";
}
@@ -590,7 +647,7 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
private void processExtension() throws MojoExecutionException {
- try (BufferedReader reader = Files.newBufferedReader(Paths.get(extension), StandardCharsets.UTF_8)) {
+ try (BufferedReader reader = Files.newBufferedReader(Paths.get(base, gextension), StandardCharsets.UTF_8)) {
StringBuilder identifier = new StringBuilder();
String nextOperation = OVERRIDEPRODUCTION;
while (read || (position.line = reader.readLine()) != null) {
@@ -607,13 +664,16 @@ public class GrammarExtensionMojo extends AbstractMojo {
switch (tokens[0]) {
case KWIMPORT:
handleImport(reader);
- // import statement
+ break;
+ case KWUNIMPORT:
+ handleUnImport(reader);
break;
case NEWPRODUCTION:
nextOperation = NEWPRODUCTION;
break;
case MERGEPRODUCTION:
nextOperation = MERGEPRODUCTION;
+ shouldReplace = shouldReplace(tokens);
break;
case OVERRIDEPRODUCTION:
nextOperation = OVERRIDEPRODUCTION;
@@ -649,7 +709,7 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
position.index = position.line.indexOf(OPEN_ANGULAR);
readFinalProduction(identifier, reader);
- addFinalProduction(identifier);
+ addFinalProduction(identifier, extensionFinals);
} else if (identifier.length() > 0 || position.line.trim().length() > 0) {
identifier.append(position.line);
identifier.append('\n');
@@ -661,6 +721,20 @@ public class GrammarExtensionMojo extends AbstractMojo {
}
}
+ private boolean shouldReplace(String[] tokens) throws MojoExecutionException {
+ boolean replace = false;
+ if (tokens.length == 5) {
+ if (tokens[1].equals(REPLACE) && tokens[3].equals(WITH)) {
+ shouldReplace = true;
+ oldWord = tokens[2];
+ newWord = tokens[4];
+ } else {
+ throw new MojoExecutionException("Allowed syntax after @merge: <REPLACE> oldWord <WITH> newWord");
+ }
+ }
+ return replace;
+ }
+
private void handleOverride(StringBuilder identifier, BufferedReader reader)
throws MojoExecutionException, IOException {
readBlock(reader, OPEN_BRACE, CLOSE_BRACE);
@@ -700,6 +774,11 @@ public class GrammarExtensionMojo extends AbstractMojo {
String sig = toSignature(identifier.toString());
if (!extensibles.containsKey(sig)) {
throw new MojoExecutionException(identifier.toString() + " doesn't exist in base grammar");
+ } else if (shouldReplace) {
+ Pair<String, String> baseMethods = extensibles.get(sig);
+ baseMethods.first = baseMethods.first.replaceAll(oldWord, newWord);
+ baseMethods.second = baseMethods.second.replaceAll(oldWord, newWord);
+ shouldReplace = false;
}
String[] amendments = new String[6];
mergeElements.put(sig, amendments);
@@ -769,8 +848,8 @@ public class GrammarExtensionMojo extends AbstractMojo {
int after = block.indexOf(AFTER);
if (before >= 0) {
// before exists
- amendments[beforeIndex] =
- block.substring(before + BEFORE.length(), (after >= 0) ? after : block.length() - 1);
+ amendments[beforeIndex] = block.substring(before + BEFORE.length(),
+ (after >= 0) ? after : block.length() - 1);
if (amendments[beforeIndex].trim().length() == 0) {
amendments[beforeIndex] = null;
}
@@ -791,7 +870,7 @@ public class GrammarExtensionMojo extends AbstractMojo {
private File prepareOutputFile() throws MojoExecutionException {
// write output
- File outputFile = new File(output);
+ File outputFile = new File(base, output);
if (outputFile.exists() && (!outputFile.delete())) {
throw new MojoExecutionException("Unable to delete file " + output);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/java/org/apache/asterix/lang/extension/EchoStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/java/org/apache/asterix/lang/extension/EchoStatement.java b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/java/org/apache/asterix/lang/extension/EchoStatement.java
index 7c6e882..3d98299 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/java/org/apache/asterix/lang/extension/EchoStatement.java
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/java/org/apache/asterix/lang/extension/EchoStatement.java
@@ -44,4 +44,9 @@ public class EchoStatement implements Statement {
return arg;
}
+ @Override
+ public byte getCategory() {
+ return Category.PROCEDURE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
index 2106e4d..fb143a3 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/lang/extension.jj
@@ -1,9 +1,14 @@
import org.apache.asterix.lang.extension.EchoStatement;
+// to remove an import, we use the keyword, unimport
+// unimport package name
// Merging of non-terminals can only be done on non-terminals which conform to the following structure.
// Content will simply be prepended or appended to the base blocks.
// Note: refrain from using the strings "before:" and "after:" in the merge areas as that will break the merge.
// As a workaround, you can always override
+// one additional possible change is direct replacement and it can be done through
+// @merge replace "baseWord" with "extensionWord"
+
@merge
Statement SingleStatement() throws ParseException:
{
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
index b8253bb..5475fa4 100644
--- a/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
+++ b/asterixdb/asterix-maven-plugins/asterix-grammar-extension-maven-plugin/src/test/resources/unit/basic-test/basic-test-plugin-config.xml
@@ -1,80 +1,86 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-opt</artifactId>
+ <version>0.1-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.extension</groupId>
+ <artifactId>grammar-extension</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-external-data</artifactId>
+ <version>0.8.9-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>com.couchbase.client</groupId>
+ <artifactId>core-io</artifactId>
+ <version>1.2.8</version>
+ </dependency>
+ <dependency>
+ <groupId>io.reactivex</groupId>
+ <artifactId>rxjava</artifactId>
+ <version>1.0.15</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven</groupId>
+ <artifactId>maven-project</artifactId>
+ <version>2.0.8</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
<groupId>org.apache.asterix</groupId>
- <artifactId>asterix-opt</artifactId>
- <version>0.1-SNAPSHOT</version>
- </parent>
- <groupId>org.apache.extension</groupId>
- <artifactId>grammar-extension</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-external-data</artifactId>
- <version>0.8.9-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>com.couchbase.client</groupId>
- <artifactId>core-io</artifactId>
- <version>1.2.8</version>
- </dependency>
- <dependency>
- <groupId>io.reactivex</groupId>
- <artifactId>rxjava</artifactId>
- <version>1.0.15</version>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-grammar-extension-maven-plugin</artifactId>
- <version>0.8.9-SNAPSHOT</version>
- <configuration>
- <base>../../asterix-lang-aql/src/main/javacc/AQL.jj</base>
- <extension>src/test/resources/lang/extension.jj</extension>
- <output>target/generated-sources/lang/grammar.jj</output>
- <parserClassName>ExtendedParser</parserClassName>
- <packageName>org.apache.asterix.lang.extension.parser</packageName>
- </configuration>
- <executions>
- <execution>
- <phase>generate-sources</phase>
- <goals>
- <goal>grammarix</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
+ <artifactId>asterix-grammar-extension-maven-plugin</artifactId>
+ <version>0.8.9-SNAPSHOT</version>
+ <configuration>
+ <base>./</base>
+ <gbase>../../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj</gbase>
+ <gextension>src/test/resources/lang/extension.jj</gextension>
+ <output>target/generated-sources/lang/grammar.jj</output>
+ <parserClassName>ExtendedParser</parserClassName>
+ <packageName>org.apache.asterix.lang.extension.parser</packageName>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>grammarix</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
<!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>1.0.0</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>org.apache.asterix</groupId>
- <artifactId>asterix-grammar-extension-maven-plugin</artifactId>
- <versionRange>[0.8.9-SNAPSHOT,)</versionRange>
- <goals>
- <goal>grammarix</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore></ignore>
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-grammar-extension-maven-plugin</artifactId>
+ <versionRange>[0.8.9-SNAPSHOT,)</versionRange>
+ <goals>
+ <goal>grammarix</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
</project>
\ No newline at end of file
[09/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
index 821822f..486b4ab 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.api.java.AsterixJavaClient;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
@@ -56,8 +57,8 @@ public class AsterixCLI {
try {
for (String queryFile : options.args) {
Reader in = new FileReader(queryFile);
- AsterixJavaClient ajc = new AsterixJavaClient(
- integrationUtil.getHyracksClientConnection(), in, compilationProvider);
+ AsterixJavaClient ajc = new AsterixJavaClient(integrationUtil.getHyracksClientConnection(), in,
+ compilationProvider, new DefaultStatementExecutorFactory(null));
try {
ajc.compile(true, false, false, false, false, true, false);
} finally {
@@ -77,8 +78,9 @@ public class AsterixCLI {
outdir.mkdirs();
File log = new File("asterix_logs");
- if (log.exists())
+ if (log.exists()) {
FileUtils.deleteDirectory(log);
+ }
File lsn = new File("last_checkpoint_lsn");
lsn.deleteOnExit();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
index 2456bc2..e0a60c6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
@@ -22,6 +22,7 @@ import java.io.FileReader;
import org.apache.asterix.api.common.AsterixClientConfig;
import org.apache.asterix.api.java.AsterixJavaClient;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.hyracks.api.client.HyracksConnection;
@@ -62,7 +63,8 @@ public class AsterixClientDriver {
boolean onlyPhysical, boolean createBinaryRuntime) throws Exception {
ILangCompilationProvider compilationProvider = new AqlCompilationProvider();
FileReader reader = new FileReader(filename);
- AsterixJavaClient q = new AsterixJavaClient(hcc, reader, compilationProvider);
+ AsterixJavaClient q = new AsterixJavaClient(hcc, reader, compilationProvider,
+ new DefaultStatementExecutorFactory(null));
q.compile(optimize, true, true, true, onlyPhysical, createBinaryRuntime, createBinaryRuntime);
return q;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
index f4c8062..8bcceca 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
@@ -19,18 +19,22 @@
package org.apache.asterix.drivers;
import org.apache.asterix.api.http.servlet.APIServlet;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
+import org.apache.asterix.compiler.provider.AqlCompilationProvider;
+import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
public class AsterixWebServer {
- public static void main(String args[]) throws Exception {
+ public static void main(String[] args) throws Exception {
Server server = new Server(8080);
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
server.setHandler(context);
- context.addServlet(new ServletHolder(new APIServlet()), "/*");
+ context.addServlet(new ServletHolder(new APIServlet(new AqlCompilationProvider(),
+ new SqlppCompilationProvider(), new DefaultStatementExecutorFactory(null))), "/*");
server.start();
server.join();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 3ab6f3d..cf57174 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -18,6 +18,9 @@
*/
package org.apache.asterix.hyracks.bootstrap;
+import static org.apache.asterix.api.http.servlet.ServletConstants.ASTERIX_BUILD_PROP_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
@@ -39,6 +42,7 @@ import org.apache.asterix.api.http.servlet.QueryStatusAPIServlet;
import org.apache.asterix.api.http.servlet.ShutdownAPIServlet;
import org.apache.asterix.api.http.servlet.UpdateAPIServlet;
import org.apache.asterix.api.http.servlet.VersionAPIServlet;
+import org.apache.asterix.app.cc.CompilerExtensionManager;
import org.apache.asterix.app.external.ActiveLifecycleListener;
import org.apache.asterix.app.external.ExternalLibraryUtils;
import org.apache.asterix.common.api.AsterixThreadFactory;
@@ -47,8 +51,6 @@ import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.common.config.AsterixMetadataProperties;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.utils.ServletUtil.Servlets;
-import org.apache.asterix.compiler.provider.AqlCompilationProvider;
-import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
import org.apache.asterix.event.service.ILookupService;
import org.apache.asterix.external.library.ExternalLibraryManager;
import org.apache.asterix.messaging.CCMessageBroker;
@@ -69,9 +71,6 @@ import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
-import static org.apache.asterix.api.http.servlet.ServletConstants.ASTERIX_BUILD_PROP_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-
public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
private static final Logger LOGGER = Logger.getLogger(CCApplicationEntryPoint.class.getName());
@@ -80,6 +79,7 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
private static IAsterixStateProxy proxy;
protected ICCApplicationContext appCtx;
+ protected CompilerExtensionManager ccExtensionManager;
@Override
public void start(ICCApplicationContext ccAppCtx, String[] args) throws Exception {
@@ -91,22 +91,25 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
}
appCtx.setThreadFactory(new AsterixThreadFactory(new LifeCycleComponentManager()));
- GlobalRecoveryManager.INSTANCE = new GlobalRecoveryManager((HyracksConnection) getNewHyracksClientConnection());
+ GlobalRecoveryManager.instantiate((HyracksConnection) getNewHyracksClientConnection());
ILibraryManager libraryManager = new ExternalLibraryManager();
ExternalLibraryUtils.setUpExternaLibraries(libraryManager, false);
- AsterixAppContextInfo.initialize(appCtx, getNewHyracksClientConnection(), GlobalRecoveryManager.INSTANCE,
+ AsterixAppContextInfo.initialize(appCtx, getNewHyracksClientConnection(), GlobalRecoveryManager.instance(),
libraryManager);
+ ccExtensionManager = new CompilerExtensionManager(
+ AsterixAppContextInfo.getInstance().getExtensionProperties().getExtensions());
+ AsterixAppContextInfo.getInstance().setExtensionManager(ccExtensionManager);
if (System.getProperty("java.rmi.server.hostname") == null) {
System.setProperty("java.rmi.server.hostname",
((ClusterControllerService) ccAppCtx.getControllerService()).getCCConfig().clusterNetIpAddress);
}
- proxy = AsterixStateProxy.registerRemoteObject();
+ setAsterixStateProxy(AsterixStateProxy.registerRemoteObject());
appCtx.setDistributedState(proxy);
AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties();
- MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
+ MetadataManager.instantiate(new MetadataManager(proxy, metadataProperties));
AsterixAppContextInfo.getInstance().getCCApplicationContext()
.addJobLifecycleListener(ActiveLifecycleListener.INSTANCE);
@@ -117,7 +120,7 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
server.start();
}
- ClusterManager.INSTANCE.registerSubscriber(GlobalRecoveryManager.INSTANCE);
+ ClusterManager.INSTANCE.registerSubscriber(GlobalRecoveryManager.instance());
ccAppCtx.addClusterLifecycleListener(ClusterLifecycleListener.INSTANCE);
ccAppCtx.setMessageBroker(messageBroker);
@@ -168,7 +171,9 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
context.setAttribute(HYRACKS_CONNECTION_ATTR, hcc);
webServer.setHandler(context);
- context.addServlet(new ServletHolder(new APIServlet()), "/*");
+ context.addServlet(new ServletHolder(new APIServlet(ccExtensionManager.getAqlCompilationProvider(),
+ ccExtensionManager.getSqlppCompilationProvider(), ccExtensionManager.getQueryTranslatorFactory())),
+ "/*");
return webServer;
}
@@ -235,27 +240,36 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
protected Servlet createServlet(Servlets key) {
switch (key) {
case AQL:
- return new AQLAPIServlet(new AqlCompilationProvider());
+ return new AQLAPIServlet(ccExtensionManager.getAqlCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case AQL_QUERY:
- return new QueryAPIServlet(new AqlCompilationProvider());
+ return new QueryAPIServlet(ccExtensionManager.getAqlCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case AQL_UPDATE:
- return new UpdateAPIServlet(new AqlCompilationProvider());
+ return new UpdateAPIServlet(ccExtensionManager.getAqlCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case AQL_DDL:
- return new DDLAPIServlet(new AqlCompilationProvider());
+ return new DDLAPIServlet(ccExtensionManager.getAqlCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case SQLPP:
- return new AQLAPIServlet(new SqlppCompilationProvider());
+ return new AQLAPIServlet(ccExtensionManager.getSqlppCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case SQLPP_QUERY:
- return new QueryAPIServlet(new SqlppCompilationProvider());
+ return new QueryAPIServlet(ccExtensionManager.getSqlppCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case SQLPP_UPDATE:
- return new UpdateAPIServlet(new SqlppCompilationProvider());
+ return new UpdateAPIServlet(ccExtensionManager.getSqlppCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case SQLPP_DDL:
- return new DDLAPIServlet(new SqlppCompilationProvider());
+ return new DDLAPIServlet(ccExtensionManager.getSqlppCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case QUERY_STATUS:
return new QueryStatusAPIServlet();
case QUERY_RESULT:
return new QueryResultAPIServlet();
case QUERY_SERVICE:
- return new QueryServiceServlet(new SqlppCompilationProvider());
+ return new QueryServiceServlet(ccExtensionManager.getSqlppCompilationProvider(),
+ ccExtensionManager.getQueryTranslatorFactory());
case CONNECTOR:
return new ConnectorAPIServlet();
case SHUTDOWN:
@@ -294,4 +308,8 @@ public class CCApplicationEntryPoint implements ICCApplicationEntryPoint {
ClusterState.ACTIVE);
}
}
-}
\ No newline at end of file
+
+ public static synchronized void setAsterixStateProxy(IAsterixStateProxy proxy) {
+ CCApplicationEntryPoint.proxy = proxy;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
index af8aa31..0d5cfb2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/GlobalRecoveryManager.java
@@ -32,7 +32,6 @@ import org.apache.asterix.common.cluster.IGlobalRecoveryMaanger;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -41,6 +40,7 @@ import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Dataverse;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.job.JobId;
@@ -48,19 +48,19 @@ import org.apache.hyracks.api.job.JobSpecification;
public class GlobalRecoveryManager implements IGlobalRecoveryMaanger {
- private static ClusterState state;
private static final Logger LOGGER = Logger.getLogger(GlobalRecoveryManager.class.getName());
+ private static GlobalRecoveryManager instance;
+ private static ClusterState state;
private HyracksConnection hcc;
- public static GlobalRecoveryManager INSTANCE;
- public GlobalRecoveryManager(HyracksConnection hcc) throws Exception {
- state = ClusterState.UNUSABLE;
+ private GlobalRecoveryManager(HyracksConnection hcc) {
+ setState(ClusterState.UNUSABLE);
this.hcc = hcc;
}
@Override
public Set<IClusterManagementWork> notifyNodeFailure(Set<String> deadNodeIds) {
- state = AsterixClusterProperties.INSTANCE.getState();
+ setState(AsterixClusterProperties.INSTANCE.getState());
AsterixClusterProperties.INSTANCE.setGlobalRecoveryCompleted(false);
return Collections.emptySet();
}
@@ -115,7 +115,7 @@ public class GlobalRecoveryManager implements IGlobalRecoveryMaanger {
// Get indexes
List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx,
dataset.getDataverseName(), dataset.getDatasetName());
- if (indexes.size() > 0) {
+ if (!indexes.isEmpty()) {
// Get the state of the dataset
ExternalDatasetDetails dsd = (ExternalDatasetDetails) dataset
.getDatasetDetails();
@@ -217,8 +217,20 @@ public class GlobalRecoveryManager implements IGlobalRecoveryMaanger {
LOGGER.info("Global Recovery Completed");
}
});
- state = newState;
+ setState(newState);
recoveryThread.start();
}
}
+
+ public static GlobalRecoveryManager instance() {
+ return instance;
+ }
+
+ public static synchronized void instantiate(HyracksConnection hcc) {
+ instance = new GlobalRecoveryManager(hcc);
+ }
+
+ public static synchronized void setState(ClusterState state) {
+ GlobalRecoveryManager.state = state;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 4cb9d70..5abe6bc 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -25,8 +25,8 @@ import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.api.common.AsterixAppRuntimeContext;
import org.apache.asterix.app.external.ExternalLibraryUtils;
+import org.apache.asterix.app.nc.AsterixNCAppRuntimeContext;
import org.apache.asterix.common.api.AsterixThreadFactory;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.config.AsterixMetadataProperties;
@@ -100,8 +100,7 @@ public class NCApplicationEntryPoint implements INCApplicationEntryPoint {
((NodeControllerService) ncAppCtx.getControllerService())
.getConfiguration().clusterNetPublicIPAddress);
}
-
- runtimeContext = new AsterixAppRuntimeContext(ncApplicationContext, metadataRmiPort);
+ runtimeContext = new AsterixNCAppRuntimeContext(ncApplicationContext, metadataRmiPort);
AsterixMetadataProperties metadataProperties = ((IAsterixPropertiesProvider) runtimeContext)
.getMetadataProperties();
if (!metadataProperties.getNodeNames().contains(ncApplicationContext.getNodeId())) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java
index 138b620..27a5365 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
-import org.apache.asterix.active.ActivePartitionMessage;
+import org.apache.asterix.active.message.ActivePartitionMessage;
import org.apache.asterix.app.external.ActiveLifecycleListener;
import org.apache.asterix.common.messaging.AbstractApplicationMessage;
import org.apache.asterix.common.messaging.CompleteFailbackResponseMessage;
@@ -81,7 +81,7 @@ public class CCMessageBroker implements ICCMessageBroker {
case COMPLETE_FAILBACK_RESPONSE:
handleCompleteFailbcakResponse(message);
break;
- case ACTIVE_ENTITY_MESSAGE:
+ case ACTIVE_ENTITY_TO_CC_MESSAGE:
handleActiveEntityMessage(message);
break;
default:
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/NCMessageBroker.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/NCMessageBroker.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/NCMessageBroker.java
index 13b0189..74a5ba2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/NCMessageBroker.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/NCMessageBroker.java
@@ -24,6 +24,8 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.asterix.active.ActiveManager;
+import org.apache.asterix.active.message.ActiveManagerMessage;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.messaging.AbstractApplicationMessage;
import org.apache.asterix.common.messaging.CompleteFailbackRequestMessage;
@@ -113,6 +115,9 @@ public class NCMessageBroker implements INCMessageBroker {
case REPLICA_EVENT:
handleReplicaEvent(message);
break;
+ case ACTIVE_MANAGER_MESSAGE:
+ ((ActiveManager) appContext.getActiveManager()).submit((ActiveManagerMessage) message);
+ break;
default:
break;
}
@@ -143,8 +148,8 @@ public class NCMessageBroker implements INCMessageBroker {
appContext.initializeMetadata(false);
appContext.exportMetadataNodeStub();
} finally {
- TakeoverMetadataNodeResponseMessage reponse = new TakeoverMetadataNodeResponseMessage(
- appContext.getTransactionSubsystem().getId());
+ TakeoverMetadataNodeResponseMessage reponse =
+ new TakeoverMetadataNodeResponseMessage(appContext.getTransactionSubsystem().getId());
sendMessage(reponse, null);
}
}
@@ -189,15 +194,15 @@ public class NCMessageBroker implements INCMessageBroker {
}
//mark the partitions to be closed as inactive
- PersistentLocalResourceRepository localResourceRepo = (PersistentLocalResourceRepository) appContext
- .getLocalResourceRepository();
+ PersistentLocalResourceRepository localResourceRepo =
+ (PersistentLocalResourceRepository) appContext.getLocalResourceRepository();
for (Integer partitionId : msg.getPartitions()) {
localResourceRepo.addInactivePartition(partitionId);
}
//send response after partitions prepared for failback
- PreparePartitionsFailbackResponseMessage reponse = new PreparePartitionsFailbackResponseMessage(msg.getPlanId(),
- msg.getRequestId(), msg.getPartitions());
+ PreparePartitionsFailbackResponseMessage reponse =
+ new PreparePartitionsFailbackResponseMessage(msg.getPlanId(), msg.getRequestId(), msg.getPartitions());
sendMessage(reponse, null);
}
@@ -207,8 +212,8 @@ public class NCMessageBroker implements INCMessageBroker {
IRemoteRecoveryManager remoteRecoeryManager = appContext.getRemoteRecoveryManager();
remoteRecoeryManager.completeFailbackProcess();
} finally {
- CompleteFailbackResponseMessage reponse = new CompleteFailbackResponseMessage(msg.getPlanId(),
- msg.getRequestId(), msg.getPartitions());
+ CompleteFailbackResponseMessage reponse =
+ new CompleteFailbackResponseMessage(msg.getPlanId(), msg.getRequestId(), msg.getPartitions());
sendMessage(reponse, null);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultPrinter.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultPrinter.java
deleted file mode 100644
index 8e16ef7..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultPrinter.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.result;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.nio.ByteBuffer;
-
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.common.utils.JSONUtil;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-
-public class ResultPrinter {
-
- // TODO(tillw): Should this be static?
- private static FrameManager resultDisplayFrameMgr = new FrameManager(ResultReader.FRAME_SIZE);
-
- private final SessionConfig conf;
- private final ResultUtils.Stats stats;
- private final ARecordType recordType;
-
- private boolean indentJSON;
- private boolean quoteRecord;
-
- // Whether we are wrapping the output sequence in an array
- private boolean wrapArray = false;
- // Whether this is the first instance being output
- private boolean notFirst = false;
-
- public ResultPrinter(SessionConfig conf, ResultUtils.Stats stats, ARecordType recordType) {
- this.conf = conf;
- this.stats = stats;
- this.recordType = recordType;
- this.indentJSON = conf.is(SessionConfig.FORMAT_INDENT_JSON);
- this.quoteRecord = conf.is(SessionConfig.FORMAT_QUOTE_RECORD);
- }
-
- private static void appendCSVHeader(Appendable app, ARecordType recordType) throws HyracksDataException {
- try {
- String[] fieldNames = recordType.getFieldNames();
- boolean notfirst = false;
- for (String name : fieldNames) {
- if (notfirst) {
- app.append(',');
- }
- notfirst = true;
- app.append('"').append(name.replace("\"", "\"\"")).append('"');
- }
- app.append("\r\n");
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
-
- private void printPrefix() throws HyracksDataException {
- // If we're outputting CSV with a header, the HTML header was already
- // output by displayCSVHeader(), so skip it here
- if (conf.is(SessionConfig.FORMAT_HTML)) {
- conf.out().println("<h4>Results:</h4>");
- conf.out().println("<pre>");
- }
-
- try {
- conf.resultPrefix(new AlgebricksAppendable(conf.out()));
- } catch (AlgebricksException e) {
- throw new HyracksDataException(e);
- }
-
- if (conf.is(SessionConfig.FORMAT_WRAPPER_ARRAY)) {
- conf.out().print("[ ");
- wrapArray = true;
- }
-
- if (conf.fmt() == SessionConfig.OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER)) {
- if (recordType == null) {
- throw new HyracksDataException("Cannot print CSV with header without specifying output-record-type");
- }
- if (quoteRecord) {
- StringWriter sw = new StringWriter();
- appendCSVHeader(sw, recordType);
- conf.out().print(JSONUtil.quoteAndEscape(sw.toString()));
- conf.out().print("\n");
- notFirst = true;
- } else {
- appendCSVHeader(conf.out(), recordType);
- }
- }
- }
-
- private void printPostfix() throws HyracksDataException {
- conf.out().flush();
- if (wrapArray) {
- conf.out().println(" ]");
- }
- try {
- conf.resultPostfix(new AlgebricksAppendable(conf.out()));
- } catch (AlgebricksException e) {
- throw new HyracksDataException(e);
- }
- if (conf.is(SessionConfig.FORMAT_HTML)) {
- conf.out().println("</pre>");
- }
- }
-
- private void displayRecord(String result) {
- String record = result;
- if (indentJSON) {
- // TODO(tillw): this is inefficient - do this during record generation
- record = JSONUtil.indent(record, 2);
- }
- if (conf.fmt() == SessionConfig.OutputFormat.CSV) {
- // TODO(tillw): this is inefficient as well
- record = record + "\r\n";
- }
- if (quoteRecord) {
- // TODO(tillw): this is inefficient as well
- record = JSONUtil.quoteAndEscape(record);
- }
- conf.out().print(record);
- ++stats.count;
- // TODO(tillw) fix this approximation
- stats.size += record.length();
- }
-
- public void print(String record) throws HyracksDataException {
- printPrefix();
- // TODO(tillw) evil hack
- quoteRecord = true;
- displayRecord(record);
- printPostfix();
- }
-
- public void print(ResultReader resultReader) throws HyracksDataException {
- printPrefix();
-
- final IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor();
- final IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
-
- while (resultReader.read(frame) > 0) {
- final ByteBuffer frameBuffer = frame.getBuffer();
- final byte[] frameBytes = frameBuffer.array();
- fta.reset(frameBuffer);
- final int last = fta.getTupleCount();
- for (int tIndex = 0; tIndex < last; tIndex++) {
- final int start = fta.getTupleStartOffset(tIndex);
- int length = fta.getTupleEndOffset(tIndex) - start;
- if (conf.fmt() == SessionConfig.OutputFormat.CSV
- && ((length > 0) && (frameBytes[start + length - 1] == '\n'))) {
- length--;
- }
- String result = new String(frameBytes, start, length, UTF_8);
- if (wrapArray && notFirst) {
- conf.out().print(", ");
- }
- notFirst = true;
- displayRecord(result);
- }
- frameBuffer.clear();
- }
-
- printPostfix();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java
deleted file mode 100644
index 373c0d0..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.result;
-
-import org.apache.asterix.om.util.AsterixAppContextInfo;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
-
-public class ResultReader {
- private final IHyracksDataset hyracksDataset;
-
- private IHyracksDatasetReader reader;
-
- private IFrameTupleAccessor frameTupleAccessor;
-
- // Number of parallel result reader buffers
- public static final int NUM_READERS = 1;
-
- public static final int FRAME_SIZE = AsterixAppContextInfo.getInstance().getCompilerProperties().getFrameSize();
-
- public ResultReader(IHyracksClientConnection hcc, IHyracksDataset hdc) throws Exception {
- hyracksDataset = hdc;
- }
-
- public void open(JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
- reader = hyracksDataset.createReader(jobId, resultSetId);
- frameTupleAccessor = new ResultFrameTupleAccessor();
- }
-
- public Status getStatus() {
- return reader.getResultStatus();
- }
-
- public int read(IFrame frame) throws HyracksDataException {
- return reader.read(frame);
- }
-
- public IFrameTupleAccessor getFrameTupleAccessor() {
- return frameTupleAccessor;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
deleted file mode 100644
index 3503549..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.result;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.http.servlet.APIServlet;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.http.ParseException;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-public class ResultUtils {
- static Map<Character, String> HTML_ENTITIES = new HashMap<Character, String>();
-
- static {
- HTML_ENTITIES.put('"', """);
- HTML_ENTITIES.put('&', "&");
- HTML_ENTITIES.put('<', "<");
- HTML_ENTITIES.put('>', ">");
- }
-
- public static class Stats {
- public long count;
- public long size;
- }
-
- public static String escapeHTML(String s) {
- for (Character c : HTML_ENTITIES.keySet()) {
- if (s.indexOf(c) >= 0) {
- s = s.replace(c.toString(), HTML_ENTITIES.get(c));
- }
- }
- return s;
- }
-
- public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats,
- ARecordType recordType) throws HyracksDataException {
- new ResultPrinter(conf, stats, recordType).print(resultReader);
- }
-
- public static void displayResults(String record, SessionConfig conf, Stats stats, ARecordType recordType)
- throws HyracksDataException {
- new ResultPrinter(conf, stats, recordType).print(record);
- }
-
- public static JSONObject getErrorResponse(int errorCode, String errorMessage, String errorSummary,
- String errorStackTrace) {
- JSONObject errorResp = new JSONObject();
- JSONArray errorArray = new JSONArray();
- errorArray.put(errorCode);
- errorArray.put(errorMessage);
- try {
- errorResp.put("error-code", errorArray);
- if (!"".equals(errorSummary)) {
- errorResp.put("summary", errorSummary);
- } else {
- //parse exception
- errorResp.put("summary", errorMessage);
- }
- errorResp.put("stacktrace", errorStackTrace);
- } catch (JSONException e) {
- // TODO(madhusudancs): Figure out what to do when JSONException occurs while building the results.
- }
- return errorResp;
- }
-
- public static void webUIErrorHandler(PrintWriter out, Exception e) {
- String errorTemplate = readTemplateFile("/webui/errortemplate.html", "%s\n%s\n%s");
-
- String errorOutput = String.format(errorTemplate, escapeHTML(extractErrorMessage(e)),
- escapeHTML(extractErrorSummary(e)), escapeHTML(extractFullStackTrace(e)));
- out.println(errorOutput);
- }
-
- public static void webUIParseExceptionHandler(PrintWriter out, Throwable e, String query) {
- String errorTemplate = readTemplateFile("/webui/errortemplate_message.html", "<pre class=\"error\">%s\n</pre>");
-
- String errorOutput = String.format(errorTemplate, buildParseExceptionMessage(e, query));
- out.println(errorOutput);
- }
-
- public static void apiErrorHandler(PrintWriter out, Exception e) {
- int errorCode = 99;
- if (e instanceof ParseException) {
- errorCode = 2;
- } else if (e instanceof AlgebricksException) {
- errorCode = 3;
- } else if (e instanceof HyracksDataException) {
- errorCode = 4;
- }
-
- JSONObject errorResp = ResultUtils.getErrorResponse(errorCode, extractErrorMessage(e), extractErrorSummary(e),
- extractFullStackTrace(e));
- out.write(errorResp.toString());
- }
-
- public static String buildParseExceptionMessage(Throwable e, String query) {
- StringBuilder errorMessage = new StringBuilder();
- String message = e.getMessage();
- message = message.replace("<", "<");
- message = message.replace(">", ">");
- errorMessage.append("Error: " + message + "\n");
- int pos = message.indexOf("line");
- if (pos > 0) {
- Pattern p = Pattern.compile("\\d+");
- Matcher m = p.matcher(message);
- if (m.find(pos)) {
- int lineNo = Integer.parseInt(message.substring(m.start(), m.end()));
- String[] lines = query.split("\n");
- if (lineNo > lines.length) {
- errorMessage.append("===> <BLANK LINE> \n");
- } else {
- String line = lines[lineNo - 1];
- errorMessage.append("==> " + line);
- }
- }
- }
- return errorMessage.toString();
- }
-
- private static Throwable getRootCause(Throwable cause) {
- Throwable nextCause = cause.getCause();
- while (nextCause != null) {
- cause = nextCause;
- nextCause = cause.getCause();
- }
- return cause;
- }
-
- /**
- * Extract the message in the root cause of the stack trace:
- *
- * @param e
- * @return error message string.
- */
- private static String extractErrorMessage(Throwable e) {
- Throwable cause = getRootCause(e);
- String fullyQualifiedExceptionClassName = cause.getClass().getName();
- String[] hierarchySplits = fullyQualifiedExceptionClassName.split("\\.");
- //try returning the class without package qualification
- String exceptionClassName = hierarchySplits[hierarchySplits.length - 1];
- String localizedMessage = cause.getLocalizedMessage();
- if (localizedMessage == null) {
- localizedMessage = "Internal error. Please check instance logs for further details.";
- }
- return localizedMessage + " [" + exceptionClassName + "]";
- }
-
- /**
- * Extract the meaningful part of a stack trace:
- * a. the causes in the stack trace hierarchy
- * b. the top exception for each cause
- *
- * @param e
- * @return the contacted message containing a and b.
- */
- private static String extractErrorSummary(Throwable e) {
- StringBuilder errorMessageBuilder = new StringBuilder();
- Throwable cause = e;
- errorMessageBuilder.append(cause.getLocalizedMessage());
- while (cause != null) {
- StackTraceElement[] stackTraceElements = cause.getStackTrace();
- errorMessageBuilder.append(stackTraceElements.length > 0 ? "\n caused by: " + stackTraceElements[0] : "");
- cause = cause.getCause();
- }
- return errorMessageBuilder.toString();
- }
-
- /**
- * Extract the full stack trace:
- *
- * @param e
- * @return the string containing the full stack trace of the error.
- */
- public static String extractFullStackTrace(Throwable e) {
- StringWriter stringWriter = new StringWriter();
- PrintWriter printWriter = new PrintWriter(stringWriter);
- e.printStackTrace(printWriter);
- return stringWriter.toString();
- }
-
- /**
- * Read the template file which is stored as a resource and return its content. If the file does not exist or is
- * not readable return the default template string.
- *
- * @param path
- * The path to the resource template file
- * @param defaultTemplate
- * The default template string if the template file does not exist or is not readable
- * @return The template string to be used to render the output.
- */
- private static String readTemplateFile(String path, String defaultTemplate) {
- String errorTemplate = defaultTemplate;
- try {
- String resourcePath = "/webui/errortemplate_message.html";
- InputStream is = APIServlet.class.getResourceAsStream(resourcePath);
- InputStreamReader isr = new InputStreamReader(is);
- StringBuilder sb = new StringBuilder();
- BufferedReader br = new BufferedReader(isr);
- String line = br.readLine();
-
- while (line != null) {
- sb.append(line);
- line = br.readLine();
- }
- errorTemplate = sb.toString();
- } catch (IOException ioe) {
- // If there is an IOException reading the error template html file, default value of error template is used.
- }
- return errorTemplate;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
index 596dc41..7e35f11 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/app/bootstrap/TestNodeController.java
@@ -23,9 +23,8 @@ import java.util.Map;
import java.util.logging.Logger;
import org.apache.asterix.algebra.operators.physical.CommitRuntime;
-import org.apache.asterix.api.common.AsterixAppRuntimeContext;
-import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.app.nc.AsterixNCAppRuntimeContext;
import org.apache.asterix.common.api.ILocalResourceMetadata;
import org.apache.asterix.common.config.AsterixTransactionProperties;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
@@ -97,8 +96,8 @@ public class TestNodeController {
protected static final Logger LOGGER = Logger.getLogger(TestNodeController.class.getName());
protected static final String PATH_ACTUAL = "unittest" + File.separator;
- protected static final String PATH_BASE =
- StringUtils.join(new String[] { "src", "test", "resources", "nodetests" }, File.separator);
+ protected static final String PATH_BASE = StringUtils.join(new String[] { "src", "test", "resources", "nodetests" },
+ File.separator);
protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
protected static AsterixTransactionProperties txnProperties;
@@ -158,18 +157,18 @@ public class TestNodeController {
PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType,
mergePolicyFactory, mergePolicyProperties, filterFields);
IndexOperation op = IndexOperation.INSERT;
- IModificationOperationCallbackFactory modOpCallbackFactory =
- new PrimaryIndexModificationOperationCallbackFactory(getTxnJobId(), dataset.getDatasetId(),
- primaryIndexInfo.primaryKeyIndexes, TXN_SUBSYSTEM_PROVIDER, op, ResourceType.LSM_BTREE, true);
- AsterixLSMTreeInsertDeleteOperatorDescriptor indexOpDesc =
- getInsertOpratorDesc(primaryIndexInfo, modOpCallbackFactory);
- LSMBTreeDataflowHelperFactory dataflowHelperFactory =
- getPrimaryIndexDataflowHelperFactory(ctx, primaryIndexInfo);
+ IModificationOperationCallbackFactory modOpCallbackFactory = new PrimaryIndexModificationOperationCallbackFactory(
+ getTxnJobId(), dataset.getDatasetId(), primaryIndexInfo.primaryKeyIndexes, TXN_SUBSYSTEM_PROVIDER, op,
+ ResourceType.LSM_BTREE, true);
+ AsterixLSMTreeInsertDeleteOperatorDescriptor indexOpDesc = getInsertOpratorDesc(primaryIndexInfo,
+ modOpCallbackFactory);
+ LSMBTreeDataflowHelperFactory dataflowHelperFactory = getPrimaryIndexDataflowHelperFactory(ctx,
+ primaryIndexInfo);
Mockito.when(indexOpDesc.getIndexDataflowHelperFactory()).thenReturn(dataflowHelperFactory);
IRecordDescriptorProvider recordDescProvider = primaryIndexInfo.getInsertRecordDescriptorProvider();
- AsterixLSMInsertDeleteOperatorNodePushable insertOp =
- new AsterixLSMInsertDeleteOperatorNodePushable(indexOpDesc, ctx, PARTITION,
- primaryIndexInfo.primaryIndexInsertFieldsPermutations, recordDescProvider, op, true);
+ AsterixLSMInsertDeleteOperatorNodePushable insertOp = new AsterixLSMInsertDeleteOperatorNodePushable(
+ indexOpDesc, ctx, PARTITION, primaryIndexInfo.primaryIndexInsertFieldsPermutations, recordDescProvider,
+ op, true);
CommitRuntime commitOp = new CommitRuntime(ctx, getTxnJobId(), dataset.getDatasetId(),
primaryIndexInfo.primaryKeyIndexes, false, true, PARTITION);
insertOp.setOutputFrameWriter(0, commitOp, primaryIndexInfo.rDesc);
@@ -185,8 +184,8 @@ public class TestNodeController {
JobSpecification spec = new JobSpecification();
PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType,
mergePolicyFactory, mergePolicyProperties, filterFields);
- LSMBTreeDataflowHelperFactory indexDataflowHelperFactory =
- getPrimaryIndexDataflowHelperFactory(ctx, primaryIndexInfo);
+ LSMBTreeDataflowHelperFactory indexDataflowHelperFactory = getPrimaryIndexDataflowHelperFactory(ctx,
+ primaryIndexInfo);
BTreeSearchOperatorDescriptor searchOpDesc = new BTreeSearchOperatorDescriptor(spec, primaryIndexInfo.rDesc,
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
primaryIndexInfo.fileSplitProvider, primaryIndexInfo.primaryIndexTypeTraits,
@@ -214,8 +213,8 @@ public class TestNodeController {
public AsterixLSMTreeInsertDeleteOperatorDescriptor getInsertOpratorDesc(PrimaryIndexInfo primaryIndexInfo,
IModificationOperationCallbackFactory modOpCallbackFactory) {
- AsterixLSMTreeInsertDeleteOperatorDescriptor indexOpDesc =
- Mockito.mock(AsterixLSMTreeInsertDeleteOperatorDescriptor.class);
+ AsterixLSMTreeInsertDeleteOperatorDescriptor indexOpDesc = Mockito
+ .mock(AsterixLSMTreeInsertDeleteOperatorDescriptor.class);
Mockito.when(indexOpDesc.getLifecycleManagerProvider())
.thenReturn(AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
Mockito.when(indexOpDesc.getStorageManager()).thenReturn(AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER);
@@ -258,12 +257,12 @@ public class TestNodeController {
int[] primaryIndexBloomFilterKeyFields, ILSMMergePolicyFactory mergePolicyFactory,
Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits,
IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields) {
- ILocalResourceMetadata localResourceMetadata =
- new LSMBTreeLocalResourceMetadata(primaryIndexTypeTraits, primaryIndexComparatorFactories,
- primaryIndexBloomFilterKeyFields, true, dataset.getDatasetId(), mergePolicyFactory,
- mergePolicyProperties, filterTypeTraits, filterCmpFactories, btreeFields, filterFields);
- ILocalResourceFactoryProvider localResourceFactoryProvider =
- new PersistentLocalResourceFactoryProvider(localResourceMetadata, LocalResource.LSMBTreeResource);
+ ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(primaryIndexTypeTraits,
+ primaryIndexComparatorFactories, primaryIndexBloomFilterKeyFields, true, dataset.getDatasetId(),
+ mergePolicyFactory, mergePolicyProperties, filterTypeTraits, filterCmpFactories, btreeFields,
+ filterFields);
+ ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
+ localResourceMetadata, LocalResource.LSMBTreeResource);
return localResourceFactoryProvider;
}
@@ -277,8 +276,8 @@ public class TestNodeController {
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMBTreeIOOperationCallbackFactory.INSTANCE,
BLOOM_FILTER_FALSE_POSITIVE_RATE, true, primaryIndexInfo.filterTypeTraits,
primaryIndexInfo.filterCmpFactories, primaryIndexInfo.btreeFields, primaryIndexInfo.filterFields, true);
- IndexDataflowHelper dataflowHelper =
- dataflowHelperFactory.createIndexDataflowHelper(indexOpDesc, ctx, PARTITION);
+ IndexDataflowHelper dataflowHelper = dataflowHelperFactory.createIndexDataflowHelper(indexOpDesc, ctx,
+ PARTITION);
return (LSMBTreeDataflowHelper) dataflowHelper;
}
@@ -309,8 +308,8 @@ public class TestNodeController {
PrimaryIndexInfo primaryIndexInfo = new PrimaryIndexInfo(dataset, primaryKeyTypes, recordType, metaType,
mergePolicyFactory, mergePolicyProperties, filterFields);
TreeIndexCreateOperatorDescriptor indexOpDesc = getIndexCreateOpDesc(primaryIndexInfo);
- LSMBTreeDataflowHelper dataflowHelper =
- getPrimaryIndexDataflowHelper(createTestContext(), primaryIndexInfo, indexOpDesc);
+ LSMBTreeDataflowHelper dataflowHelper = getPrimaryIndexDataflowHelper(createTestContext(), primaryIndexInfo,
+ indexOpDesc);
dataflowHelper.create();
}
@@ -323,11 +322,10 @@ public class TestNodeController {
}
private IBinaryComparatorFactory[] createPrimaryIndexComparatorFactories(IAType[] primaryKeyTypes) {
- IBinaryComparatorFactory[] primaryIndexComparatorFactories =
- new IBinaryComparatorFactory[primaryKeyTypes.length];
+ IBinaryComparatorFactory[] primaryIndexComparatorFactories = new IBinaryComparatorFactory[primaryKeyTypes.length];
for (int j = 0; j < primaryKeyTypes.length; ++j) {
- primaryIndexComparatorFactories[j] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(primaryKeyTypes[j], true);
+ primaryIndexComparatorFactories[j] = AqlBinaryComparatorFactoryProvider.INSTANCE
+ .getBinaryComparatorFactory(primaryKeyTypes[j], true);
}
return primaryIndexComparatorFactories;
}
@@ -337,8 +335,8 @@ public class TestNodeController {
int i = 0;
ISerializerDeserializer<?>[] primaryIndexSerdes = new ISerializerDeserializer<?>[primaryIndexNumOfTupleFields];
for (; i < primaryKeyTypes.length; i++) {
- primaryIndexSerdes[i] =
- AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(primaryKeyTypes[i]);
+ primaryIndexSerdes[i] = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(primaryKeyTypes[i]);
}
primaryIndexSerdes[i++] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recordType);
if (metaType != null) {
@@ -372,7 +370,7 @@ public class TestNodeController {
}
public TransactionSubsystem getTransactionSubsystem() {
- return (TransactionSubsystem) ((AsterixAppRuntimeContext) ExecutionTestUtil.integrationUtil.ncs[0]
+ return (TransactionSubsystem) ((AsterixNCAppRuntimeContext) ExecutionTestUtil.integrationUtil.ncs[0]
.getApplicationContext().getApplicationObject()).getTransactionSubsystem();
}
@@ -380,8 +378,8 @@ public class TestNodeController {
return getTransactionSubsystem().getTransactionManager();
}
- public AsterixAppRuntimeContext getAppRuntimeContext() {
- return (AsterixAppRuntimeContext) ExecutionTestUtil.integrationUtil.ncs[0].getApplicationContext()
+ public AsterixNCAppRuntimeContext getAppRuntimeContext() {
+ return (AsterixNCAppRuntimeContext) ExecutionTestUtil.integrationUtil.ncs[0].getApplicationContext()
.getApplicationObject();
}
@@ -423,8 +421,8 @@ public class TestNodeController {
this.mergePolicyProperties = mergePolicyProperties;
this.filterFields = filterFields;
primaryIndexNumOfTupleFields = primaryKeyTypes.length + (1 + ((metaType == null) ? 0 : 1));
- primaryIndexTypeTraits =
- createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, primaryKeyTypes, recordType, metaType);
+ primaryIndexTypeTraits = createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, primaryKeyTypes,
+ recordType, metaType);
primaryIndexComparatorFactories = createPrimaryIndexComparatorFactories(primaryKeyTypes);
primaryIndexBloomFilterKeyFields = createPrimaryIndexBloomFilterFields(primaryKeyTypes.length);
filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recordType);
@@ -435,8 +433,8 @@ public class TestNodeController {
primaryIndexComparatorFactories, primaryIndexBloomFilterKeyFields, mergePolicyFactory,
mergePolicyProperties, filterTypeTraits, filterCmpFactories, btreeFields, filterFields);
fileSplitProvider = getFileSplitProvider(dataset);
- primaryIndexSerdes =
- createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, primaryKeyTypes, recordType, metaType);
+ primaryIndexSerdes = createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, primaryKeyTypes, recordType,
+ metaType);
rDesc = new RecordDescriptor(primaryIndexSerdes, primaryIndexTypeTraits);
primaryIndexInsertFieldsPermutations = new int[primaryIndexNumOfTupleFields];
for (int i = 0; i < primaryIndexNumOfTupleFields; i++) {
@@ -459,8 +457,8 @@ public class TestNodeController {
ISerializerDeserializer<?>[] primaryKeySerdes = new ISerializerDeserializer<?>[primaryKeyTypes.length];
for (int i = 0; i < primaryKeyTypes.length; i++) {
primaryKeyTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(primaryKeyTypes[i]);
- primaryKeySerdes[i] =
- AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(primaryKeyTypes[i]);
+ primaryKeySerdes[i] = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(primaryKeyTypes[i]);
}
RecordDescriptor searcgRecDesc = new RecordDescriptor(primaryKeySerdes, primaryKeyTypeTraits);
IRecordDescriptorProvider rDescProvider = Mockito.mock(IRecordDescriptorProvider.class);
@@ -472,10 +470,10 @@ public class TestNodeController {
public RecordDescriptor getSearchOutputDesc(IAType[] keyTypes, ARecordType recordType, ARecordType metaType) {
int primaryIndexNumOfTupleFields = keyTypes.length + (1 + ((metaType == null) ? 0 : 1));
- ITypeTraits[] primaryIndexTypeTraits =
- createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, keyTypes, recordType, metaType);
- ISerializerDeserializer<?>[] primaryIndexSerdes =
- createPrimaryIndexSerdes(primaryIndexNumOfTupleFields, keyTypes, recordType, metaType);
+ ITypeTraits[] primaryIndexTypeTraits = createPrimaryIndexTypeTraits(primaryIndexNumOfTupleFields, keyTypes,
+ recordType, metaType);
+ ISerializerDeserializer<?>[] primaryIndexSerdes = createPrimaryIndexSerdes(primaryIndexNumOfTupleFields,
+ keyTypes, recordType, metaType);
return new RecordDescriptor(primaryIndexSerdes, primaryIndexTypeTraits);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
index 532cea9..3591509 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
@@ -28,7 +28,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.asterix.api.common.SessionConfig;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
+import org.apache.asterix.common.app.SessionConfig;
import org.apache.asterix.common.config.AsterixExternalProperties;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.event.schema.cluster.Cluster;
@@ -37,6 +38,7 @@ import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.statement.RunStatement;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.om.util.AsterixClusterProperties;
+import org.apache.asterix.translator.IStatementExecutor;
import org.junit.Assert;
import org.junit.Test;
@@ -67,7 +69,7 @@ public class QueryTranslatorTest {
when(mockCluster.getMasterNode()).thenReturn(mockMasterNode);
when(mockMasterNode.getClientIp()).thenReturn("127.0.0.1");
- QueryTranslator aqlTranslator = new QueryTranslator(statements, mockSessionConfig,
+ IStatementExecutor aqlTranslator = new DefaultStatementExecutorFactory(null).create(statements, mockSessionConfig,
new AqlCompilationProvider());
List<String> parameters = new ArrayList<String>();
parameters.add("examples/pregelix-example-jar-with-dependencies.jar");
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
index 84fb2b3..34dab28 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
@@ -27,6 +27,7 @@ import java.io.Reader;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.api.java.AsterixJavaClient;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.test.aql.TestExecutor;
@@ -59,9 +60,8 @@ public class DmlTest {
integrationUtil.init(true);
Reader loadReader = new BufferedReader(
new InputStreamReader(new FileInputStream(LOAD_FOR_ENLIST_FILE), "UTF-8"));
- AsterixJavaClient asterixLoad = new AsterixJavaClient(
- integrationUtil.getHyracksClientConnection(), loadReader, ERR,
- new AqlCompilationProvider());
+ AsterixJavaClient asterixLoad = new AsterixJavaClient(integrationUtil.getHyracksClientConnection(), loadReader,
+ ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory(null));
try {
asterixLoad.compile(true, false, false, false, false, true, false);
} catch (AsterixException e) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index b29c0b0..1f64868 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -30,6 +30,7 @@ import java.util.logging.Logger;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
import org.apache.asterix.api.java.AsterixJavaClient;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
@@ -171,7 +172,8 @@ public class OptimizerTest {
ILangCompilationProvider provider = queryFile.getName().endsWith("aql") ? aqlCompilationProvider
: sqlppCompilationProvider;
IHyracksClientConnection hcc = integrationUtil.getHyracksClientConnection();
- AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider);
+ AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider,
+ new DefaultStatementExecutorFactory(null));
try {
asterix.compile(true, false, false, true, true, false, false);
} catch (AsterixException e) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
index 545faed..3195c39 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
@@ -49,11 +49,15 @@ public class ExecutionTestUtil {
public static AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
public static List<ILibraryManager> setUp(boolean cleanup) throws Exception {
+ return setUp(cleanup, TEST_CONFIG_FILE_NAME);
+ }
+
+ public static List<ILibraryManager> setUp(boolean cleanup, String configFile) throws Exception {
System.out.println("Starting setup");
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Starting setup");
}
- System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, configFile);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("initializing pseudo cluster");
@@ -80,8 +84,8 @@ public class ExecutionTestUtil {
libraryManagers.add(AsterixAppContextInfo.getInstance().getLibraryManager());
// Adds library managers for NCs, one-per-NC.
for (NodeControllerService nc : integrationUtil.ncs) {
- IAsterixAppRuntimeContext runtimeCtx = (IAsterixAppRuntimeContext) nc.getApplicationContext()
- .getApplicationObject();
+ IAsterixAppRuntimeContext runtimeCtx =
+ (IAsterixAppRuntimeContext) nc.getApplicationContext().getApplicationObject();
libraryManagers.add(runtimeCtx.getLibraryManager());
}
return libraryManagers;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
index d3dc541..74c7091 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/HDFSCluster.java
@@ -25,16 +25,17 @@ import org.apache.asterix.external.dataset.adapter.GenericAdapter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
/**
* Manages a Mini (local VM) HDFS cluster with a configured number of datanodes.
- * @author ramangrover29
*/
public class HDFSCluster {
@@ -90,10 +91,12 @@ public class HDFSCluster {
Path destDir = new Path(HDFS_PATH);
dfs.mkdirs(destDir);
File srcDir = new File(localDataRoot + DATA_PATH);
- File[] listOfFiles = srcDir.listFiles();
- for (File srcFile : listOfFiles) {
- Path path = new Path(srcFile.getAbsolutePath());
- dfs.copyFromLocalFile(path, destDir);
+ if (srcDir.exists()) {
+ File[] listOfFiles = srcDir.listFiles();
+ for (File srcFile : listOfFiles) {
+ Path path = new Path(srcFile.getAbsolutePath());
+ dfs.copyFromLocalFile(path, destDir);
+ }
}
}
@@ -110,6 +113,7 @@ public class HDFSCluster {
cleanupLocal();
}
}
+
public static void main(String[] args) throws Exception {
HDFSCluster cluster = new HDFSCluster();
cluster.setup();
@@ -125,10 +129,10 @@ public class HDFSCluster {
String hdfsUrl = "hdfs://127.0.0.1:31888";
String hdfsPath = "/asterix/extrasmalltweets.txt";
conf.set("fs.default.name", hdfsUrl);
- conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
+ conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName());
conf.setClassLoader(GenericAdapter.class.getClassLoader());
conf.set("mapred.input.dir", hdfsPath);
- conf.set("mapred.input.format.class", "org.apache.hadoop.mapred.TextInputFormat");
+ conf.set("mapred.input.format.class", TextInputFormat.class.getName());
return conf;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ExtensionId.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ExtensionId.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ExtensionId.java
new file mode 100644
index 0000000..1cd0ce7
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/ExtensionId.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import java.io.Serializable;
+
+import org.apache.commons.lang3.ObjectUtils;
+
+public class ExtensionId implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final String name;
+ private final int version;
+
+ public ExtensionId(String name, int version) {
+ this.name = name;
+ this.version = version;
+ }
+
+ @Override
+ public int hashCode() {
+ return ObjectUtils.hashCodeMulti(getName().hashCode(), version);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ } else if (o instanceof ExtensionId) {
+ ExtensionId oExtensionId = (ExtensionId) o;
+ return version == oExtensionId.version && ObjectUtils.equals(name, oExtensionId.getName());
+ }
+ return false;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return name + ":" + version;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
index 100da63..046d5c1 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IAsterixAppRuntimeContext.java
@@ -74,7 +74,7 @@ public interface IAsterixAppRuntimeContext {
public double getBloomFilterFalsePositiveRate();
- public Object getFeedManager();
+ public Object getActiveManager();
public IRemoteRecoveryManager getRemoteRecoveryManager();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java
new file mode 100644
index 0000000..9551935
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IExtension.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import java.util.List;
+
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+/**
+ * an interface for an extension that provides a mechanism to override system behaviour
+ * or add new features
+ */
+public interface IExtension {
+
+ enum ExtensionKind {
+ /**
+ * Extends Query translator
+ */
+ STATEMENT_EXECUTOR,
+ /**
+ * Extends Metadata
+ */
+ METADATA,
+ /**
+ * Extends Language Syntax and Algebric Operations
+ */
+ LANG
+ }
+
+ /**
+ * @return Unique Id for the extension (used for lookup operations)
+ */
+ ExtensionId getId();
+
+ /**
+ * Configures the extension with configuration parameters.
+ * This method is called on system boot
+ *
+ * @param args
+ */
+ void configure(List<Pair<String, String>> args);
+
+ /**
+ * @return The extension point implemented by this extension
+ */
+ ExtensionKind getExtensionKind();
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/app/SessionConfig.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/app/SessionConfig.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/app/SessionConfig.java
new file mode 100644
index 0000000..f4e3be2
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/app/SessionConfig.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.app;
+
+import java.io.PrintWriter;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
+
+/**
+ * SessionConfig captures several different parameters for controlling
+ * the execution of an APIFramework call.
+ * <li>It specifies how the execution will proceed (for instance,
+ * whether to optimize, or whether to execute at all).
+ * <li>It allows you specify where the primary execution output will
+ * be sent.
+ * <li>It also allows you to request additional output for optional
+ * out-of-band data about the execution (query plan, etc).
+ * <li>It allows you to specify the output format for the primary
+ * execution output - LOSSLESS_JSON, CSV, etc.
+ * <li>It allows you to specify output format-specific parameters.
+ */
+
+public class SessionConfig {
+ /**
+ * Used to specify the output format for the primary execution.
+ */
+ public enum OutputFormat {
+ ADM,
+ CSV,
+ CLEAN_JSON,
+ LOSSLESS_JSON
+ };
+
+ /**
+ * Produce out-of-band output for Hyracks Job.
+ */
+ public static final String OOB_HYRACKS_JOB = "oob-hyracks-job";
+
+ /**
+ * Produce out-of-band output for Expression Tree.
+ */
+ public static final String OOB_EXPR_TREE = "oob-expr-tree";
+
+ /**
+ * Produce out-of-band output for Rewritten Expression Tree.
+ */
+ public static final String OOB_REWRITTEN_EXPR_TREE = "oob-rewritten-expr-tree";
+
+ /**
+ * Produce out-of-band output for Logical Plan.
+ */
+ public static final String OOB_LOGICAL_PLAN = "oob-logical-plan";
+
+ /**
+ * Produce out-of-band output for Optimized Logical Plan.
+ */
+ public static final String OOB_OPTIMIZED_LOGICAL_PLAN = "oob-optimized-logical-plan";
+
+ /**
+ * Format flag: print only physical ops (for optimizer tests).
+ */
+ public static final String FORMAT_ONLY_PHYSICAL_OPS = "format-only-physical-ops";
+
+ /**
+ * Format flag: wrap out-of-band data in HTML.
+ */
+ public static final String FORMAT_HTML = "format-html";
+
+ /**
+ * Format flag: print CSV header line.
+ */
+ public static final String FORMAT_CSV_HEADER = "format-csv-header";
+
+ /**
+ * Format flag: wrap results in outer array brackets (JSON or ADM).
+ */
+ public static final String FORMAT_WRAPPER_ARRAY = "format-wrapper-array";
+
+ /**
+ * Format flag: indent JSON results.
+ */
+ public static final String FORMAT_INDENT_JSON = "indent-json";
+
+ /**
+ * Format flag: quote records in the results array.
+ */
+ public static final String FORMAT_QUOTE_RECORD = "quote-record";
+
+ @FunctionalInterface
+ public interface ResultDecorator {
+ AlgebricksAppendable append(AlgebricksAppendable app) throws AlgebricksException;
+ }
+
+ // Standard execution flags.
+ private final boolean executeQuery;
+ private final boolean generateJobSpec;
+ private final boolean optimize;
+
+ // Output path for primary execution.
+ private final PrintWriter out;
+
+ // Output format.
+ private final OutputFormat fmt;
+
+ private final ResultDecorator preResultDecorator;
+ private final ResultDecorator postResultDecorator;
+
+ // Flags.
+ private final Map<String, Boolean> flags;
+
+ /**
+ * Create a SessionConfig object with all default values:
+ * - All format flags set to "false".
+ * - All out-of-band outputs set to "null".
+ * - "Optimize" set to "true".
+ * - "Execute Query" set to "true".
+ * - "Generate Job Spec" set to "true".
+ *
+ * @param out
+ * PrintWriter for execution output.
+ * @param fmt
+ * Output format for execution output.
+ */
+ public SessionConfig(PrintWriter out, OutputFormat fmt) {
+ this(out, fmt, null, null, true, true, true);
+ }
+
+ public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
+ ResultDecorator postResultDecorator) {
+ this(out, fmt, preResultDecorator, postResultDecorator, true, true, true);
+ }
+
+ public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery,
+ boolean generateJobSpec) {
+ this(out, fmt, null, null, optimize, executeQuery, generateJobSpec);
+ }
+
+ /**
+ * Create a SessionConfig object with all optional values set to defaults:
+ * - All format flags set to "false".
+ * - All out-of-band outputs set to "false".
+ *
+ * @param out
+ * PrintWriter for execution output.
+ * @param fmt
+ * Output format for execution output.
+ * @param optimize
+ * Whether to optimize the execution.
+ * @param executeQuery
+ * Whether to execute the query or not.
+ * @param generateJobSpec
+ * Whether to generate the Hyracks job specification (if
+ * false, job cannot be executed).
+ */
+ public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
+ ResultDecorator postResultDecorator, boolean optimize, boolean executeQuery, boolean generateJobSpec) {
+ this.out = out;
+ this.fmt = fmt;
+ this.preResultDecorator = preResultDecorator;
+ this.postResultDecorator = postResultDecorator;
+ this.optimize = optimize;
+ this.executeQuery = executeQuery;
+ this.generateJobSpec = generateJobSpec;
+ this.flags = new HashMap<>();
+ }
+
+ /**
+ * Retrieve the PrintWriter to produce output to.
+ */
+ public PrintWriter out() {
+ return this.out;
+ }
+
+ /**
+ * Retrieve the OutputFormat for this execution.
+ */
+ public OutputFormat fmt() {
+ return this.fmt;
+ }
+
+ public AlgebricksAppendable resultPrefix(AlgebricksAppendable app) throws AlgebricksException {
+ return this.preResultDecorator != null ? this.preResultDecorator.append(app) : app;
+ };
+
+ public AlgebricksAppendable resultPostfix(AlgebricksAppendable app) throws AlgebricksException {
+ return this.postResultDecorator != null ? this.postResultDecorator.append(app) : app;
+ };
+
+ /**
+ * Retrieve the value of the "execute query" flag.
+ */
+ public boolean isExecuteQuery() {
+ return executeQuery;
+ }
+
+ /**
+ * Retrieve the value of the "optimize" flag.
+ */
+ public boolean isOptimize() {
+ return optimize;
+ }
+
+ /**
+ * Retrieve the value of the "generate job spec" flag.
+ */
+ public boolean isGenerateJobSpec() {
+ return generateJobSpec;
+ }
+
+ /**
+ * Specify all out-of-band settings at once. For convenience of older code.
+ */
+ public void setOOBData(boolean exprTree, boolean rewrittenExprTree, boolean logicalPlan,
+ boolean optimizedLogicalPlan, boolean hyracksJob) {
+ this.set(OOB_EXPR_TREE, exprTree);
+ this.set(OOB_REWRITTEN_EXPR_TREE, rewrittenExprTree);
+ this.set(OOB_LOGICAL_PLAN, logicalPlan);
+ this.set(OOB_OPTIMIZED_LOGICAL_PLAN, optimizedLogicalPlan);
+ this.set(OOB_HYRACKS_JOB, hyracksJob);
+ }
+
+ /**
+ * Specify a flag.
+ *
+ * @param flag
+ * One of the OOB_ or FORMAT_ constants from this class.
+ * @param value
+ * Value for the flag (all flags default to "false").
+ */
+ public void set(String flag, boolean value) {
+ flags.put(flag, Boolean.valueOf(value));
+ }
+
+ /**
+ * Retrieve the setting of a format-specific flag.
+ *
+ * @param flag
+ * One of the FORMAT_ constants from this class.
+ * @returns true or false (all flags default to "false").
+ */
+ public boolean is(String flag) {
+ Boolean value = flags.get(flag);
+ return value == null ? false : value.booleanValue();
+ }
+}
[14/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
index d5a6f2b..f483d70 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
@@ -21,6 +21,8 @@ package org.apache.asterix.optimizer.rules.am;
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.utils.DatasetUtils;
@@ -38,6 +40,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
@@ -59,24 +62,24 @@ public class OptimizableOperatorSubTree {
NO_DATASOURCE
}
- public ILogicalOperator root = null;
- public Mutable<ILogicalOperator> rootRef = null;
- public final List<Mutable<ILogicalOperator>> assignsAndUnnestsRefs = new ArrayList<Mutable<ILogicalOperator>>();
- public final List<AbstractLogicalOperator> assignsAndUnnests = new ArrayList<AbstractLogicalOperator>();
- public Mutable<ILogicalOperator> dataSourceRef = null;
- public DataSourceType dataSourceType = DataSourceType.NO_DATASOURCE;
+ private ILogicalOperator root = null;
+ private Mutable<ILogicalOperator> rootRef = null;
+ private final List<Mutable<ILogicalOperator>> assignsAndUnnestsRefs = new ArrayList<>();
+ private final List<AbstractLogicalOperator> assignsAndUnnests = new ArrayList<>();
+ private Mutable<ILogicalOperator> dataSourceRef = null;
+ private DataSourceType dataSourceType = DataSourceType.NO_DATASOURCE;
// Dataset and type metadata. Set in setDatasetAndTypeMetadata().
- public Dataset dataset = null;
- public ARecordType recordType = null;
- public ARecordType metaRecordType = null;
+ private Dataset dataset = null;
+ private ARecordType recordType = null;
+ private ARecordType metaRecordType = null;
// Additional datasources can exist if IntroduceJoinAccessMethodRule has been applied.
// (E.g. There are index-nested-loop-joins in the plan.)
- public List<Mutable<ILogicalOperator>> ixJoinOuterAdditionalDataSourceRefs = null;
- public List<DataSourceType> ixJoinOuterAdditionalDataSourceTypes = null;
- public List<Dataset> ixJoinOuterAdditionalDatasets = null;
- public List<ARecordType> ixJoinOuterAdditionalRecordTypes = null;
+ private List<Mutable<ILogicalOperator>> ixJoinOuterAdditionalDataSourceRefs = null;
+ private List<DataSourceType> ixJoinOuterAdditionalDataSourceTypes = null;
+ private List<Dataset> ixJoinOuterAdditionalDatasets = null;
+ private List<ARecordType> ixJoinOuterAdditionalRecordTypes = null;
public boolean initFromSubTree(Mutable<ILogicalOperator> subTreeOpRef) throws AlgebricksException {
reset();
@@ -102,12 +105,12 @@ public class OptimizableOperatorSubTree {
if (!OperatorPropertiesUtil.isMovable(subTreeOp)) {
return false;
} else {
- assignsAndUnnestsRefs.add(subTreeOpRef);
- assignsAndUnnests.add(subTreeOp);
+ getAssignsAndUnnestsRefs().add(subTreeOpRef);
+ getAssignsAndUnnests().add(subTreeOp);
}
subTreeOpRef = subTreeOp.getInputs().get(0);
subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
- };
+ }
} while (subTreeOp.getOperatorTag() == LogicalOperatorTag.SELECT);
// Match data source (datasource scan or primary index search).
@@ -118,12 +121,12 @@ public class OptimizableOperatorSubTree {
AbstractLogicalOperator subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
if (subTreeOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
- dataSourceType = DataSourceType.DATASOURCE_SCAN;
- dataSourceRef = subTreeOpRef;
+ setDataSourceType(DataSourceType.DATASOURCE_SCAN);
+ setDataSourceRef(subTreeOpRef);
return true;
} else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
- dataSourceType = DataSourceType.COLLECTION_SCAN;
- dataSourceRef = subTreeOpRef;
+ setDataSourceType(DataSourceType.COLLECTION_SCAN);
+ setDataSourceRef(subTreeOpRef);
return true;
} else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
// There can be multiple unnest-map or datasource-scan operators
@@ -141,40 +144,40 @@ public class OptimizableOperatorSubTree {
AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
jobGenParams.readFromFuncArgs(f.getArguments());
if (jobGenParams.isPrimaryIndex()) {
- if (dataSourceRef == null) {
- dataSourceRef = subTreeOpRef;
- dataSourceType = DataSourceType.PRIMARY_INDEX_LOOKUP;
+ if (getDataSourceRef() == null) {
+ setDataSourceRef(subTreeOpRef);
+ setDataSourceType(DataSourceType.PRIMARY_INDEX_LOOKUP);
} else {
// One datasource already exists. This is an additional datasource.
initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
- ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.PRIMARY_INDEX_LOOKUP);
- ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+ getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.PRIMARY_INDEX_LOOKUP);
+ getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
}
dataSourceFound = true;
}
} else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.EXTERNAL_LOOKUP)) {
// External lookup case
- if (dataSourceRef == null) {
- dataSourceRef = subTreeOpRef;
- dataSourceType = DataSourceType.EXTERNAL_SCAN;
+ if (getDataSourceRef() == null) {
+ setDataSourceRef(subTreeOpRef);
+ setDataSourceType(DataSourceType.EXTERNAL_SCAN);
} else {
// One datasource already exists. This is an additional datasource.
initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
- ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.EXTERNAL_SCAN);
- ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+ getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.EXTERNAL_SCAN);
+ getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
}
dataSourceFound = true;
}
}
} else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
- ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.DATASOURCE_SCAN);
- ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+ getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.DATASOURCE_SCAN);
+ getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
dataSourceFound = true;
} else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
- ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.COLLECTION_SCAN);
- ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+ getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.COLLECTION_SCAN);
+ getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
}
// Traverse the subtree while there are operators in the path.
@@ -208,14 +211,14 @@ public class OptimizableOperatorSubTree {
List<Mutable<ILogicalOperator>> sourceOpRefs = new ArrayList<Mutable<ILogicalOperator>>();
List<DataSourceType> dsTypes = new ArrayList<DataSourceType>();
- sourceOpRefs.add(dataSourceRef);
- dsTypes.add(dataSourceType);
+ sourceOpRefs.add(getDataSourceRef());
+ dsTypes.add(getDataSourceType());
// If there are multiple datasources in the subtree, we need to find the dataset for these.
- if (ixJoinOuterAdditionalDataSourceRefs != null) {
- for (int i = 0; i < ixJoinOuterAdditionalDataSourceRefs.size(); i++) {
- sourceOpRefs.add(ixJoinOuterAdditionalDataSourceRefs.get(i));
- dsTypes.add(ixJoinOuterAdditionalDataSourceTypes.get(i));
+ if (getIxJoinOuterAdditionalDataSourceRefs() != null) {
+ for (int i = 0; i < getIxJoinOuterAdditionalDataSourceRefs().size(); i++) {
+ sourceOpRefs.add(getIxJoinOuterAdditionalDataSourceRefs().get(i));
+ dsTypes.add(getIxJoinOuterAdditionalDataSourceTypes().get(i));
}
}
@@ -223,6 +226,14 @@ public class OptimizableOperatorSubTree {
switch (dsTypes.get(i)) {
case DATASOURCE_SCAN:
DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) sourceOpRefs.get(i).getValue();
+ IDataSource<?> datasource = dataSourceScan.getDataSource();
+ if (datasource instanceof AqlDataSource) {
+ AqlDataSourceType dsType = ((AqlDataSource) datasource).getDatasourceType();
+ if (dsType != AqlDataSourceType.INTERNAL_DATASET
+ && dsType != AqlDataSourceType.EXTERNAL_DATASET) {
+ return false;
+ }
+ }
Pair<String, String> datasetInfo = AnalysisUtil.getDatasetInfo(dataSourceScan);
dataverseName = datasetInfo.first;
datasetName = datasetInfo.second;
@@ -244,8 +255,8 @@ public class OptimizableOperatorSubTree {
break;
case COLLECTION_SCAN:
if (i != 0) {
- ixJoinOuterAdditionalDatasets.add(null);
- ixJoinOuterAdditionalRecordTypes.add(null);
+ getIxJoinOuterAdditionalDatasets().add(null);
+ getIxJoinOuterAdditionalRecordTypes().add(null);
}
continue;
case NO_DATASOURCE:
@@ -266,24 +277,24 @@ public class OptimizableOperatorSubTree {
if (i == 0) {
return false;
} else {
- ixJoinOuterAdditionalDatasets.add(null);
- ixJoinOuterAdditionalRecordTypes.add(null);
+ getIxJoinOuterAdditionalDatasets().add(null);
+ getIxJoinOuterAdditionalRecordTypes().add(null);
}
}
rType = (ARecordType) itemType;
// Get the meta record type for that dataset.
- IAType metaItemType = metadataProvider.findType(ds.getMetaItemTypeDataverseName(),
- ds.getMetaItemTypeName());
+ IAType metaItemType =
+ metadataProvider.findType(ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
// First index is always the primary datasource in this subtree.
if (i == 0) {
- dataset = ds;
- recordType = rType;
- metaRecordType = (ARecordType) metaItemType;
+ setDataset(ds);
+ setRecordType(rType);
+ setMetaRecordType((ARecordType) metaItemType);
} else {
- ixJoinOuterAdditionalDatasets.add(ds);
- ixJoinOuterAdditionalRecordTypes.add(rType);
+ getIxJoinOuterAdditionalDatasets().add(ds);
+ getIxJoinOuterAdditionalRecordTypes().add(rType);
}
dataverseName = null;
@@ -296,14 +307,14 @@ public class OptimizableOperatorSubTree {
}
public boolean hasDataSource() {
- return dataSourceType != DataSourceType.NO_DATASOURCE;
+ return getDataSourceType() != DataSourceType.NO_DATASOURCE;
}
public boolean hasIxJoinOuterAdditionalDataSource() {
boolean dataSourceFound = false;
- if (ixJoinOuterAdditionalDataSourceTypes != null) {
- for (int i = 0; i < ixJoinOuterAdditionalDataSourceTypes.size(); i++) {
- if (ixJoinOuterAdditionalDataSourceTypes.get(i) != DataSourceType.NO_DATASOURCE) {
+ if (getIxJoinOuterAdditionalDataSourceTypes() != null) {
+ for (int i = 0; i < getIxJoinOuterAdditionalDataSourceTypes().size(); i++) {
+ if (getIxJoinOuterAdditionalDataSourceTypes().get(i) != DataSourceType.NO_DATASOURCE) {
dataSourceFound = true;
break;
}
@@ -313,13 +324,13 @@ public class OptimizableOperatorSubTree {
}
public boolean hasDataSourceScan() {
- return dataSourceType == DataSourceType.DATASOURCE_SCAN;
+ return getDataSourceType() == DataSourceType.DATASOURCE_SCAN;
}
public boolean hasIxJoinOuterAdditionalDataSourceScan() {
- if (ixJoinOuterAdditionalDataSourceTypes != null) {
- for (int i = 0; i < ixJoinOuterAdditionalDataSourceTypes.size(); i++) {
- if (ixJoinOuterAdditionalDataSourceTypes.get(i) == DataSourceType.DATASOURCE_SCAN) {
+ if (getIxJoinOuterAdditionalDataSourceTypes() != null) {
+ for (int i = 0; i < getIxJoinOuterAdditionalDataSourceTypes().size(); i++) {
+ if (getIxJoinOuterAdditionalDataSourceTypes().get(i) == DataSourceType.DATASOURCE_SCAN) {
return true;
}
}
@@ -328,33 +339,33 @@ public class OptimizableOperatorSubTree {
}
public void reset() {
- root = null;
- rootRef = null;
- assignsAndUnnestsRefs.clear();
- assignsAndUnnests.clear();
- dataSourceRef = null;
- dataSourceType = DataSourceType.NO_DATASOURCE;
- ixJoinOuterAdditionalDataSourceRefs = null;
- ixJoinOuterAdditionalDataSourceTypes = null;
- dataset = null;
- ixJoinOuterAdditionalDatasets = null;
- recordType = null;
- ixJoinOuterAdditionalRecordTypes = null;
+ setRoot(null);
+ setRootRef(null);
+ getAssignsAndUnnestsRefs().clear();
+ getAssignsAndUnnests().clear();
+ setDataSourceRef(null);
+ setDataSourceType(DataSourceType.NO_DATASOURCE);
+ setIxJoinOuterAdditionalDataSourceRefs(null);
+ setIxJoinOuterAdditionalDataSourceTypes(null);
+ setDataset(null);
+ setIxJoinOuterAdditionalDatasets(null);
+ setRecordType(null);
+ setIxJoinOuterAdditionalRecordTypes(null);
}
public void getPrimaryKeyVars(List<LogicalVariable> target) throws AlgebricksException {
- switch (dataSourceType) {
+ switch (getDataSourceType()) {
case DATASOURCE_SCAN:
- DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) dataSourceRef.getValue();
- int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
+ DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) getDataSourceRef().getValue();
+ int numPrimaryKeys = DatasetUtils.getPartitioningKeys(getDataset()).size();
for (int i = 0; i < numPrimaryKeys; i++) {
target.add(dataSourceScan.getVariables().get(i));
}
break;
case PRIMARY_INDEX_LOOKUP:
- UnnestMapOperator unnestMapOp = (UnnestMapOperator) dataSourceRef.getValue();
+ UnnestMapOperator unnestMapOp = (UnnestMapOperator) getDataSourceRef().getValue();
List<LogicalVariable> primaryKeys = null;
- primaryKeys = AccessMethodUtils.getPrimaryKeyVarsFromPrimaryUnnestMap(dataset, unnestMapOp);
+ primaryKeys = AccessMethodUtils.getPrimaryKeyVarsFromPrimaryUnnestMap(getDataset(), unnestMapOp);
target.addAll(primaryKeys);
break;
case NO_DATASOURCE:
@@ -364,14 +375,14 @@ public class OptimizableOperatorSubTree {
}
public List<LogicalVariable> getDataSourceVariables() throws AlgebricksException {
- switch (dataSourceType) {
+ switch (getDataSourceType()) {
case DATASOURCE_SCAN:
case EXTERNAL_SCAN:
case PRIMARY_INDEX_LOOKUP:
- AbstractScanOperator scanOp = (AbstractScanOperator) dataSourceRef.getValue();
+ AbstractScanOperator scanOp = (AbstractScanOperator) getDataSourceRef().getValue();
return scanOp.getVariables();
case COLLECTION_SCAN:
- return new ArrayList<LogicalVariable>();
+ return new ArrayList<>();
case NO_DATASOURCE:
default:
throw new AlgebricksException("The subtree does not have any data source.");
@@ -379,16 +390,16 @@ public class OptimizableOperatorSubTree {
}
public List<LogicalVariable> getIxJoinOuterAdditionalDataSourceVariables(int idx) throws AlgebricksException {
- if (ixJoinOuterAdditionalDataSourceRefs != null && ixJoinOuterAdditionalDataSourceRefs.size() > idx) {
- switch (ixJoinOuterAdditionalDataSourceTypes.get(idx)) {
+ if (getIxJoinOuterAdditionalDataSourceRefs() != null && getIxJoinOuterAdditionalDataSourceRefs().size() > idx) {
+ switch (getIxJoinOuterAdditionalDataSourceTypes().get(idx)) {
case DATASOURCE_SCAN:
case EXTERNAL_SCAN:
case PRIMARY_INDEX_LOOKUP:
- AbstractScanOperator scanOp = (AbstractScanOperator) ixJoinOuterAdditionalDataSourceRefs.get(idx)
- .getValue();
+ AbstractScanOperator scanOp =
+ (AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs().get(idx).getValue();
return scanOp.getVariables();
case COLLECTION_SCAN:
- return new ArrayList<LogicalVariable>();
+ return new ArrayList<>();
case NO_DATASOURCE:
default:
throw new AlgebricksException("The subtree does not have any additional data sources.");
@@ -399,12 +410,109 @@ public class OptimizableOperatorSubTree {
}
public void initializeIxJoinOuterAddtionalDataSourcesIfEmpty() {
- if (ixJoinOuterAdditionalDataSourceRefs == null) {
- ixJoinOuterAdditionalDataSourceRefs = new ArrayList<Mutable<ILogicalOperator>>();
- ixJoinOuterAdditionalDataSourceTypes = new ArrayList<DataSourceType>();
- ixJoinOuterAdditionalDatasets = new ArrayList<Dataset>();
- ixJoinOuterAdditionalRecordTypes = new ArrayList<ARecordType>();
+ if (getIxJoinOuterAdditionalDataSourceRefs() == null) {
+ setIxJoinOuterAdditionalDataSourceRefs(new ArrayList<Mutable<ILogicalOperator>>());
+ setIxJoinOuterAdditionalDataSourceTypes(new ArrayList<DataSourceType>());
+ setIxJoinOuterAdditionalDatasets(new ArrayList<Dataset>());
+ setIxJoinOuterAdditionalRecordTypes(new ArrayList<ARecordType>());
}
}
+ public ILogicalOperator getRoot() {
+ return root;
+ }
+
+ public void setRoot(ILogicalOperator root) {
+ this.root = root;
+ }
+
+ public Mutable<ILogicalOperator> getRootRef() {
+ return rootRef;
+ }
+
+ public void setRootRef(Mutable<ILogicalOperator> rootRef) {
+ this.rootRef = rootRef;
+ }
+
+ public List<Mutable<ILogicalOperator>> getAssignsAndUnnestsRefs() {
+ return assignsAndUnnestsRefs;
+ }
+
+ public List<AbstractLogicalOperator> getAssignsAndUnnests() {
+ return assignsAndUnnests;
+ }
+
+ public Mutable<ILogicalOperator> getDataSourceRef() {
+ return dataSourceRef;
+ }
+
+ public void setDataSourceRef(Mutable<ILogicalOperator> dataSourceRef) {
+ this.dataSourceRef = dataSourceRef;
+ }
+
+ public DataSourceType getDataSourceType() {
+ return dataSourceType;
+ }
+
+ public void setDataSourceType(DataSourceType dataSourceType) {
+ this.dataSourceType = dataSourceType;
+ }
+
+ public Dataset getDataset() {
+ return dataset;
+ }
+
+ public void setDataset(Dataset dataset) {
+ this.dataset = dataset;
+ }
+
+ public ARecordType getRecordType() {
+ return recordType;
+ }
+
+ public void setRecordType(ARecordType recordType) {
+ this.recordType = recordType;
+ }
+
+ public ARecordType getMetaRecordType() {
+ return metaRecordType;
+ }
+
+ public void setMetaRecordType(ARecordType metaRecordType) {
+ this.metaRecordType = metaRecordType;
+ }
+
+ public List<Mutable<ILogicalOperator>> getIxJoinOuterAdditionalDataSourceRefs() {
+ return ixJoinOuterAdditionalDataSourceRefs;
+ }
+
+ public void setIxJoinOuterAdditionalDataSourceRefs(
+ List<Mutable<ILogicalOperator>> ixJoinOuterAdditionalDataSourceRefs) {
+ this.ixJoinOuterAdditionalDataSourceRefs = ixJoinOuterAdditionalDataSourceRefs;
+ }
+
+ public List<DataSourceType> getIxJoinOuterAdditionalDataSourceTypes() {
+ return ixJoinOuterAdditionalDataSourceTypes;
+ }
+
+ public void setIxJoinOuterAdditionalDataSourceTypes(List<DataSourceType> ixJoinOuterAdditionalDataSourceTypes) {
+ this.ixJoinOuterAdditionalDataSourceTypes = ixJoinOuterAdditionalDataSourceTypes;
+ }
+
+ public List<Dataset> getIxJoinOuterAdditionalDatasets() {
+ return ixJoinOuterAdditionalDatasets;
+ }
+
+ public void setIxJoinOuterAdditionalDatasets(List<Dataset> ixJoinOuterAdditionalDatasets) {
+ this.ixJoinOuterAdditionalDatasets = ixJoinOuterAdditionalDatasets;
+ }
+
+ public List<ARecordType> getIxJoinOuterAdditionalRecordTypes() {
+ return ixJoinOuterAdditionalRecordTypes;
+ }
+
+ public void setIxJoinOuterAdditionalRecordTypes(List<ARecordType> ixJoinOuterAdditionalRecordTypes) {
+ this.ixJoinOuterAdditionalRecordTypes = ixJoinOuterAdditionalRecordTypes;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index eba2d33..c3c162e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -50,10 +50,8 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBina
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
/**
@@ -61,13 +59,13 @@ import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
*/
public class RTreeAccessMethod implements IAccessMethod {
- private static List<FunctionIdentifier> funcIdents = new ArrayList<FunctionIdentifier>();
+ private static List<FunctionIdentifier> funcIdents = new ArrayList<>();
static {
funcIdents.add(AsterixBuiltinFunctions.SPATIAL_INTERSECT);
}
- public static RTreeAccessMethod INSTANCE = new RTreeAccessMethod();
+ public static final RTreeAccessMethod INSTANCE = new RTreeAccessMethod();
@Override
public List<FunctionIdentifier> getOptimizableFunctions() {
@@ -107,7 +105,7 @@ public class RTreeAccessMethod implements IAccessMethod {
return false;
}
// Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
- subTree.dataSourceRef.setValue(primaryIndexUnnestOp);
+ subTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
return true;
}
@@ -125,16 +123,17 @@ public class RTreeAccessMethod implements IAccessMethod {
OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin,
boolean hasGroupBy) throws AlgebricksException {
- // Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
+ // Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left
+ // side).
Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex);
- OptimizableOperatorSubTree indexSubTree = null;
- OptimizableOperatorSubTree probeSubTree = null;
+ OptimizableOperatorSubTree indexSubTree;
+ OptimizableOperatorSubTree probeSubTree;
// We assume that the left subtree is the outer branch and the right subtree is the inner branch.
// This assumption holds true since we only use an index from the right subtree.
// The following is just a sanity check.
if (rightSubTree.hasDataSourceScan()
- && dataset.getDatasetName().equals(rightSubTree.dataset.getDatasetName())) {
+ && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
indexSubTree = rightSubTree;
probeSubTree = leftSubTree;
} else {
@@ -143,8 +142,8 @@ public class RTreeAccessMethod implements IAccessMethod {
LogicalVariable newNullPlaceHolderVar = null;
if (isLeftOuterJoin) {
- //get a new null place holder variable that is the first field variable of the primary key
- //from the indexSubTree's datasourceScanOp
+ // get a new null place holder variable that is the first field variable of the primary key
+ // from the indexSubTree's datasourceScanOp
newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
}
@@ -156,15 +155,15 @@ public class RTreeAccessMethod implements IAccessMethod {
}
if (isLeftOuterJoin && hasGroupBy) {
- //reset the null place holder variable
+ // reset the null place holder variable
AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
}
- indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp);
+ indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
// Change join into a select with the same condition.
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
SelectOperator topSelect = new SelectOperator(joinOp.getCondition(), isLeftOuterJoin, newNullPlaceHolderVar);
- topSelect.getInputs().add(indexSubTree.rootRef);
+ topSelect.getInputs().add(indexSubTree.getRootRef());
topSelect.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(topSelect);
// Replace the original join with the new subtree rooted at the select op.
@@ -175,12 +174,12 @@ public class RTreeAccessMethod implements IAccessMethod {
private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
- throws AlgebricksException {
+ throws AlgebricksException {
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
- Dataset dataset = indexSubTree.dataset;
- ARecordType recordType = indexSubTree.recordType;
- ARecordType metaRecordType = indexSubTree.metaRecordType;
+ Dataset dataset = indexSubTree.getDataset();
+ ARecordType recordType = indexSubTree.getRecordType();
+ ARecordType metaRecordType = indexSubTree.getMetaRecordType();
int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx);
Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx),
@@ -194,16 +193,18 @@ public class RTreeAccessMethod implements IAccessMethod {
int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numSecondaryKeys = numDimensions * 2;
// we made sure indexSubTree has datasource scan
- AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.dataSourceRef.getValue();
+ AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef()
+ .getValue();
RTreeJobGenParams jobGenParams = new RTreeJobGenParams(chosenIndex.getIndexName(), IndexType.RTREE,
dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
// A spatial object is serialized in the constant of the func expr we are optimizing.
// The R-Tree expects as input an MBR represented with 1 field per dimension.
- // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the R-Tree expects them).
+ // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the
+ // R-Tree expects them).
// List of variables for the assign.
- ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
+ ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
// List of expressions for the assign.
- ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
+ ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr,
indexSubTree, probeSubTree);
ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
@@ -237,23 +238,18 @@ public class RTreeAccessMethod implements IAccessMethod {
assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
} else {
// We are optimizing a join, place the assign op top of the probe subtree.
- assignSearchKeys.getInputs().add(probeSubTree.rootRef);
+ assignSearchKeys.getInputs().add(probeSubTree.getRootRef());
}
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput, retainNull);
// Generate the rest of the upstream plan which feeds the search results into the primary index.
- if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
- UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp,
- dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull);
- return externalDataAccessOp;
- } else {
- AbstractUnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
- dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, false,
- false);
- return primaryIndexUnnestOp;
- }
+ return dataset.getDatasetType() == DatasetType.EXTERNAL
+ ? AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType,
+ secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull)
+ : AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType,
+ secondaryIndexUnnestOp, context, true, retainInput, false, false);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
index 9030da9..bf7b975 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
@@ -24,18 +24,18 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.statement.DatasetDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
import org.apache.asterix.metadata.dataset.hints.DatasetHints;
import org.apache.asterix.metadata.entities.AsterixBuiltinTypeMap;
import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.util.AsterixAppContextInfo;
import org.apache.asterix.om.util.AsterixClusterProperties;
@@ -149,7 +149,7 @@ public abstract class AbstractLangTranslator {
break;
case Statement.Kind.DATASET_DROP:
- DropStatement dropStmt = (DropStatement) stmt;
+ DropDatasetStatement dropStmt = (DropDatasetStatement) stmt;
if (dropStmt.getDataverseName() != null) {
dataverse = dropStmt.getDataverseName().getValue();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
index 79342c9..5c09978 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -30,8 +30,8 @@ import org.apache.asterix.lang.aql.expression.UnionExpr;
import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
import org.apache.asterix.lang.common.base.Clause;
import org.apache.asterix.lang.common.base.Expression;
-import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
index 334436a..05bd343 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
@@ -81,7 +81,7 @@ import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
@@ -1218,7 +1218,7 @@ public class AqlPlusExpressionToPlanTranslator extends AbstractLangTranslator
}
@Override
- public Pair<ILogicalOperator, LogicalVariable> visit(DropStatement del, Mutable<ILogicalOperator> arg)
+ public Pair<ILogicalOperator, LogicalVariable> visit(DropDatasetStatement del, Mutable<ILogicalOperator> arg)
throws AsterixException {
// TODO Auto-generated method stub
return null;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
new file mode 100644
index 0000000..3c24cbc
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.rmi.RemoteException;
+
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.json.JSONException;
+
+/**
+ * An interface that takes care of executing a list of statements that are submitted through an Asterix API
+ */
+public interface IStatementExecutor {
+
+ /**
+ * Specifies result delivery of executed statements
+ */
+ public enum ResultDelivery {
+ /**
+ * Wait for results to be read
+ */
+ SYNC,
+ /**
+ * Flush out result handle beofre waiting for the result
+ */
+ ASYNC,
+ /**
+ * Return result handle and don't wait for the result
+ */
+ ASYNC_DEFERRED
+ }
+
+ public static class Stats {
+ private long count;
+ private long size;
+
+ public long getCount() {
+ return count;
+ }
+
+ public void setCount(long count) {
+ this.count = count;
+ }
+
+ public long getSize() {
+ return size;
+ }
+
+ public void setSize(long size) {
+ this.size = size;
+ }
+
+ }
+
+ /**
+ * Compiles and execute a list of statements.
+ *
+ * @param hcc
+ * A Hyracks client connection that is used to submit a jobspec to Hyracks.
+ * @param hdc
+ * A Hyracks dataset client object that is used to read the results.
+ * @param resultDelivery
+ * The {@code ResultDelivery} kind required for queries in the list of statements
+ * @throws Exception
+ */
+ void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery)
+ throws Exception;
+
+ /**
+ * Compiles and execute a list of statements.
+ *
+ * @param hcc
+ * A Hyracks client connection that is used to submit a jobspec to Hyracks.
+ * @param hdc
+ * A Hyracks dataset client object that is used to read the results.
+ * @param resultDelivery
+ * The {@code ResultDelivery} kind required for queries in the list of statements
+ * @param stats
+ * a reference to write the stats of executed queries
+ * @throws Exception
+ */
+ void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery,
+ Stats stats) throws Exception;
+
+ /**
+ * rewrites and compiles query into a hyracks job specifications
+ *
+ * @param metadataProvider
+ * The metadataProvider used to access metadata and build runtimes
+ * @param query
+ * The query to be compiled
+ * @param dmlStatement
+ * The data modification statement when the query results in a modification to a dataset
+ * @return the compiled {@code JobSpecification}
+ * @throws AsterixException
+ * @throws RemoteException
+ * @throws AlgebricksException
+ * @throws JSONException
+ * @throws ACIDException
+ */
+ JobSpecification rewriteCompileQuery(AqlMetadataProvider metadataProvider, Query query,
+ ICompiledDmlStatement dmlStatement)
+ throws AsterixException, RemoteException, AlgebricksException, JSONException, ACIDException;
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java
new file mode 100644
index 0000000..9addc87
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.util.List;
+
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.lang.common.base.Statement;
+
+/**
+ * An interface for creating a statement executor which can be provided by
+ * implementing {@code IStatementExecutorExtension}
+ */
+@FunctionalInterface
+public interface IStatementExecutorFactory {
+
+ /**
+ * create a statement executor
+ *
+ * @param statements
+ * Statements to execute
+ * @param conf
+ * request configuration
+ * @param compilationProvider
+ * @return an implementation of {@code IStatementExecutor} that is used to execute the passed list of statements
+ */
+ IStatementExecutor create(List<Statement> statements, SessionConfig conf,
+ ILangCompilationProvider compilationProvider);
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index bd87e6e..ed9c1e6 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.translator;
-import static java.util.logging.Logger.global;
-
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -29,8 +27,8 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
@@ -41,15 +39,15 @@ import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.aql.util.RangeMapBuilder;
import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.base.Expression.Kind;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
import org.apache.asterix.lang.common.clause.OrderbyClause;
-import org.apache.asterix.lang.common.clause.WhereClause;
import org.apache.asterix.lang.common.clause.OrderbyClause.OrderModifier;
+import org.apache.asterix.lang.common.clause.WhereClause;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.expression.FieldAccessor;
import org.apache.asterix.lang.common.expression.FieldBinding;
@@ -57,14 +55,14 @@ import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
import org.apache.asterix.lang.common.expression.IfExpr;
import org.apache.asterix.lang.common.expression.IndexAccessor;
import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.ListConstructor.Type;
import org.apache.asterix.lang.common.expression.LiteralExpr;
import org.apache.asterix.lang.common.expression.OperatorExpr;
import org.apache.asterix.lang.common.expression.QuantifiedExpression;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression.Quantifier;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.expression.ListConstructor.Type;
-import org.apache.asterix.lang.common.expression.QuantifiedExpression.Quantifier;
import org.apache.asterix.lang.common.literal.StringLiteral;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
@@ -75,13 +73,13 @@ import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.lang.common.visitor.base.AbstractQueryExpressionVisitor;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.declared.AqlSourceId;
import org.apache.asterix.metadata.declared.DatasetDataSource;
import org.apache.asterix.metadata.declared.LoadableDataSource;
import org.apache.asterix.metadata.declared.ResultSetDataSink;
import org.apache.asterix.metadata.declared.ResultSetSinkId;
-import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
@@ -117,15 +115,15 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
import org.apache.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
-import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
@@ -141,13 +139,13 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDelete
import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
@@ -198,10 +196,10 @@ class LangExpressionToPlanTranslator
"Unable to load dataset " + clffs.getDatasetName() + " since it does not exist");
}
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
- IAType metaItemType =
- metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
- DatasetDataSource targetDatasource =
- validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
+ IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
+ DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(),
+ stmt.getDatasetName());
List<List<String>> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset());
if (dataset.hasMetaPart()) {
throw new AlgebricksException(
@@ -264,8 +262,8 @@ class LangExpressionToPlanTranslator
additionalFilteringExpressions = new ArrayList<>();
PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, payloadVar, additionalFilteringVars,
additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
- additionalFilteringAssign =
- new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
+ additionalFilteringAssign = new AssignOperator(additionalFilteringVars,
+ additionalFilteringAssignExpressions);
}
InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef,
@@ -284,12 +282,11 @@ class LangExpressionToPlanTranslator
return new ALogicalPlanImpl(new MutableObject<>(leafOperator));
}
- @SuppressWarnings("unchecked")
@Override
public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt)
throws AlgebricksException {
- Pair<ILogicalOperator, LogicalVariable> p =
- expr.accept(this, new MutableObject<>(new EmptyTupleSourceOperator()));
+ Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this,
+ new MutableObject<>(new EmptyTupleSourceOperator()));
ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<>();
ILogicalOperator topOp = p.first;
ProjectOperator project = (ProjectOperator) topOp;
@@ -322,7 +319,10 @@ class LangExpressionToPlanTranslator
* because dataset only accept non-collection records
*/
LogicalVariable seqVar = context.newVar();
- /** This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest it; otherwise do nothing. */
+ /**
+ * This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest
+ * it; otherwise do nothing.
+ */
AssignOperator assignCollectionToSequence = new AssignOperator(seqVar,
new MutableObject<>(new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.COLLECTION_TO_SEQUENCE),
@@ -331,11 +331,10 @@ class LangExpressionToPlanTranslator
project.getInputs().get(0).setValue(assignCollectionToSequence);
project.getVariables().set(0, seqVar);
resVar = seqVar;
- DatasetDataSource targetDatasource =
- validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
- List<Integer> keySourceIndicator =
- ((InternalDatasetDetails) targetDatasource.getDataset().getDatasetDetails())
- .getKeySourceIndicator();
+ DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(),
+ stmt.getDatasetName());
+ List<Integer> keySourceIndicator = ((InternalDatasetDetails) targetDatasource.getDataset()
+ .getDatasetDetails()).getKeySourceIndicator();
ArrayList<LogicalVariable> vars = new ArrayList<>();
ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
@@ -367,8 +366,8 @@ class LangExpressionToPlanTranslator
PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, resVar, additionalFilteringVars,
additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
- additionalFilteringAssign =
- new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
+ additionalFilteringAssign = new AssignOperator(additionalFilteringVars,
+ additionalFilteringAssignExpressions);
additionalFilteringAssign.getInputs().add(new MutableObject<>(project));
assign.getInputs().add(new MutableObject<>(additionalFilteringAssign));
} else {
@@ -384,7 +383,8 @@ class LangExpressionToPlanTranslator
break;
case Statement.Kind.UPSERT:
leafOperator = translateUpsert(targetDatasource, varRef, varRefsForLoading,
- additionalFilteringExpressions, assign, additionalFilteringField);
+ additionalFilteringExpressions, assign, additionalFilteringField, unnestVar, project, exprs,
+ resVar, additionalFilteringAssign);
break;
case Statement.Kind.DELETE:
leafOperator = translateDelete(targetDatasource, varRef, varRefsForLoading,
@@ -451,8 +451,8 @@ class LangExpressionToPlanTranslator
List<LogicalVariable> metaAndKeysVars = null;
List<Mutable<ILogicalExpression>> metaAndKeysExprs = null;
List<Mutable<ILogicalExpression>> metaExpSingletonList = null;
- boolean isChangeFeed =
- FeedMetadataUtil.isChangeFeed(metadataProvider, sfs.getDataverseName(), sfs.getFeedName());
+ boolean isChangeFeed = FeedMetadataUtil.isChangeFeed(metadataProvider, sfs.getDataverseName(),
+ sfs.getFeedName());
if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
metaAndKeysVars = new ArrayList<>();
metaAndKeysExprs = new ArrayList<>();
@@ -527,26 +527,94 @@ class LangExpressionToPlanTranslator
private ILogicalOperator translateUpsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
List<Mutable<ILogicalExpression>> varRefsForLoading,
List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign,
- List<String> additionalFilteringField) throws AlgebricksException {
- if (targetDatasource.getDataset().hasMetaPart()) {
+ List<String> additionalFilteringField, LogicalVariable unnestVar, ProjectOperator project,
+ List<Mutable<ILogicalExpression>> exprs, LogicalVariable resVar, AssignOperator additionalFilteringAssign)
+ throws AlgebricksException {
+ if (!targetDatasource.getDataset().allow(project, Dataset.OP_UPSERT)) {
throw new AlgebricksException(targetDatasource.getDataset().getDatasetName()
+ ": upsert into dataset is not supported on Datasets with Meta records");
}
- InsertDeleteUpsertOperator upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
- varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT, false);
- upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
- upsertOp.getInputs().add(new MutableObject<>(assign));
- // Create and add a new variable used for representing the original record
- ARecordType recordType = (ARecordType) targetDatasource.getItemType();
- upsertOp.setPrevRecordVar(context.newVar());
- upsertOp.setPrevRecordType(recordType);
- if (additionalFilteringField != null) {
- upsertOp.setPrevFilterVar(context.newVar());
- upsertOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0)));
+ if (targetDatasource.getDataset().hasMetaPart()) {
+ InsertDeleteUpsertOperator feedModificationOp;
+ AssignOperator metaAndKeysAssign;
+ List<LogicalVariable> metaAndKeysVars;
+ List<Mutable<ILogicalExpression>> metaAndKeysExprs;
+ List<Mutable<ILogicalExpression>> metaExpSingletonList;
+ metaAndKeysVars = new ArrayList<>();
+ metaAndKeysExprs = new ArrayList<>();
+ // add the meta function
+ IFunctionInfo finfoMeta = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.META);
+ ScalarFunctionCallExpression metaFunction = new ScalarFunctionCallExpression(finfoMeta,
+ new MutableObject<>(new VariableReferenceExpression(unnestVar)));
+ // create assign for the meta part
+ LogicalVariable metaVar = context.newVar();
+ metaExpSingletonList = new ArrayList<>(1);
+ metaExpSingletonList.add(new MutableObject<>(new VariableReferenceExpression(metaVar)));
+ metaAndKeysVars.add(metaVar);
+ metaAndKeysExprs.add(new MutableObject<>(metaFunction));
+ project.getVariables().add(metaVar);
+ varRefsForLoading.clear();
+ for (Mutable<ILogicalExpression> assignExpr : exprs) {
+ if (assignExpr.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression funcCall = (AbstractFunctionCallExpression) assignExpr.getValue();
+ funcCall.substituteVar(resVar, unnestVar);
+ LogicalVariable pkVar = context.newVar();
+ metaAndKeysVars.add(pkVar);
+ metaAndKeysExprs.add(new MutableObject<>(assignExpr.getValue()));
+ project.getVariables().add(pkVar);
+ varRefsForLoading.add(new MutableObject<>(new VariableReferenceExpression(pkVar)));
+ }
+ }
+ // A change feed, we don't need the assign to access PKs
+ feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading,
+ metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false);
+ // Create and add a new variable used for representing the original record
+ feedModificationOp.setPrevRecordVar(context.newVar());
+ feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
+ if (targetDatasource.getDataset().hasMetaPart()) {
+ List<LogicalVariable> metaVars = new ArrayList<>();
+ metaVars.add(context.newVar());
+ feedModificationOp.setPrevAdditionalNonFilteringVars(metaVars);
+ List<Object> metaTypes = new ArrayList<>();
+ metaTypes.add(targetDatasource.getMetaItemType());
+ feedModificationOp.setPrevAdditionalNonFilteringTypes(metaTypes);
+ }
+
+ if (additionalFilteringField != null) {
+ feedModificationOp.setPrevFilterVar(context.newVar());
+ feedModificationOp.setPrevFilterType(
+ ((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0)));
+ additionalFilteringAssign.getInputs().clear();
+ additionalFilteringAssign.getInputs().add(assign.getInputs().get(0));
+ feedModificationOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
+ } else {
+ feedModificationOp.getInputs().add(assign.getInputs().get(0));
+ }
+ metaAndKeysAssign = new AssignOperator(metaAndKeysVars, metaAndKeysExprs);
+ metaAndKeysAssign.getInputs().add(project.getInputs().get(0));
+ project.getInputs().set(0, new MutableObject<>(metaAndKeysAssign));
+ feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+ SinkOperator leafOperator = new SinkOperator();
+ leafOperator.getInputs().add(new MutableObject<>(feedModificationOp));
+ return leafOperator;
+ } else {
+ InsertDeleteUpsertOperator feedModificationOp;
+ feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading,
+ InsertDeleteUpsertOperator.Kind.UPSERT, false);
+ feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+ feedModificationOp.getInputs().add(new MutableObject<>(assign));
+ // Create and add a new variable used for representing the original record
+ ARecordType recordType = (ARecordType) targetDatasource.getItemType();
+ feedModificationOp.setPrevRecordVar(context.newVar());
+ feedModificationOp.setPrevRecordType(recordType);
+ if (additionalFilteringField != null) {
+ feedModificationOp.setPrevFilterVar(context.newVar());
+ feedModificationOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0)));
+ }
+ SinkOperator leafOperator = new SinkOperator();
+ leafOperator.getInputs().add(new MutableObject<>(feedModificationOp));
+ return leafOperator;
}
- SinkOperator leafOperator = new SinkOperator();
- leafOperator.getInputs().add(new MutableObject<>(upsertOp));
- return leafOperator;
}
private ILogicalOperator translateInsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
@@ -577,8 +645,8 @@ class LangExpressionToPlanTranslator
}
AqlSourceId sourceId = new AqlSourceId(dataverseName, datasetName);
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
- IAType metaItemType =
- metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
INodeDomain domain = metadataProvider.findNodeDomain(dataset.getNodeGroupName());
return new DatasetDataSource(sourceId, dataset, itemType, metaItemType, AqlDataSourceType.INTERNAL_DATASET,
dataset.getDatasetDetails(), domain);
@@ -586,8 +654,8 @@ class LangExpressionToPlanTranslator
private FileSplit getDefaultOutputFileLocation() throws MetadataException {
String outputDir = System.getProperty("java.io.tmpDir");
- String filePath =
- outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX + outputFileID.incrementAndGet();
+ String filePath = outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX
+ + outputFileID.incrementAndGet();
AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties();
return new FileSplit(metadataProperties.getMetadataNodeName(), new FileReference(new File(filePath)));
}
@@ -604,8 +672,8 @@ class LangExpressionToPlanTranslator
returnedOp.getInputs().add(tupSource);
} else {
v = context.newVar(lc.getVarExpr());
- Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo =
- langExprToAlgExpression(lc.getBindingExpr(), tupSource);
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(lc.getBindingExpr(),
+ tupSource);
returnedOp = new AssignOperator(v, new MutableObject<>(eo.first));
returnedOp.getInputs().add(eo.second);
}
@@ -620,8 +688,8 @@ class LangExpressionToPlanTranslator
AbstractFunctionCallExpression fldAccess = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME));
fldAccess.getArguments().add(new MutableObject<>(p.first));
- ILogicalExpression faExpr =
- new ConstantExpression(new AsterixConstantValue(new AString(fa.getIdent().getValue())));
+ ILogicalExpression faExpr = new ConstantExpression(
+ new AsterixConstantValue(new AString(fa.getIdent().getValue())));
fldAccess.getArguments().add(new MutableObject<>(faExpr));
AssignOperator a = new AssignOperator(v, new MutableObject<>(fldAccess));
a.getInputs().add(p.second);
@@ -639,8 +707,8 @@ class LangExpressionToPlanTranslator
FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.ANY_COLLECTION_MEMBER));
f.getArguments().add(new MutableObject<>(p.first));
} else {
- Pair<ILogicalExpression, Mutable<ILogicalOperator>> indexPair =
- langExprToAlgExpression(ia.getIndexExpr(), tupSource);
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> indexPair = langExprToAlgExpression(ia.getIndexExpr(),
+ tupSource);
f = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.GET_ITEM));
f.getArguments().add(new MutableObject<>(p.first));
f.getArguments().add(new MutableObject<>(indexPair.first));
@@ -747,8 +815,8 @@ class LangExpressionToPlanTranslator
if (AsterixBuiltinFunctions.isBuiltinAggregateFunction(fi)) {
f = AsterixBuiltinFunctions.makeAggregateFunctionExpression(fi, args);
} else if (AsterixBuiltinFunctions.isBuiltinUnnestingFunction(fi)) {
- UnnestingFunctionCallExpression ufce =
- new UnnestingFunctionCallExpression(FunctionUtil.getFunctionInfo(fi), args);
+ UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression(FunctionUtil.getFunctionInfo(fi),
+ args);
ufce.setReturnsUniqueValues(AsterixBuiltinFunctions.returnsUniqueValues(fi));
f = ufce;
} else {
@@ -762,7 +830,6 @@ class LangExpressionToPlanTranslator
throw new IllegalStateException("Function declarations should be inlined at AST rewriting phase.");
}
- @SuppressWarnings("unchecked")
@Override
public Pair<ILogicalOperator, LogicalVariable> visit(GroupbyClause gc, Mutable<ILogicalOperator> tupSource)
throws AsterixException {
@@ -771,9 +838,8 @@ class LangExpressionToPlanTranslator
List<Pair<Expression, Identifier>> groupFieldList = gc.getGroupFieldList();
List<Mutable<ILogicalExpression>> groupRecordConstructorArgList = new ArrayList<>();
for (Pair<Expression, Identifier> groupField : groupFieldList) {
- ILogicalExpression groupFieldNameExpr =
- langExprToAlgExpression(new LiteralExpr(new StringLiteral(groupField.second.getValue())),
- topOp).first;
+ ILogicalExpression groupFieldNameExpr = langExprToAlgExpression(
+ new LiteralExpr(new StringLiteral(groupField.second.getValue())), topOp).first;
groupRecordConstructorArgList.add(new MutableObject<>(groupFieldNameExpr));
ILogicalExpression groupFieldExpr = langExprToAlgExpression(groupField.first, topOp).first;
groupRecordConstructorArgList.add(new MutableObject<>(groupFieldExpr));
@@ -802,14 +868,13 @@ class LangExpressionToPlanTranslator
// Calls the first-element aggregate function on a decoration variable to make sure the value
// is propagated through a global aggregation.
AggregateFunctionCallExpression firstElementAgg = AsterixBuiltinFunctions
- .makeAggregateFunctionExpression(
- AsterixBuiltinFunctions.FIRST_ELEMENT, flArgs);
+ .makeAggregateFunctionExpression(AsterixBuiltinFunctions.FIRST_ELEMENT, flArgs);
aggVars.add(decorVar);
aggFuncs.add(new MutableObject<>(firstElementAgg));
}
for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) {
- Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput =
- langExprToAlgExpression(entry.getKey(), topOp);
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression(
+ entry.getKey(), topOp);
topOp = listifyInput.second;
List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>();
flArgs.add(new MutableObject<>(listifyInput.first));
@@ -892,7 +957,7 @@ class LangExpressionToPlanTranslator
Pair<ILogicalOperator, LogicalVariable> pCond = ifexpr.getCondExpr().accept(this, tupSource);
LogicalVariable varCond = pCond.second;
- //Creates a subplan for the "then" branch.
+ // Creates a subplan for the "then" branch.
Pair<ILogicalOperator, LogicalVariable> opAndVarForThen = constructSubplanOperatorForBranch(pCond.first,
new MutableObject<>(new VariableReferenceExpression(varCond)), ifexpr.getThenExpr());
@@ -925,8 +990,8 @@ class LangExpressionToPlanTranslator
// Produces the final result.
LogicalVariable resultVar = context.newVar();
- AssignOperator finalAssignOp =
- new AssignOperator(resultVar, new MutableObject<>(new VariableReferenceExpression(unnestVar)));
+ AssignOperator finalAssignOp = new AssignOperator(resultVar,
+ new MutableObject<>(new VariableReferenceExpression(unnestVar)));
finalAssignOp.getInputs().add(new MutableObject<>(unnestOp));
return new Pair<>(finalAssignOp, resultVar);
}
@@ -1222,29 +1287,22 @@ class LangExpressionToPlanTranslator
return new ScalarFunctionCallExpression(finfo);
}
- private FunctionIdentifier operatorTypeToFunctionIdentifier(OperatorType t) {
+ private static FunctionIdentifier operatorTypeToFunctionIdentifier(OperatorType t) {
switch (t) {
- case EQ: {
+ case EQ:
return AlgebricksBuiltinFunctions.EQ;
- }
- case NEQ: {
+ case NEQ:
return AlgebricksBuiltinFunctions.NEQ;
- }
- case GT: {
+ case GT:
return AlgebricksBuiltinFunctions.GT;
- }
- case GE: {
+ case GE:
return AlgebricksBuiltinFunctions.GE;
- }
- case LT: {
+ case LT:
return AlgebricksBuiltinFunctions.LT;
- }
- case LE: {
+ case LE:
return AlgebricksBuiltinFunctions.LE;
- }
- default: {
+ default:
throw new IllegalStateException();
- }
}
}
@@ -1300,8 +1358,8 @@ class LangExpressionToPlanTranslator
Mutable<ILogicalOperator> topOpRef) throws AsterixException {
switch (expr.getKind()) {
case VARIABLE_EXPRESSION:
- VariableReferenceExpression ve =
- new VariableReferenceExpression(context.getVar(((VariableExpr) expr).getVar().getId()));
+ VariableReferenceExpression ve = new VariableReferenceExpression(
+ context.getVar(((VariableExpr) expr).getVar().getId()));
return new Pair<>(ve, topOpRef);
case LITERAL_EXPRESSION:
LiteralExpr val = (LiteralExpr) expr;
@@ -1344,7 +1402,7 @@ class LangExpressionToPlanTranslator
LogicalVariable varListified = context.newSubplanOutputVar();
AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(varListified),
- (List) mkSingletonArrayList(new MutableObject<>(funAgg)));
+ mkSingletonArrayList(new MutableObject<>(funAgg)));
agg.getInputs().add(opRef);
ILogicalOperator res;
if (bProject) {
@@ -1510,8 +1568,8 @@ class LangExpressionToPlanTranslator
// Recursively eliminate shared operator reference for the operator subtree,
// even if it is a deep copy of some other one.
- Map<LogicalVariable, LogicalVariable> childVarMap =
- eliminateSharedOperatorReference(childRef, opRefSet);
+ Map<LogicalVariable, LogicalVariable> childVarMap = eliminateSharedOperatorReference(childRef,
+ opRefSet);
// Substitute variables according to the new subtree.
VariableUtilities.substituteVariables(currentOperator, childVarMap, null);
@@ -1554,8 +1612,8 @@ class LangExpressionToPlanTranslator
context.enterSubplan();
SubplanOperator subplanOp = new SubplanOperator();
subplanOp.getInputs().add(new MutableObject<>(inputOp));
- Mutable<ILogicalOperator> nestedSource =
- new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(subplanOp)));
+ Mutable<ILogicalOperator> nestedSource = new MutableObject<>(
+ new NestedTupleSourceOperator(new MutableObject<>(subplanOp)));
SelectOperator select = new SelectOperator(selectExpr, false, null);
// The select operator cannot be moved up and down, otherwise it will cause typing issues (ASTERIXDB-1203).
OperatorPropertiesUtil.markMovable(select, false);
@@ -1575,8 +1633,8 @@ class LangExpressionToPlanTranslator
// Processes EXISTS and NOT EXISTS.
private AssignOperator processExists(ILogicalExpression inputExpr, LogicalVariable v1, boolean not) {
- AbstractFunctionCallExpression count =
- new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.SCALAR_COUNT));
+ AbstractFunctionCallExpression count = new ScalarFunctionCallExpression(
+ FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.SCALAR_COUNT));
count.getArguments().add(new MutableObject<>(inputExpr));
AbstractFunctionCallExpression comparison = new ScalarFunctionCallExpression(
FunctionUtil.getFunctionInfo(not ? AsterixBuiltinFunctions.EQ : AsterixBuiltinFunctions.NEQ));
@@ -1648,8 +1706,8 @@ class LangExpressionToPlanTranslator
while (inputOpRefIterator.hasNext()) {
// Generates the variable triple <leftVar, rightVar, outputVar> .
topUnionVar = context.newVar();
- Triple<LogicalVariable, LogicalVariable, LogicalVariable> varTriple =
- new Triple<>(leftInputVar, inputVarIterator.next(), topUnionVar);
+ Triple<LogicalVariable, LogicalVariable, LogicalVariable> varTriple = new Triple<>(leftInputVar,
+ inputVarIterator.next(), topUnionVar);
List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varTriples = new ArrayList<>();
varTriples.add(varTriple);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
index 63d1908..ea3ec2c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
@@ -71,7 +71,6 @@ public class PlanTranslationUtil {
}
}
- @SuppressWarnings("unchecked")
private static ScalarFunctionCallExpression createFieldAccessExpression(ILogicalExpression target,
List<String> field) {
FunctionIdentifier functionIdentifier;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj b/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
index e0e9388..725f50a 100644
--- a/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
+++ b/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
@@ -41,7 +41,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.aql.clause.DistinctClause;
@@ -100,6 +99,7 @@ import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
[15/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index b3e9e09..16ac80d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -72,15 +72,15 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
private AqlMetadataProvider metadataProvider;
// Function Identifier sets that retain the original field variable through each function's arguments
- private final ImmutableSet<FunctionIdentifier> funcIDSetThatRetainFieldName = ImmutableSet.of(
- AsterixBuiltinFunctions.WORD_TOKENS, AsterixBuiltinFunctions.GRAM_TOKENS, AsterixBuiltinFunctions.SUBSTRING,
- AsterixBuiltinFunctions.SUBSTRING_BEFORE, AsterixBuiltinFunctions.SUBSTRING_AFTER,
- AsterixBuiltinFunctions.CREATE_POLYGON, AsterixBuiltinFunctions.CREATE_MBR,
- AsterixBuiltinFunctions.CREATE_RECTANGLE, AsterixBuiltinFunctions.CREATE_CIRCLE,
- AsterixBuiltinFunctions.CREATE_LINE, AsterixBuiltinFunctions.CREATE_POINT,
- AsterixBuiltinFunctions.NUMERIC_ADD, AsterixBuiltinFunctions.NUMERIC_SUBTRACT,
- AsterixBuiltinFunctions.NUMERIC_MULTIPLY, AsterixBuiltinFunctions.NUMERIC_DIVIDE,
- AsterixBuiltinFunctions.NUMERIC_MOD);
+ private final ImmutableSet<FunctionIdentifier> funcIDSetThatRetainFieldName =
+ ImmutableSet.of(AsterixBuiltinFunctions.WORD_TOKENS, AsterixBuiltinFunctions.GRAM_TOKENS,
+ AsterixBuiltinFunctions.SUBSTRING, AsterixBuiltinFunctions.SUBSTRING_BEFORE,
+ AsterixBuiltinFunctions.SUBSTRING_AFTER, AsterixBuiltinFunctions.CREATE_POLYGON,
+ AsterixBuiltinFunctions.CREATE_MBR, AsterixBuiltinFunctions.CREATE_RECTANGLE,
+ AsterixBuiltinFunctions.CREATE_CIRCLE, AsterixBuiltinFunctions.CREATE_LINE,
+ AsterixBuiltinFunctions.CREATE_POINT, AsterixBuiltinFunctions.NUMERIC_ADD,
+ AsterixBuiltinFunctions.NUMERIC_SUBTRACT, AsterixBuiltinFunctions.NUMERIC_MULTIPLY,
+ AsterixBuiltinFunctions.NUMERIC_DIVIDE, AsterixBuiltinFunctions.NUMERIC_MOD);
public abstract Map<FunctionIdentifier, List<IAccessMethod>> getAccessMethods();
@@ -145,15 +145,15 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
return list.isEmpty() ? null : list.get(0);
}
- protected List<Pair<IAccessMethod, Index>> chooseAllIndex(
- Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
+ protected List<Pair<IAccessMethod, Index>>
+ chooseAllIndex(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
List<Pair<IAccessMethod, Index>> result = new ArrayList<Pair<IAccessMethod, Index>>();
Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
while (amIt.hasNext()) {
Map.Entry<IAccessMethod, AccessMethodAnalysisContext> amEntry = amIt.next();
AccessMethodAnalysisContext analysisCtx = amEntry.getValue();
- Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt = analysisCtx.indexExprsAndVars.entrySet()
- .iterator();
+ Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt =
+ analysisCtx.indexExprsAndVars.entrySet().iterator();
while (indexIt.hasNext()) {
Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry = indexIt.next();
// To avoid a case where the chosen access method and a chosen
@@ -167,11 +167,11 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
// LENGTH_PARTITIONED_NGRAM_INVIX]
IAccessMethod chosenAccessMethod = amEntry.getKey();
Index chosenIndex = indexEntry.getKey();
- boolean isKeywordOrNgramIndexChosen = chosenIndex
- .getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX
- || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
- || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX;
+ boolean isKeywordOrNgramIndexChosen =
+ chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || chosenIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX
+ || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
+ || chosenIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX;
if ((chosenAccessMethod == BTreeAccessMethod.INSTANCE && chosenIndex.getIndexType() == IndexType.BTREE)
|| (chosenAccessMethod == RTreeAccessMethod.INSTANCE
@@ -196,8 +196,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
*/
public void pruneIndexCandidates(IAccessMethod accessMethod, AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException {
- Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexExprAndVarIt = analysisCtx.indexExprsAndVars
- .entrySet().iterator();
+ Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexExprAndVarIt =
+ analysisCtx.indexExprsAndVars.entrySet().iterator();
// Used to keep track of matched expressions (added for prefix search)
int numMatchedKeys = 0;
ArrayList<Integer> matchedExpressions = new ArrayList<Integer>();
@@ -293,8 +293,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
// Check if any field name in the optFuncExpr matches.
if (optFuncExpr.findFieldName(keyField) != -1) {
- foundKeyField = typeMatch
- && optFuncExpr.getOperatorSubTree(exprAndVarIdx.second).hasDataSourceScan();
+ foundKeyField =
+ typeMatch && optFuncExpr.getOperatorSubTree(exprAndVarIdx.second).hasDataSourceScan();
if (foundKeyField) {
matchedExpressions.add(exprAndVarIdx.first);
numMatchedKeys++;
@@ -369,8 +369,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
continue;
}
AbstractFunctionCallExpression argFuncExpr = (AbstractFunctionCallExpression) argExpr;
- boolean matchFound = analyzeFunctionExpr(argFuncExpr, assignsAndUnnests, analyzedAMs, context,
- typeEnvironment);
+ boolean matchFound =
+ analyzeFunctionExpr(argFuncExpr, assignsAndUnnests, analyzedAMs, context, typeEnvironment);
found = found || matchFound;
}
return found;
@@ -441,14 +441,15 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
// fieldName
for (Index index : datasetIndexes) {
// Need to also verify the index is pending no op
- if (index.getKeyFieldNames().contains(fieldName) && index.getPendingOp() == IMetadataEntity.PENDING_NO_OP) {
+ if (index.getKeyFieldNames().contains(fieldName)
+ && index.getPendingOp() == IMetadataEntity.PENDING_NO_OP) {
indexCandidates.add(index);
if (optFuncExpr.getFieldType(varIdx) == BuiltinType.AMISSING
|| optFuncExpr.getFieldType(varIdx) == BuiltinType.ANY) {
optFuncExpr.setFieldType(varIdx,
index.getKeyFieldTypes().get(index.getKeyFieldNames().indexOf(fieldName)));
}
- analysisCtx.addIndexExpr(matchedSubTree.dataset, index, matchedFuncExprIndex, varIdx);
+ analysisCtx.addIndexExpr(matchedSubTree.getDataset(), index, matchedFuncExprIndex, varIdx);
}
}
// No index candidates for fieldName.
@@ -464,11 +465,11 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
List<Index> datasetIndexes = new ArrayList<Index>();
LogicalVariable datasetMetaVar = null;
LogicalVariable datasetRecordVar = null;
- if (subTree.dataSourceType != DataSourceType.COLLECTION_SCAN) {
- datasetIndexes = metadataProvider.getDatasetIndexes(subTree.dataset.getDataverseName(),
- subTree.dataset.getDatasetName());
+ if (subTree.getDataSourceType() != DataSourceType.COLLECTION_SCAN) {
+ datasetIndexes = metadataProvider.getDatasetIndexes(subTree.getDataset().getDataverseName(),
+ subTree.getDataset().getDatasetName());
List<LogicalVariable> datasetVars = subTree.getDataSourceVariables();
- if (subTree.dataset.hasMetaPart()) {
+ if (subTree.getDataset().hasMetaPart()) {
datasetMetaVar = datasetVars.get(datasetVars.size() - 1);
datasetRecordVar = datasetVars.get(datasetVars.size() - 2);
} else {
@@ -477,9 +478,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
}
for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
// Try to match variables from optFuncExpr to assigns or unnests.
- for (int assignOrUnnestIndex = 0; assignOrUnnestIndex < subTree.assignsAndUnnests
+ for (int assignOrUnnestIndex = 0; assignOrUnnestIndex < subTree.getAssignsAndUnnests()
.size(); assignOrUnnestIndex++) {
- AbstractLogicalOperator op = subTree.assignsAndUnnests.get(assignOrUnnestIndex);
+ AbstractLogicalOperator op = subTree.getAssignsAndUnnests().get(assignOrUnnestIndex);
if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
analyzeAssignOp((AssignOperator) op, optFuncExpr, subTree, assignOrUnnestIndex, datasetRecordVar,
datasetMetaVar, context, datasetIndexes, optFuncExprIndex, analysisCtx);
@@ -501,7 +502,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
if (subTree.hasIxJoinOuterAdditionalDataSource()) {
additionalDsVarList = new ArrayList<LogicalVariable>();
- for (int i = 0; i < subTree.ixJoinOuterAdditionalDataSourceRefs.size(); i++) {
+ for (int i = 0; i < subTree.getIxJoinOuterAdditionalDataSourceRefs().size(); i++) {
additionalDsVarList.addAll(subTree.getIxJoinOuterAdditionalDataSourceVariables(i));
}
@@ -529,18 +530,18 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
// Remember matching subtree.
optFuncExpr.setOptimizableSubTree(funcVarIndex, subTree);
List<String> fieldName = null;
- if (subTree.dataSourceType == DataSourceType.COLLECTION_SCAN) {
+ if (subTree.getDataSourceType() == DataSourceType.COLLECTION_SCAN) {
optFuncExpr.setLogicalExpr(funcVarIndex, new VariableReferenceExpression(var));
} else {
- fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, 0, subTree.recordType,
+ fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, 0, subTree.getRecordType(),
funcVarIndex, optFuncExpr.getFuncExpr().getArguments().get(funcVarIndex).getValue(),
- datasetRecordVar, subTree.metaRecordType, datasetMetaVar);
+ datasetRecordVar, subTree.getMetaRecordType(), datasetMetaVar);
if (fieldName == null) {
return;
}
}
- IAType fieldType = (IAType) context.getOutputTypeEnvironment(unnestOp)
- .getType(optFuncExpr.getLogicalExpr(funcVarIndex));
+ IAType fieldType =
+ (IAType) context.getOutputTypeEnvironment(unnestOp).getType(optFuncExpr.getLogicalExpr(funcVarIndex));
// Set the fieldName in the corresponding matched function
// expression.
optFuncExpr.setFieldName(funcVarIndex, fieldName);
@@ -570,10 +571,11 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
// Remember matching subtree.
optFuncExpr.setOptimizableSubTree(optVarIndex, subTree);
- List<String> fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, varIndex,
- subTree.recordType, optVarIndex,
- optFuncExpr.getFuncExpr().getArguments().get(optVarIndex).getValue(), datasetRecordVar,
- subTree.metaRecordType, datasetMetaVar);
+ List<String> fieldName =
+ getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, varIndex,
+ subTree.getRecordType(), optVarIndex,
+ optFuncExpr.getFuncExpr().getArguments().get(optVarIndex).getValue(),
+ datasetRecordVar, subTree.getMetaRecordType(), datasetMetaVar);
if (fieldName == null) {
continue;
}
@@ -609,25 +611,27 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
List<List<String>> subTreePKs = null;
if (!fromAdditionalDataSource) {
- subTreePKs = DatasetUtils.getPartitioningKeys(subTree.dataset);
+ subTreePKs = DatasetUtils.getPartitioningKeys(subTree.getDataset());
// Check whether this variable is PK, not a record variable.
if (varIndex <= subTreePKs.size() - 1) {
fieldName = subTreePKs.get(varIndex);
- fieldType = (IAType) context.getOutputTypeEnvironment(subTree.dataSourceRef.getValue())
- .getVarType(var);
+ fieldType =
+ (IAType) context.getOutputTypeEnvironment(
+ subTree.getDataSourceRef().getValue()).getVarType(var);
}
} else {
// Need to check additional dataset one by one
- for (int i = 0; i < subTree.ixJoinOuterAdditionalDatasets.size(); i++) {
- if (subTree.ixJoinOuterAdditionalDatasets.get(i) != null) {
- subTreePKs = DatasetUtils.getPartitioningKeys(subTree.ixJoinOuterAdditionalDatasets.get(i));
+ for (int i = 0; i < subTree.getIxJoinOuterAdditionalDatasets().size(); i++) {
+ if (subTree.getIxJoinOuterAdditionalDatasets().get(i) != null) {
+ subTreePKs = DatasetUtils.getPartitioningKeys(
+ subTree.getIxJoinOuterAdditionalDatasets().get(i));
// Check whether this variable is PK, not a record variable.
if (subTreePKs.contains(var) && varIndex <= subTreePKs.size() - 1) {
fieldName = subTreePKs.get(varIndex);
fieldType = (IAType) context
.getOutputTypeEnvironment(
- subTree.ixJoinOuterAdditionalDataSourceRefs.get(i).getValue())
+ subTree.getIxJoinOuterAdditionalDataSourceRefs().get(i).getValue())
.getVarType(var);
break;
}
@@ -651,7 +655,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
private void setTypeTag(IOptimizationContext context, OptimizableOperatorSubTree subTree,
IOptimizableFuncExpr optFuncExpr, int funcVarIndex) throws AlgebricksException {
// Set the typeTag if the type is not null
- IAType type = (IAType) context.getOutputTypeEnvironment(subTree.root)
+ IAType type = (IAType) context.getOutputTypeEnvironment(subTree.getRoot())
.getVarType(optFuncExpr.getLogicalVar(funcVarIndex));
optFuncExpr.setFieldType(funcVarIndex, type);
}
@@ -663,14 +667,15 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
*
* @throws AlgebricksException
*/
- protected List<String> getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree subTree,
- int opIndex, int assignVarIndex, ARecordType recordType, int funcVarIndex,
- ILogicalExpression parentFuncExpr, LogicalVariable recordVar, ARecordType metaType, LogicalVariable metaVar)
+ protected List<String> getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr,
+ OptimizableOperatorSubTree subTree, int opIndex, int assignVarIndex, ARecordType recordType,
+ int funcVarIndex, ILogicalExpression parentFuncExpr, LogicalVariable recordVar,
+ ARecordType metaType, LogicalVariable metaVar)
throws AlgebricksException {
// Get expression corresponding to opVar at varIndex.
AbstractLogicalExpression expr = null;
AbstractFunctionCallExpression childFuncExpr = null;
- AbstractLogicalOperator op = subTree.assignsAndUnnests.get(opIndex);
+ AbstractLogicalOperator op = subTree.getAssignsAndUnnests().get(opIndex);
if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
AssignOperator assignOp = (AssignOperator) op;
expr = (AbstractLogicalExpression) assignOp.getExpressions().get(assignVarIndex).getValue();
@@ -718,8 +723,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
return null;
}
ConstantExpression constExpr = (ConstantExpression) nameArg;
- AOrderedList orderedNestedFieldName = (AOrderedList) ((AsterixConstantValue) constExpr.getValue())
- .getObject();
+ AOrderedList orderedNestedFieldName =
+ (AOrderedList) ((AsterixConstantValue) constExpr.getValue()).getObject();
nestedAccessFieldName = new ArrayList<String>();
for (int i = 0; i < orderedNestedFieldName.size(); i++) {
nestedAccessFieldName.add(((AString) orderedNestedFieldName.getItem(i)).getStringValue());
@@ -728,14 +733,14 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
isByName = true;
}
if (isFieldAccess) {
- LogicalVariable sourceVar = ((VariableReferenceExpression) funcExpr.getArguments().get(0).getValue())
- .getVariableReference();
+ LogicalVariable sourceVar =
+ ((VariableReferenceExpression) funcExpr.getArguments().get(0).getValue()).getVariableReference();
optFuncExpr.setLogicalExpr(funcVarIndex, parentFuncExpr);
int[] assignAndExpressionIndexes = null;
//go forward through nested assigns until you find the relevant one
- for (int i = opIndex + 1; i < subTree.assignsAndUnnests.size(); i++) {
- AbstractLogicalOperator subOp = subTree.assignsAndUnnests.get(i);
+ for (int i = opIndex + 1; i < subTree.getAssignsAndUnnests().size(); i++) {
+ AbstractLogicalOperator subOp = subTree.getAssignsAndUnnests().get(i);
List<LogicalVariable> varList;
if (subOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
@@ -822,9 +827,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
LogicalVariable curVar = ((VariableReferenceExpression) argExpr).getVariableReference();
// We look for the assign or unnest operator that produces curVar below
// the current operator
- for (int assignOrUnnestIndex = opIndex + 1; assignOrUnnestIndex < subTree.assignsAndUnnests
+ for (int assignOrUnnestIndex = opIndex + 1; assignOrUnnestIndex < subTree.getAssignsAndUnnests()
.size(); assignOrUnnestIndex++) {
- AbstractLogicalOperator curOp = subTree.assignsAndUnnests.get(assignOrUnnestIndex);
+ AbstractLogicalOperator curOp = subTree.getAssignsAndUnnests().get(assignOrUnnestIndex);
if (curOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
AssignOperator assignOp = (AssignOperator) curOp;
List<LogicalVariable> varList = assignOp.getVariables();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 4dc7bf4..eb7d3a4 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -22,6 +22,7 @@ package org.apache.asterix.optimizer.rules.am;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -80,30 +81,28 @@ public class BTreeAccessMethod implements IAccessMethod {
EQUAL
}
- // TODO: There is some redundancy here, since these are listed in AlgebricksBuiltinFunctions as well.
- private static List<FunctionIdentifier> funcIdents = new ArrayList<FunctionIdentifier>();
+ private static final List<FunctionIdentifier> FUNC_IDENTIFIERS =
+ Collections.unmodifiableList(Arrays.asList(
+ AlgebricksBuiltinFunctions.EQ,
+ AlgebricksBuiltinFunctions.LE,
+ AlgebricksBuiltinFunctions.GE,
+ AlgebricksBuiltinFunctions.LT,
+ AlgebricksBuiltinFunctions.GT));
- static {
- funcIdents.add(AlgebricksBuiltinFunctions.EQ);
- funcIdents.add(AlgebricksBuiltinFunctions.LE);
- funcIdents.add(AlgebricksBuiltinFunctions.GE);
- funcIdents.add(AlgebricksBuiltinFunctions.LT);
- funcIdents.add(AlgebricksBuiltinFunctions.GT);
- }
-
- public static BTreeAccessMethod INSTANCE = new BTreeAccessMethod();
+ public static final BTreeAccessMethod INSTANCE = new BTreeAccessMethod();
@Override
public List<FunctionIdentifier> getOptimizableFunctions() {
- return funcIdents;
+ return FUNC_IDENTIFIERS;
}
@Override
public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr,
List<AbstractLogicalOperator> assignsAndUnnests, AccessMethodAnalysisContext analysisCtx,
IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException {
- boolean matches = AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx, context,
- typeEnvironment);
+ boolean matches =
+ AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(
+ funcExpr, analysisCtx, context, typeEnvironment);
if (!matches) {
matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVars(funcExpr, analysisCtx);
}
@@ -131,8 +130,8 @@ public class BTreeAccessMethod implements IAccessMethod {
if (primaryIndexUnnestOp == null) {
return false;
}
- Mutable<ILogicalOperator> opRef = (subTree.assignsAndUnnestsRefs.isEmpty()) ? null
- : subTree.assignsAndUnnestsRefs.get(0);
+ Mutable<ILogicalOperator> opRef =
+ subTree.getAssignsAndUnnestsRefs().isEmpty() ? null : subTree.getAssignsAndUnnestsRefs().get(0);
ILogicalOperator op = null;
if (opRef != null) {
op = opRef.getValue();
@@ -141,7 +140,7 @@ public class BTreeAccessMethod implements IAccessMethod {
if (conditionRef.getValue() != null) {
select.getInputs().clear();
if (op != null) {
- subTree.dataSourceRef.setValue(primaryIndexUnnestOp);
+ subTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
select.getInputs().add(new MutableObject<ILogicalOperator>(op));
} else {
select.getInputs().add(new MutableObject<ILogicalOperator>(primaryIndexUnnestOp));
@@ -149,7 +148,7 @@ public class BTreeAccessMethod implements IAccessMethod {
} else {
((AbstractLogicalOperator) primaryIndexUnnestOp).setExecutionMode(ExecutionMode.PARTITIONED);
if (op != null) {
- subTree.dataSourceRef.setValue(primaryIndexUnnestOp);
+ subTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
selectRef.setValue(op);
} else {
selectRef.setValue(primaryIndexUnnestOp);
@@ -165,15 +164,16 @@ public class BTreeAccessMethod implements IAccessMethod {
boolean hasGroupBy) throws AlgebricksException {
AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
Mutable<ILogicalExpression> conditionRef = joinOp.getCondition();
- // Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
+ // Determine if the index is applicable on the left or right side
+ // (if both, we arbitrarily prefer the left side).
Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex);
- OptimizableOperatorSubTree indexSubTree = null;
- OptimizableOperatorSubTree probeSubTree = null;
+ OptimizableOperatorSubTree indexSubTree;
+ OptimizableOperatorSubTree probeSubTree;
// We assume that the left subtree is the outer branch and the right subtree is the inner branch.
// This assumption holds true since we only use an index from the right subtree.
// The following is just a sanity check.
if (rightSubTree.hasDataSourceScan()
- && dataset.getDatasetName().equals(rightSubTree.dataset.getDatasetName())) {
+ && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
indexSubTree = rightSubTree;
probeSubTree = leftSubTree;
} else {
@@ -199,16 +199,16 @@ public class BTreeAccessMethod implements IAccessMethod {
}
// If there are conditions left, add a new select operator on top.
- indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp);
+ indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
if (conditionRef.getValue() != null) {
SelectOperator topSelect = new SelectOperator(conditionRef, isLeftOuterJoin, newNullPlaceHolderVar);
- topSelect.getInputs().add(indexSubTree.rootRef);
+ topSelect.getInputs().add(indexSubTree.getRootRef());
topSelect.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(topSelect);
// Replace the original join with the new subtree rooted at the select op.
joinRef.setValue(topSelect);
} else {
- joinRef.setValue(indexSubTree.rootRef.getValue());
+ joinRef.setValue(indexSubTree.getRootRef().getValue());
}
return true;
}
@@ -216,19 +216,20 @@ public class BTreeAccessMethod implements IAccessMethod {
@Override
public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
- AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast,
- IOptimizationContext context) throws AlgebricksException {
- Dataset dataset = indexSubTree.dataset;
- ARecordType recordType = indexSubTree.recordType;
- ARecordType metaRecordType = indexSubTree.metaRecordType;
+ AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull,
+ boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException {
+ Dataset dataset = indexSubTree.getDataset();
+ ARecordType recordType = indexSubTree.getRecordType();
+ ARecordType metaRecordType = indexSubTree.getMetaRecordType();
// we made sure indexSubTree has datasource scan
- AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.dataSourceRef.getValue();
+ AbstractDataSourceOperator dataSourceOp =
+ (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue();
List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.indexExprsAndVars.get(chosenIndex);
List<IOptimizableFuncExpr> matchedFuncExprs = analysisCtx.matchedFuncExprs;
int numSecondaryKeys = analysisCtx.indexNumMatchedKeys.get(chosenIndex);
// List of function expressions that will be replaced by the secondary-index search.
// These func exprs will be removed from the select condition at the very end of this method.
- Set<ILogicalExpression> replacedFuncExprs = new HashSet<ILogicalExpression>();
+ Set<ILogicalExpression> replacedFuncExprs = new HashSet<>();
// Info on high and low keys for the BTree search predicate.
ILogicalExpression[] lowKeyExprs = new ILogicalExpression[numSecondaryKeys];
@@ -240,12 +241,12 @@ public class BTreeAccessMethod implements IAccessMethod {
ILogicalExpression[] constantAtRuntimeExpressions = new ILogicalExpression[numSecondaryKeys];
LogicalVariable[] constAtRuntimeExprVars = new LogicalVariable[numSecondaryKeys];
- // TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range predicate.
- // If we can't figure out how to integrate a certain funcExpr into the current predicate, we just bail by setting this flag.
+ /* TODO: For now we don't do any sophisticated analysis of the func exprs to come up with "the best" range
+ * predicate. If we can't figure out how to integrate a certain funcExpr into the current predicate,
+ * we just bail by setting this flag.*/
boolean couldntFigureOut = false;
boolean doneWithExprs = false;
boolean isEqCondition = false;
- // TODO: For now don't consider prefix searches.
BitSet setLowKeys = new BitSet(numSecondaryKeys);
BitSet setHighKeys = new BitSet(numSecondaryKeys);
// Go through the func exprs listed as optimizable by the chosen index,
@@ -253,24 +254,22 @@ public class BTreeAccessMethod implements IAccessMethod {
// checks whether a type casting happened from a real (FLOAT, DOUBLE) value to an INT value
// since we have a round issues when dealing with LT(<) OR GT(>) operator.
- boolean realTypeConvertedToIntegerType = false;
+ boolean realTypeConvertedToIntegerType;
for (Pair<Integer, Integer> exprIndex : exprAndVarList) {
// Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs.
IOptimizableFuncExpr optFuncExpr = matchedFuncExprs.get(exprIndex.first);
int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames());
- if (keyPos < 0) {
- if (optFuncExpr.getNumLogicalVars() > 1) {
- // If we are optimizing a join, the matching field may be the second field name.
- keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames());
- }
+ if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) {
+ // If we are optimizing a join, the matching field may be the second field name.
+ keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames());
}
if (keyPos < 0) {
throw new AlgebricksException(
"Could not match optimizable function expression to any index field name.");
}
- Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr,
- indexSubTree, probeSubTree);
+ Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr =
+ AccessMethodUtils.createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree);
ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
if (searchKeyExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
constantAtRuntimeExpressions[keyPos] = searchKeyExpr;
@@ -288,7 +287,7 @@ public class BTreeAccessMethod implements IAccessMethod {
//
// for $emp in dataset empDataset
// where $emp.age > double("2.3") and $emp.age < double("3.3")
- // return $emp.id;
+ // return $emp.id
//
// It should generate a result if there is a tuple that satisfies the condition, which is 3,
// however, it does not generate the desired result since finding candidates
@@ -316,7 +315,8 @@ public class BTreeAccessMethod implements IAccessMethod {
setHighKeys.set(keyPos);
isEqCondition = true;
} else {
- // Has already been set to the identical values. When optimizing join we may encounter the same optimizable expression twice
+ // Has already been set to the identical values.
+ // When optimizing join we may encounter the same optimizable expression twice
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true
&& lowKeyExprs[keyPos].equals(searchKeyExpr) && highKeyLimits[keyPos] == limit
@@ -339,7 +339,8 @@ public class BTreeAccessMethod implements IAccessMethod {
highKeyExprs[keyPos] = searchKeyExpr;
highKeyInclusive[keyPos] = false;
} else {
- // Has already been set to the identical values. When optimizing join we may encounter the same optimizable expression twice
+ // Has already been set to the identical values. When optimizing join we may encounter the
+ // same optimizable expression twice
// (once from analyzing each side of the join)
if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == false
&& highKeyExprs[keyPos].equals(searchKeyExpr)) {
@@ -356,7 +357,8 @@ public class BTreeAccessMethod implements IAccessMethod {
highKeyExprs[keyPos] = searchKeyExpr;
highKeyInclusive[keyPos] = true;
} else {
- // Has already been set to the identical values. When optimizing join we may encounter the same optimizable expression twice
+ // Has already been set to the identical values. When optimizing join we may encounter the
+ // same optimizable expression twice
// (once from analyzing each side of the join)
if (highKeyLimits[keyPos] == limit && highKeyInclusive[keyPos] == true
&& highKeyExprs[keyPos].equals(searchKeyExpr)) {
@@ -373,7 +375,8 @@ public class BTreeAccessMethod implements IAccessMethod {
lowKeyExprs[keyPos] = searchKeyExpr;
lowKeyInclusive[keyPos] = false;
} else {
- // Has already been set to the identical values. When optimizing join we may encounter the same optimizable expression twice
+ // Has already been set to the identical values. When optimizing join we may encounter the
+ // same optimizable expression twice
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == false
&& lowKeyExprs[keyPos].equals(searchKeyExpr)) {
@@ -390,7 +393,8 @@ public class BTreeAccessMethod implements IAccessMethod {
lowKeyExprs[keyPos] = searchKeyExpr;
lowKeyInclusive[keyPos] = true;
} else {
- // Has already been set to the identical values. When optimizing join we may encounter the same optimizable expression twice
+ // Has already been set to the identical values. When optimizing join we may encounter the
+ // same optimizable expression twice
// (once from analyzing each side of the join)
if (lowKeyLimits[keyPos] == limit && lowKeyInclusive[keyPos] == true
&& lowKeyExprs[keyPos].equals(searchKeyExpr)) {
@@ -417,7 +421,8 @@ public class BTreeAccessMethod implements IAccessMethod {
return null;
}
- // If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals closed to obtain a superset of answers and leave the original selection in place.
+ // If the select condition contains mixed open/closed intervals on multiple keys, then we make all intervals
+ // closed to obtain a superset of answers and leave the original selection in place.
boolean primaryIndexPostProccessingIsNeeded = false;
for (int i = 1; i < numSecondaryKeys; ++i) {
if (lowKeyInclusive[i] != lowKeyInclusive[0]) {
@@ -450,7 +455,8 @@ public class BTreeAccessMethod implements IAccessMethod {
highKeyInclusive[0] = true;
}
- // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index search.
+ // Here we generate vars and funcs for assigning the secondary-index keys to be fed into the secondary-index
+ // search.
// List of variables for the assign.
ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
// List of variables and expressions for the assign.
@@ -480,7 +486,7 @@ public class BTreeAccessMethod implements IAccessMethod {
inputOp = assignConstantSearchKeys;
} else {
// All index search keys are variables.
- inputOp = probeSubTree.root;
+ inputOp = probeSubTree.getRoot();
}
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
@@ -494,7 +500,7 @@ public class BTreeAccessMethod implements IAccessMethod {
// External dataset
UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp,
dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull);
- indexSubTree.dataSourceRef.setValue(externalDataAccessOp);
+ indexSubTree.getDataSourceRef().setValue(externalDataAccessOp);
return externalDataAccessOp;
} else if (!isPrimaryIndex) {
primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
@@ -542,8 +548,8 @@ public class BTreeAccessMethod implements IAccessMethod {
// An index search is expressed as an unnest-map over an
// index-search function.
IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH);
- UnnestingFunctionCallExpression primaryIndexSearchFunc = new UnnestingFunctionCallExpression(
- primaryIndexSearch, primaryIndexFuncArgs);
+ UnnestingFunctionCallExpression primaryIndexSearchFunc =
+ new UnnestingFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs);
primaryIndexSearchFunc.setReturnsUniqueValues(true);
if (!leftOuterUnnestMapRequired) {
primaryIndexUnnestOp = new UnnestMapOperator(scanVariables,
@@ -651,8 +657,8 @@ public class BTreeAccessMethod implements IAccessMethod {
}
private LimitType getLimitType(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree probeSubTree) {
- ComparisonKind ck = AlgebricksBuiltinFunctions
- .getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier());
+ ComparisonKind ck =
+ AlgebricksBuiltinFunctions.getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier());
LimitType limit = null;
switch (ck) {
case EQ: {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
index e1cda09..ff4d219 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
@@ -108,11 +108,11 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod
boolean matchInLeftSubTree = false;
boolean matchInRightSubTree = false;
if (leftSubTree.hasDataSource()) {
- matchInLeftSubTree = analyzeCondition(joinCond, leftSubTree.assignsAndUnnests, analyzedAMs, context,
+ matchInLeftSubTree = analyzeCondition(joinCond, leftSubTree.getAssignsAndUnnests(), analyzedAMs, context,
typeEnvironment);
}
if (rightSubTree.hasDataSource()) {
- matchInRightSubTree = analyzeCondition(joinCond, rightSubTree.assignsAndUnnests, analyzedAMs, context,
+ matchInRightSubTree = analyzeCondition(joinCond, rightSubTree.getAssignsAndUnnests(), analyzedAMs, context,
typeEnvironment);
}
if (!matchInLeftSubTree && !matchInRightSubTree) {
@@ -178,8 +178,8 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod
*/
protected void removeIndexCandidatesFromOuterBranch(Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) {
String innerDataset = null;
- if (rightSubTree.dataset != null) {
- innerDataset = rightSubTree.dataset.getDatasetName();
+ if (rightSubTree.getDataset() != null) {
+ innerDataset = rightSubTree.getDataset().getDatasetName();
}
Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt = analyzedAMs.entrySet().iterator();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
index 45c5e34..0e54640 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
import org.apache.asterix.metadata.declared.AqlMetadataProvider;
import org.apache.asterix.metadata.declared.DatasetDataSource;
import org.apache.asterix.metadata.entities.Dataset;
@@ -102,7 +103,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
List<Index> datasetIndexes = ((AqlMetadataProvider) context.getMetadataProvider())
.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
- List<IOptimizableFuncExpr> optFuncExprs = new ArrayList<IOptimizableFuncExpr>();
+ List<IOptimizableFuncExpr> optFuncExprs = new ArrayList<>();
for (int i = 0; i < analysisCtx.matchedFuncExprs.size(); i++) {
IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(i);
@@ -123,8 +124,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
private AssignOperator createAssignOperator(List<IOptimizableFuncExpr> optFuncExprs,
List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOptimizationContext context) {
- List<LogicalVariable> assignKeyVarList = new ArrayList<LogicalVariable>();
- List<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<Mutable<ILogicalExpression>>();
+ List<LogicalVariable> assignKeyVarList = new ArrayList<>();
+ List<Mutable<ILogicalExpression>> assignKeyExprList = new ArrayList<>();
for (IOptimizableFuncExpr optFuncExpr : optFuncExprs) {
ComparisonKind ck = AlgebricksBuiltinFunctions
@@ -154,15 +155,15 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
DataSourceScanOperator dataSourceScanOp = (DataSourceScanOperator) descendantOp;
AqlDataSource ds = (AqlDataSource) dataSourceScanOp.getDataSource();
if (dataset.getDatasetName().compareTo(((DatasetDataSource) ds).getDataset().getDatasetName()) == 0) {
- List<LogicalVariable> minFilterVars = new ArrayList<LogicalVariable>();
- List<LogicalVariable> maxFilterVars = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> minFilterVars = new ArrayList<>();
+ List<LogicalVariable> maxFilterVars = new ArrayList<>();
AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, context);
dataSourceScanOp.setMinFilterVars(minFilterVars);
dataSourceScanOp.setMaxFilterVars(maxFilterVars);
- List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<Mutable<ILogicalExpression>>();;
+ List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<>();
for (LogicalVariable var : assignOp.getVariables()) {
additionalFilteringExpressions
.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
@@ -186,8 +187,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
jobGenParams.readFromFuncArgs(f.getArguments());
if (dataset.getDatasetName().compareTo(jobGenParams.datasetName) == 0) {
- List<LogicalVariable> minFilterVars = new ArrayList<LogicalVariable>();
- List<LogicalVariable> maxFilterVars = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> minFilterVars = new ArrayList<>();
+ List<LogicalVariable> maxFilterVars = new ArrayList<>();
AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars,
context);
@@ -195,7 +196,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
unnestMapOp.setMinFilterVars(minFilterVars);
unnestMapOp.setMaxFilterVars(maxFilterVars);
- List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<Mutable<ILogicalExpression>>();;
+ List<Mutable<ILogicalExpression>> additionalFilteringExpressions = new ArrayList<>();
for (LogicalVariable var : assignOp.getVariables()) {
additionalFilteringExpressions
.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
@@ -220,6 +221,9 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
if (descendantOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
DataSourceScanOperator dataSourceScanOp = (DataSourceScanOperator) descendantOp;
AqlDataSource ds = (AqlDataSource) dataSourceScanOp.getDataSource();
+ if (ds.getDatasourceType() != AqlDataSourceType.INTERNAL_DATASET) {
+ return null;
+ }
return ((DatasetDataSource) ds).getDataset();
} else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
UnnestMapOperator unnestMapOp = (UnnestMapOperator) descendantOp;
@@ -227,8 +231,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
FunctionIdentifier fid = f.getFunctionIdentifier();
- String dataverseName = null;
- String datasetName = null;
+ String dataverseName;
+ String datasetName;
if (AsterixBuiltinFunctions.EXTERNAL_LOOKUP.equals(fid)) {
dataverseName = AccessMethodUtils.getStringConstant(f.getArguments().get(0));
datasetName = AccessMethodUtils.getStringConstant(f.getArguments().get(1));
@@ -299,7 +303,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
private boolean findMacthedExprFieldName(IOptimizableFuncExpr optFuncExpr, AbstractLogicalOperator op,
Dataset dataset, ARecordType recType, List<Index> datasetIndexes, IOptimizationContext context)
- throws AlgebricksException {
+ throws AlgebricksException {
AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
while (descendantOp != null) {
if (descendantOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
@@ -346,7 +350,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
continue;
}
- String indexName = null;
+ String indexName;
Index index = null;
ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
@@ -408,10 +412,10 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
|| funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
//get the variable from here. Figure out which input it came from. Go to that input!!!
- ArrayList<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+ ArrayList<LogicalVariable> usedVars = new ArrayList<>();
expr.getUsedVariables(usedVars);
LogicalVariable usedVar = usedVars.get(0);
- List<String> returnList = new ArrayList<String>();
+ List<String> returnList = new ArrayList<>();
//Find the input that it came from
for (int varCheck = 0; varCheck < op.getInputs().size(); varCheck++) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
index 5b9a9b6..2464c06 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
@@ -106,7 +106,7 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth
// Analyze select condition.
Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs = new TreeMap<IAccessMethod, AccessMethodAnalysisContext>();
- if (!analyzeCondition(selectCond, subTree.assignsAndUnnests, analyzedAMs, context, typeEnvironment)) {
+ if (!analyzeCondition(selectCond, subTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment)) {
return false;
}
@@ -164,7 +164,7 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth
}
ILogicalOperator primaryUnnest = connectAll2ndarySearchPlanWithIntersect(subRoots, context);
- subTree.dataSourceRef.setValue(primaryUnnest);
+ subTree.getDataSourceRef().setValue(primaryUnnest);
return primaryUnnest != null;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index 58cc32e..1052a72 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -99,7 +99,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
INVALID
}
- private static List<FunctionIdentifier> funcIdents = new ArrayList<FunctionIdentifier>();
+ private static List<FunctionIdentifier> funcIdents = new ArrayList<>();
static {
funcIdents.add(AsterixBuiltinFunctions.STRING_CONTAINS);
@@ -110,7 +110,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
// These function identifiers are matched in this AM's analyzeFuncExprArgs(),
// and are not visible to the outside driver.
- private static HashSet<FunctionIdentifier> secondLevelFuncIdents = new HashSet<FunctionIdentifier>();
+ private static HashSet<FunctionIdentifier> secondLevelFuncIdents = new HashSet<>();
static {
secondLevelFuncIdents.add(AsterixBuiltinFunctions.SIMILARITY_JACCARD_CHECK);
@@ -276,11 +276,12 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
ILogicalExpression arg1 = funcExpr.getArguments().get(0).getValue();
ILogicalExpression arg2 = funcExpr.getArguments().get(1).getValue();
// Determine whether one arg is constant, and the other is non-constant.
- ILogicalExpression constArg = null;
- ILogicalExpression nonConstArg = null;
+ ILogicalExpression constArg;
+ ILogicalExpression nonConstArg;
if (arg1.getExpressionTag() == LogicalExpressionTag.CONSTANT
&& arg2.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
- // The arguments of edit-distance-contains() function are asymmetrical, we can only use index if it is on the first argument
+ // The arguments of edit-distance-contains() function are asymmetrical, we can only use index if it is on
+ // the first argument
if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) {
return false;
}
@@ -368,11 +369,11 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
IOptimizationContext context) throws AlgebricksException {
IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
- Dataset dataset = indexSubTree.dataset;
- ARecordType recordType = indexSubTree.recordType;
- ARecordType metaRecordType = indexSubTree.metaRecordType;
+ Dataset dataset = indexSubTree.getDataset();
+ ARecordType recordType = indexSubTree.getRecordType();
+ ARecordType metaRecordType = indexSubTree.getMetaRecordType();
// we made sure indexSubTree has datasource scan
- DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.dataSourceRef.getValue();
+ DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.getDataSourceRef().getValue();
InvertedIndexJobGenParams jobGenParams = new InvertedIndexJobGenParams(chosenIndex.getIndexName(),
chosenIndex.getIndexType(), dataset.getDataverseName(), dataset.getDatasetName(), retainInput,
@@ -403,7 +404,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
// We are optimizing a join. Add the input variable to the secondaryIndexFuncArgs.
LogicalVariable inputSearchVariable = getInputSearchVar(optFuncExpr, indexSubTree);
keyVarList.add(inputSearchVariable);
- inputOp = (AbstractLogicalOperator) probeSubTree.root;
+ inputOp = (AbstractLogicalOperator) probeSubTree.getRoot();
}
jobGenParams.setKeyVarList(keyVarList);
ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
@@ -440,7 +441,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx,
false, false, false, context);
// Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
- subTree.dataSourceRef.setValue(indexPlanRootOp);
+ subTree.getDataSourceRef().setValue(indexPlanRootOp);
return true;
}
@@ -451,14 +452,14 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
boolean hasGroupBy) throws AlgebricksException {
// Figure out if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex);
- OptimizableOperatorSubTree indexSubTree = null;
- OptimizableOperatorSubTree probeSubTree = null;
+ OptimizableOperatorSubTree indexSubTree;
+ OptimizableOperatorSubTree probeSubTree;
// We assume that the left subtree is the outer branch and the right subtree is the inner branch.
// This assumption holds true since we only use an index from the right subtree.
// The following is just a sanity check.
if (rightSubTree.hasDataSourceScan()
- && dataset.getDatasetName().equals(rightSubTree.dataset.getDatasetName())) {
+ && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
indexSubTree = rightSubTree;
probeSubTree = leftSubTree;
} else {
@@ -469,8 +470,8 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
// The arguments of edit-distance-contains() function are asymmetrical, we can only use index
// if the dataset of index subtree and the dataset of first argument's subtree is the same
if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS
- && optFuncExpr.getOperatorSubTree(0).dataset != null && !optFuncExpr.getOperatorSubTree(0).dataset
- .getDatasetName().equals(indexSubTree.dataset.getDatasetName())) {
+ && optFuncExpr.getOperatorSubTree(0).getDataset() != null && !optFuncExpr.getOperatorSubTree(0)
+ .getDataset().getDatasetName().equals(indexSubTree.getDataset().getDatasetName())) {
return false;
}
@@ -489,9 +490,9 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
// Remember the original probe subtree, and its primary-key variables,
// so we can later retrieve the missing attributes via an equi join.
- List<LogicalVariable> originalSubTreePKs = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> originalSubTreePKs = new ArrayList<>();
// Remember the primary-keys of the new probe subtree for the top-level equi join.
- List<LogicalVariable> surrogateSubTreePKs = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> surrogateSubTreePKs = new ArrayList<>();
// Copy probe subtree, replacing their variables with new ones. We will use the original variables
// to stitch together a top-level equi join.
@@ -499,8 +500,8 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
join.getCondition().getValue(), optFuncExpr, originalSubTreePKs, surrogateSubTreePKs, context);
// Remember original live variables from the index sub tree.
- List<LogicalVariable> indexSubTreeLiveVars = new ArrayList<LogicalVariable>();
- VariableUtilities.getLiveVariables(indexSubTree.root, indexSubTreeLiveVars);
+ List<LogicalVariable> indexSubTreeLiveVars = new ArrayList<>();
+ VariableUtilities.getLiveVariables(indexSubTree.getRoot(), indexSubTreeLiveVars);
// Clone the original join condition because we may have to modify it (and we also need the original).
ILogicalExpression joinCond = join.getCondition().getValue().cloneExpression();
@@ -510,22 +511,22 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
if (optFuncExpr.getFuncExpr().getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CHECK
|| optFuncExpr.getFuncExpr()
.getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) {
- panicJoinRef = new MutableObject<ILogicalOperator>(joinRef.getValue());
- panicVarMap = new HashMap<LogicalVariable, LogicalVariable>();
+ panicJoinRef = new MutableObject<>(joinRef.getValue());
+ panicVarMap = new HashMap<>();
Mutable<ILogicalOperator> newProbeRootRef = createPanicNestedLoopJoinPlan(panicJoinRef, indexSubTree,
probeSubTree, optFuncExpr, chosenIndex, panicVarMap, context);
- probeSubTree.rootRef.setValue(newProbeRootRef.getValue());
- probeSubTree.root = newProbeRootRef.getValue();
+ probeSubTree.getRootRef().setValue(newProbeRootRef.getValue());
+ probeSubTree.setRoot(newProbeRootRef.getValue());
}
// Create regular indexed-nested loop join path.
ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, indexSubTree, probeSubTree, chosenIndex,
analysisCtx, true, isLeftOuterJoin, true, context);
- indexSubTree.dataSourceRef.setValue(indexPlanRootOp);
+ indexSubTree.getDataSourceRef().setValue(indexPlanRootOp);
// Change join into a select with the same condition.
SelectOperator topSelect = new SelectOperator(new MutableObject<ILogicalExpression>(joinCond), isLeftOuterJoin,
newNullPlaceHolderVar);
- topSelect.getInputs().add(indexSubTree.rootRef);
+ topSelect.getInputs().add(indexSubTree.getRootRef());
topSelect.setExecutionMode(ExecutionMode.LOCAL);
context.computeAndSetTypeEnvironmentForOperator(topSelect);
ILogicalOperator topOp = topSelect;
@@ -535,10 +536,10 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
LogicalVariable inputSearchVar = getInputSearchVar(optFuncExpr, indexSubTree);
indexSubTreeLiveVars.addAll(originalSubTreePKs);
indexSubTreeLiveVars.add(inputSearchVar);
- List<LogicalVariable> panicPlanLiveVars = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> panicPlanLiveVars = new ArrayList<>();
VariableUtilities.getLiveVariables(panicJoinRef.getValue(), panicPlanLiveVars);
// Create variable mapping for union all operator.
- List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>();
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<>();
for (int i = 0; i < indexSubTreeLiveVars.size(); i++) {
LogicalVariable indexSubTreeVar = indexSubTreeLiveVars.get(i);
LogicalVariable panicPlanVar = panicVarMap.get(indexSubTreeVar);
@@ -593,7 +594,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
Map<LogicalVariable, LogicalVariable> joinInputSubTreeVarMap = new HashMap<LogicalVariable, LogicalVariable>();
// Init with all live vars.
List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
- VariableUtilities.getLiveVariables(probeSubTree.root, liveVars);
+ VariableUtilities.getLiveVariables(probeSubTree.getRoot(), liveVars);
for (LogicalVariable var : liveVars) {
joinInputSubTreeVarMap.put(var, var);
}
@@ -612,25 +613,25 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
// Create first copy.
LogicalOperatorDeepCopyWithNewVariablesVisitor firstDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(
context, context, newProbeSubTreeVarMap);
- ILogicalOperator newProbeSubTree = firstDeepCopyVisitor.deepCopy(probeSubTree.root);
+ ILogicalOperator newProbeSubTree = firstDeepCopyVisitor.deepCopy(probeSubTree.getRoot());
inferTypes(newProbeSubTree, context);
Mutable<ILogicalOperator> newProbeSubTreeRootRef = new MutableObject<ILogicalOperator>(newProbeSubTree);
// Create second copy.
LogicalOperatorDeepCopyWithNewVariablesVisitor secondDeepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(
context, context, joinInputSubTreeVarMap);
- ILogicalOperator joinInputSubTree = secondDeepCopyVisitor.deepCopy(probeSubTree.root);
+ ILogicalOperator joinInputSubTree = secondDeepCopyVisitor.deepCopy(probeSubTree.getRoot());
inferTypes(joinInputSubTree, context);
- probeSubTree.rootRef.setValue(joinInputSubTree);
+ probeSubTree.getRootRef().setValue(joinInputSubTree);
// Remember the original probe subtree reference so we can return it.
- Mutable<ILogicalOperator> originalProbeSubTreeRootRef = probeSubTree.rootRef;
+ Mutable<ILogicalOperator> originalProbeSubTreeRootRef = probeSubTree.getRootRef();
// Replace the original probe subtree with its copy.
- Dataset origDataset = probeSubTree.dataset;
- ARecordType origRecordType = probeSubTree.recordType;
+ Dataset origDataset = probeSubTree.getDataset();
+ ARecordType origRecordType = probeSubTree.getRecordType();
probeSubTree.initFromSubTree(newProbeSubTreeRootRef);
- probeSubTree.dataset = origDataset;
- probeSubTree.recordType = origRecordType;
+ probeSubTree.setDataset(origDataset);
+ probeSubTree.setRecordType(origRecordType);
// Replace the variables in the join condition based on the mapping of variables
// in the new probe subtree.
@@ -673,12 +674,12 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
// We split the plan into two "branches", and add selections on each side.
AbstractLogicalOperator replicateOp = new ReplicateOperator(2);
- replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(probeSubTree.root));
+ replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(probeSubTree.getRoot()));
replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
context.computeAndSetTypeEnvironmentForOperator(replicateOp);
// Create select ops for removing tuples that are filterable and not filterable, respectively.
- IVariableTypeEnvironment probeTypeEnv = context.getOutputTypeEnvironment(probeSubTree.root);
+ IVariableTypeEnvironment probeTypeEnv = context.getOutputTypeEnvironment(probeSubTree.getRoot());
IAType inputSearchVarType;
if (chosenIndex.isEnforcingKeyFileds()) {
inputSearchVarType = optFuncExpr.getFieldType(optFuncExpr.findLogicalVar(inputSearchVar));
@@ -691,12 +692,12 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
isFilterableSelectOpRef, isNotFilterableSelectOpRef);
List<LogicalVariable> originalLiveVars = new ArrayList<LogicalVariable>();
- VariableUtilities.getLiveVariables(indexSubTree.root, originalLiveVars);
+ VariableUtilities.getLiveVariables(indexSubTree.getRoot(), originalLiveVars);
// Copy the scan subtree in indexSubTree.
LogicalOperatorDeepCopyWithNewVariablesVisitor deepCopyVisitor = new LogicalOperatorDeepCopyWithNewVariablesVisitor(
context, context);
- ILogicalOperator scanSubTree = deepCopyVisitor.deepCopy(indexSubTree.root);
+ ILogicalOperator scanSubTree = deepCopyVisitor.deepCopy(indexSubTree.getRoot());
Map<LogicalVariable, LogicalVariable> copyVarMap = deepCopyVisitor.getInputToOutputVariableMapping();
panicVarMap.putAll(copyVarMap);
@@ -1020,7 +1021,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
return null;
}
- for (AbstractLogicalOperator op : subTree.assignsAndUnnests) {
+ for (AbstractLogicalOperator op : subTree.getAssignsAndUnnests()) {
if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
continue;
}
[10/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
deleted file mode 100644
index d6065fb..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ /dev/null
@@ -1,3107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.aql.translator;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.rmi.RemoteException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Random;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.active.ActiveJobNotificationHandler;
-import org.apache.asterix.active.ActivityState;
-import org.apache.asterix.active.EntityId;
-import org.apache.asterix.active.IActiveEntityEventsListener;
-import org.apache.asterix.api.common.APIFramework;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.app.external.ExternalIndexingOperations;
-import org.apache.asterix.app.external.FeedJoint;
-import org.apache.asterix.app.external.FeedOperations;
-import org.apache.asterix.common.config.AsterixExternalProperties;
-import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
-import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
-import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.config.GlobalConfig;
-import org.apache.asterix.common.config.MetadataConstants;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.functions.FunctionSignature;
-import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.external.api.IAdapterFactory;
-import org.apache.asterix.external.feed.api.IFeed;
-import org.apache.asterix.external.feed.api.IFeed.FeedType;
-import org.apache.asterix.external.feed.api.IFeedJoint;
-import org.apache.asterix.external.feed.api.IFeedJoint.FeedJointType;
-import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
-import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
-import org.apache.asterix.external.feed.management.FeedConnectionId;
-import org.apache.asterix.external.feed.management.FeedConnectionRequest;
-import org.apache.asterix.external.feed.management.FeedEventsListener;
-import org.apache.asterix.external.feed.management.FeedJointKey;
-import org.apache.asterix.external.feed.management.FeedLifecycleEventSubscriber;
-import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
-import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
-import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
-import org.apache.asterix.external.feed.watch.FeedIntakeInfo;
-import org.apache.asterix.external.indexing.ExternalFile;
-import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType;
-import org.apache.asterix.file.DatasetOperations;
-import org.apache.asterix.file.DataverseOperations;
-import org.apache.asterix.file.IndexOperations;
-import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
-import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
-import org.apache.asterix.lang.common.base.IRewriterFactory;
-import org.apache.asterix.lang.common.base.IStatementRewriter;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.expression.TypeExpression;
-import org.apache.asterix.lang.common.statement.CompactStatement;
-import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
-import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
-import org.apache.asterix.lang.common.statement.CreateFeedStatement;
-import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
-import org.apache.asterix.lang.common.statement.CreateIndexStatement;
-import org.apache.asterix.lang.common.statement.CreatePrimaryFeedStatement;
-import org.apache.asterix.lang.common.statement.CreateSecondaryFeedStatement;
-import org.apache.asterix.lang.common.statement.DatasetDecl;
-import org.apache.asterix.lang.common.statement.DataverseDecl;
-import org.apache.asterix.lang.common.statement.DataverseDropStatement;
-import org.apache.asterix.lang.common.statement.DeleteStatement;
-import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
-import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
-import org.apache.asterix.lang.common.statement.FeedDropStatement;
-import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
-import org.apache.asterix.lang.common.statement.FunctionDecl;
-import org.apache.asterix.lang.common.statement.FunctionDropStatement;
-import org.apache.asterix.lang.common.statement.IDatasetDetailsDecl;
-import org.apache.asterix.lang.common.statement.IndexDropStatement;
-import org.apache.asterix.lang.common.statement.InsertStatement;
-import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
-import org.apache.asterix.lang.common.statement.LoadStatement;
-import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
-import org.apache.asterix.lang.common.statement.NodegroupDecl;
-import org.apache.asterix.lang.common.statement.Query;
-import org.apache.asterix.lang.common.statement.RefreshExternalDatasetStatement;
-import org.apache.asterix.lang.common.statement.RunStatement;
-import org.apache.asterix.lang.common.statement.SetStatement;
-import org.apache.asterix.lang.common.statement.TypeDecl;
-import org.apache.asterix.lang.common.statement.TypeDropStatement;
-import org.apache.asterix.lang.common.statement.WriteStatement;
-import org.apache.asterix.lang.common.struct.Identifier;
-import org.apache.asterix.lang.common.util.FunctionUtil;
-import org.apache.asterix.metadata.IDatasetDetails;
-import org.apache.asterix.metadata.MetadataException;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.metadata.api.IMetadataEntity;
-import org.apache.asterix.metadata.dataset.hints.DatasetHints;
-import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetNodegroupCardinalityHint;
-import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.metadata.entities.CompactionPolicy;
-import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.Datatype;
-import org.apache.asterix.metadata.entities.Dataverse;
-import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
-import org.apache.asterix.metadata.entities.Feed;
-import org.apache.asterix.metadata.entities.FeedPolicyEntity;
-import org.apache.asterix.metadata.entities.Function;
-import org.apache.asterix.metadata.entities.Index;
-import org.apache.asterix.metadata.entities.InternalDatasetDetails;
-import org.apache.asterix.metadata.entities.NodeGroup;
-import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
-import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
-import org.apache.asterix.metadata.utils.DatasetUtils;
-import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
-import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
-import org.apache.asterix.metadata.utils.MetadataLockManager;
-import org.apache.asterix.om.types.ARecordType;
-import org.apache.asterix.om.types.ATypeTag;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.TypeSignature;
-import org.apache.asterix.om.util.AsterixAppContextInfo;
-import org.apache.asterix.om.util.AsterixClusterProperties;
-import org.apache.asterix.optimizer.rules.IntroduceSecondaryIndexInsertDeleteRule;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
-import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory;
-import org.apache.asterix.translator.AbstractLangTranslator;
-import org.apache.asterix.translator.CompiledStatements.CompiledConnectFeedStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledDatasetDropStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledDeleteStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledIndexCompactStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledSubscribeFeedStatement;
-import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement;
-import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
-import org.apache.asterix.translator.TypeTranslator;
-import org.apache.asterix.translator.util.ValidateUtil;
-import org.apache.asterix.util.FlushDatasetUtils;
-import org.apache.asterix.util.JobUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.algebricks.common.utils.Triple;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
-import org.apache.hyracks.algebricks.data.IAWriterFactory;
-import org.apache.hyracks.algebricks.data.IResultSerializerFactoryProvider;
-import org.apache.hyracks.algebricks.runtime.serializer.ResultSerializerFactoryProvider;
-import org.apache.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.file.FileSplit;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-import com.google.common.collect.Lists;
-
-/*
- * Provides functionality for executing a batch of Query statements (queries included)
- * sequentially.
- */
-public class QueryTranslator extends AbstractLangTranslator {
-
- private static final Logger LOGGER = Logger.getLogger(QueryTranslator.class.getName());
-
- private enum ProgressState {
- NO_PROGRESS,
- ADDED_PENDINGOP_RECORD_TO_METADATA
- }
-
- public enum ResultDelivery {
- SYNC,
- ASYNC,
- ASYNC_DEFERRED
- }
-
- public static final boolean IS_DEBUG_MODE = false;// true
- private final List<Statement> statements;
- private final SessionConfig sessionConfig;
- private Dataverse activeDefaultDataverse;
- private final List<FunctionDecl> declaredFunctions;
- private final APIFramework apiFramework;
- private final IRewriterFactory rewriterFactory;
-
- public QueryTranslator(List<Statement> aqlStatements, SessionConfig conf,
- ILangCompilationProvider compliationProvider) {
- this.statements = aqlStatements;
- this.sessionConfig = conf;
- this.declaredFunctions = getDeclaredFunctions(aqlStatements);
- this.apiFramework = new APIFramework(compliationProvider);
- this.rewriterFactory = compliationProvider.getRewriterFactory();
- }
-
- private List<FunctionDecl> getDeclaredFunctions(List<Statement> statements) {
- List<FunctionDecl> functionDecls = new ArrayList<>();
- for (Statement st : statements) {
- if (st.getKind() == Statement.Kind.FUNCTION_DECL) {
- functionDecls.add((FunctionDecl) st);
- }
- }
- return functionDecls;
- }
-
- /**
- * Compiles and submits for execution a list of AQL statements.
- *
- * @param hcc
- * A Hyracks client connection that is used to submit a jobspec to Hyracks.
- * @param hdc
- * A Hyracks dataset client object that is used to read the results.
- * @param resultDelivery
- * True if the results should be read asynchronously or false if we should wait for results to be read.
- * @return A List<QueryResult> containing a QueryResult instance corresponding to each submitted query.
- * @throws Exception
- */
- public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery)
- throws Exception {
- compileAndExecute(hcc, hdc, resultDelivery, new ResultUtils.Stats());
- }
-
- public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery,
- ResultUtils.Stats stats) throws Exception {
- int resultSetIdCounter = 0;
- FileSplit outputFile = null;
- IAWriterFactory writerFactory = PrinterBasedWriterFactory.INSTANCE;
- IResultSerializerFactoryProvider resultSerializerFactoryProvider = ResultSerializerFactoryProvider.INSTANCE;
- Map<String, String> config = new HashMap<>();
- /* Since the system runs a large number of threads, when HTTP requests don't return, it becomes difficult to
- * find the thread running the request to determine where it has stopped.
- * Setting the thread name helps make that easier
- */
- String threadName = Thread.currentThread().getName();
- Thread.currentThread().setName(QueryTranslator.class.getSimpleName());
- try {
- for (Statement stmt : statements) {
- if (sessionConfig.is(SessionConfig.FORMAT_HTML)) {
- sessionConfig.out().println(APIFramework.HTML_STATEMENT_SEPARATOR);
- }
- validateOperation(activeDefaultDataverse, stmt);
- rewriteStatement(stmt); // Rewrite the statement's AST.
- AqlMetadataProvider metadataProvider = new AqlMetadataProvider(activeDefaultDataverse);
- metadataProvider.setWriterFactory(writerFactory);
- metadataProvider.setResultSerializerFactoryProvider(resultSerializerFactoryProvider);
- metadataProvider.setOutputFile(outputFile);
- metadataProvider.setConfig(config);
- switch (stmt.getKind()) {
- case Statement.Kind.SET:
- handleSetStatement(stmt, config);
- break;
- case Statement.Kind.DATAVERSE_DECL:
- activeDefaultDataverse = handleUseDataverseStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.CREATE_DATAVERSE:
- handleCreateDataverseStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.DATASET_DECL:
- handleCreateDatasetStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.CREATE_INDEX:
- handleCreateIndexStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.TYPE_DECL:
- handleCreateTypeStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.NODEGROUP_DECL:
- handleCreateNodeGroupStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.DATAVERSE_DROP:
- handleDataverseDropStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.DATASET_DROP:
- handleDatasetDropStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.INDEX_DROP:
- handleIndexDropStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.TYPE_DROP:
- handleTypeDropStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.NODEGROUP_DROP:
- handleNodegroupDropStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.CREATE_FUNCTION:
- handleCreateFunctionStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.FUNCTION_DROP:
- handleFunctionDropStatement(metadataProvider, stmt);
- break;
- case Statement.Kind.LOAD:
- handleLoadStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.INSERT:
- case Statement.Kind.UPSERT:
- handleInsertUpsertStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.DELETE:
- handleDeleteStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.CREATE_PRIMARY_FEED:
- case Statement.Kind.CREATE_SECONDARY_FEED:
- handleCreateFeedStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.DROP_FEED:
- handleDropFeedStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.DROP_FEED_POLICY:
- handleDropFeedPolicyStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.CONNECT_FEED:
- handleConnectFeedStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.DISCONNECT_FEED:
- handleDisconnectFeedStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.SUBSCRIBE_FEED:
- handleSubscribeFeedStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.CREATE_FEED_POLICY:
- handleCreateFeedPolicyStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.QUERY:
- metadataProvider.setResultSetId(new ResultSetId(resultSetIdCounter++));
- metadataProvider.setResultAsyncMode(resultDelivery == ResultDelivery.ASYNC
- || resultDelivery == ResultDelivery.ASYNC_DEFERRED);
- handleQuery(metadataProvider, (Query) stmt, hcc, hdc, resultDelivery, stats);
- break;
- case Statement.Kind.COMPACT:
- handleCompactStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.EXTERNAL_DATASET_REFRESH:
- handleExternalDatasetRefreshStatement(metadataProvider, stmt, hcc);
- break;
- case Statement.Kind.WRITE:
- Pair<IAWriterFactory, FileSplit> result = handleWriteStatement(stmt);
- writerFactory = (result.first != null) ? result.first : writerFactory;
- outputFile = result.second;
- break;
- case Statement.Kind.RUN:
- handleRunStatement(metadataProvider, stmt, hcc);
- break;
- default:
- // Default should delegate unknown statement to extension-manager
- break;
- }
- }
- } finally {
- Thread.currentThread().setName(threadName);
- }
- }
-
- private void handleSetStatement(Statement stmt, Map<String, String> config) {
- SetStatement ss = (SetStatement) stmt;
- String pname = ss.getPropName();
- String pvalue = ss.getPropValue();
- config.put(pname, pvalue);
- }
-
- private Pair<IAWriterFactory, FileSplit> handleWriteStatement(Statement stmt)
- throws InstantiationException, IllegalAccessException, ClassNotFoundException {
- WriteStatement ws = (WriteStatement) stmt;
- File f = new File(ws.getFileName());
- FileSplit outputFile = new FileSplit(ws.getNcName().getValue(), new FileReference(f));
- IAWriterFactory writerFactory = null;
- if (ws.getWriterClassName() != null) {
- writerFactory = (IAWriterFactory) Class.forName(ws.getWriterClassName()).newInstance();
- }
- return new Pair<>(writerFactory, outputFile);
- }
-
- private Dataverse handleUseDataverseStatement(AqlMetadataProvider metadataProvider, Statement stmt)
- throws Exception {
- DataverseDecl dvd = (DataverseDecl) stmt;
- String dvName = dvd.getDataverseName().getValue();
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- MetadataLockManager.INSTANCE.acquireDataverseReadLock(dvName);
- try {
- Dataverse dv = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dvName);
- if (dv == null) {
- throw new MetadataException("Unknown dataverse " + dvName);
- }
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return dv;
- } catch (Exception e) {
- abort(e, e, mdTxnCtx);
- throw new MetadataException(e);
- } finally {
- MetadataLockManager.INSTANCE.releaseDataverseReadLock(dvName);
- }
- }
-
- private void handleCreateDataverseStatement(AqlMetadataProvider metadataProvider, Statement stmt) throws Exception {
-
- CreateDataverseStatement stmtCreateDataverse = (CreateDataverseStatement) stmt;
- String dvName = stmtCreateDataverse.getDataverseName().getValue();
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- MetadataLockManager.INSTANCE.acquireDataverseReadLock(dvName);
- try {
- Dataverse dv = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dvName);
- if (dv != null) {
- if (stmtCreateDataverse.getIfNotExists()) {
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return;
- } else {
- throw new AlgebricksException("A dataverse with this name " + dvName + " already exists.");
- }
- }
- MetadataManager.INSTANCE.addDataverse(metadataProvider.getMetadataTxnContext(),
- new Dataverse(dvName, stmtCreateDataverse.getFormat(), IMetadataEntity.PENDING_NO_OP));
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e) {
- abort(e, e, mdTxnCtx);
- throw e;
- } finally {
- MetadataLockManager.INSTANCE.releaseDataverseReadLock(dvName);
- }
- }
-
- private void validateCompactionPolicy(String compactionPolicy, Map<String, String> compactionPolicyProperties,
- MetadataTransactionContext mdTxnCtx, boolean isExternalDataset) throws AsterixException, Exception {
- CompactionPolicy compactionPolicyEntity = MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx,
- MetadataConstants.METADATA_DATAVERSE_NAME, compactionPolicy);
- if (compactionPolicyEntity == null) {
- throw new AsterixException("Unknown compaction policy: " + compactionPolicy);
- }
- String compactionPolicyFactoryClassName = compactionPolicyEntity.getClassName();
- ILSMMergePolicyFactory mergePolicyFactory =
- (ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance();
- if (isExternalDataset && mergePolicyFactory.getName().compareTo("correlated-prefix") == 0) {
- throw new AsterixException("The correlated-prefix merge policy cannot be used with external dataset.");
- }
- if (compactionPolicyProperties == null) {
- if (mergePolicyFactory.getName().compareTo("no-merge") != 0) {
- throw new AsterixException("Compaction policy properties are missing.");
- }
- } else {
- for (Map.Entry<String, String> entry : compactionPolicyProperties.entrySet()) {
- if (!mergePolicyFactory.getPropertiesNames().contains(entry.getKey())) {
- throw new AsterixException("Invalid compaction policy property: " + entry.getKey());
- }
- }
- for (String p : mergePolicyFactory.getPropertiesNames()) {
- if (!compactionPolicyProperties.containsKey(p)) {
- throw new AsterixException("Missing compaction policy property: " + p);
- }
- }
- }
- }
-
- private void handleCreateDatasetStatement(AqlMetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc) throws AsterixException, Exception {
-
- ProgressState progress = ProgressState.NO_PROGRESS;
- DatasetDecl dd = (DatasetDecl) stmt;
- String dataverseName = getActiveDataverse(dd.getDataverse());
- String datasetName = dd.getName().getValue();
- DatasetType dsType = dd.getDatasetType();
- String itemTypeDataverseName = dd.getItemTypeDataverse().getValue();
- String itemTypeName = dd.getItemTypeName().getValue();
- String metaItemTypeDataverseName = dd.getMetaItemTypeDataverse().getValue();
- String metaItemTypeName = dd.getMetaItemTypeName().getValue();
- Identifier ngNameId = dd.getNodegroupName();
- String nodegroupName = getNodeGroupName(ngNameId, dd, dataverseName);
- String compactionPolicy = dd.getCompactionPolicy();
- Map<String, String> compactionPolicyProperties = dd.getCompactionPolicyProperties();
- boolean defaultCompactionPolicy = (compactionPolicy == null);
- boolean temp = dd.getDatasetDetailsDecl().isTemp();
-
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- boolean bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- MetadataLockManager.INSTANCE.createDatasetBegin(dataverseName, itemTypeDataverseName,
- itemTypeDataverseName + "." + itemTypeName, metaItemTypeDataverseName,
- metaItemTypeDataverseName + "." + metaItemTypeName, nodegroupName, compactionPolicy,
- dataverseName + "." + datasetName, defaultCompactionPolicy);
- Dataset dataset = null;
- try {
-
- IDatasetDetails datasetDetails = null;
- Dataset ds = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName);
- if (ds != null) {
- if (dd.getIfNotExists()) {
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return;
- } else {
- throw new AlgebricksException("A dataset with this name " + datasetName + " already exists.");
- }
- }
- Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- itemTypeDataverseName, itemTypeName);
- if (dt == null) {
- throw new AlgebricksException(": type " + itemTypeName + " could not be found.");
- }
- String ngName =
- ngNameId != null ? ngNameId.getValue() : configureNodegroupForDataset(dd, dataverseName, mdTxnCtx);
-
- if (compactionPolicy == null) {
- compactionPolicy = GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME;
- compactionPolicyProperties = GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES;
- } else {
- validateCompactionPolicy(compactionPolicy, compactionPolicyProperties, mdTxnCtx, false);
- }
- switch (dd.getDatasetType()) {
- case INTERNAL:
- IAType itemType = dt.getDatatype();
- if (itemType.getTypeTag() != ATypeTag.RECORD) {
- throw new AlgebricksException("Dataset type has to be a record type.");
- }
-
- IAType metaItemType = null;
- if (metaItemTypeDataverseName != null && metaItemTypeName != null) {
- metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
- }
- if (metaItemType != null && metaItemType.getTypeTag() != ATypeTag.RECORD) {
- throw new AlgebricksException("Dataset meta type has to be a record type.");
- }
- ARecordType metaRecType = (ARecordType) metaItemType;
-
- List<List<String>> partitioningExprs =
- ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs();
- List<Integer> keySourceIndicators =
- ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getKeySourceIndicators();
- boolean autogenerated = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated();
- ARecordType aRecordType = (ARecordType) itemType;
- List<IAType> partitioningTypes = ValidateUtil.validatePartitioningExpressions(aRecordType,
- metaRecType, partitioningExprs, keySourceIndicators, autogenerated);
-
- List<String> filterField = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getFilterField();
- if (filterField != null) {
- ValidateUtil.validateFilterField(aRecordType, filterField);
- }
- if (compactionPolicy == null) {
- if (filterField != null) {
- // If the dataset has a filter and the user didn't specify a merge
- // policy, then we will pick the
- // correlated-prefix as the default merge policy.
- compactionPolicy = GlobalConfig.DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME;
- compactionPolicyProperties = GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES;
- }
- }
- datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
- InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
- keySourceIndicators, partitioningTypes, autogenerated, filterField, temp);
- break;
- case EXTERNAL:
- String adapter = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getAdapter();
- Map<String, String> properties = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getProperties();
-
- datasetDetails = new ExternalDatasetDetails(adapter, properties, new Date(),
- ExternalDatasetTransactionState.COMMIT);
- break;
- default:
- throw new AsterixException("Unknown datatype " + dd.getDatasetType());
- }
-
- // #. initialize DatasetIdFactory if it is not initialized.
- if (!DatasetIdFactory.isInitialized()) {
- DatasetIdFactory.initialize(MetadataManager.INSTANCE.getMostRecentDatasetId());
- }
-
- // #. add a new dataset with PendingAddOp
- dataset = new Dataset(dataverseName, datasetName, itemTypeDataverseName, itemTypeName,
- metaItemTypeDataverseName, metaItemTypeName, ngName, compactionPolicy, compactionPolicyProperties,
- datasetDetails, dd.getHints(), dsType, DatasetIdFactory.generateDatasetId(),
- IMetadataEntity.PENDING_ADD_OP);
- MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset);
-
- if (dd.getDatasetType() == DatasetType.INTERNAL) {
- Dataverse dataverse =
- MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dataverseName);
- JobSpecification jobSpec =
- DatasetOperations.createDatasetJobSpec(dataverse, datasetName, metadataProvider);
-
- // #. make metadataTxn commit before calling runJob.
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
-
- // #. runJob
- JobUtils.runJob(hcc, jobSpec, true);
-
- // #. begin new metadataTxn
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- }
-
- // #. add a new dataset with PendingNoOp after deleting the dataset with PendingAddOp
- MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName);
- dataset.setPendingOp(IMetadataEntity.PENDING_NO_OP);
- MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e) {
- if (bActiveTxn) {
- abort(e, e, mdTxnCtx);
- }
-
- if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
-
- // #. execute compensation operations
- // remove the index in NC
- // [Notice]
- // As long as we updated(and committed) metadata, we should remove any effect of the job
- // because an exception occurs during runJob.
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
- try {
- JobSpecification jobSpec = DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- JobUtils.runJob(hcc, jobSpec, true);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- if (bActiveTxn) {
- abort(e, e2, mdTxnCtx);
- }
- }
-
- // remove the record from the metadata.
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
- MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is inconsistent state: pending dataset(" + dataverseName
- + "." + datasetName + ") couldn't be removed from the metadata", e);
- }
- }
-
- throw e;
- } finally {
- MetadataLockManager.INSTANCE.createDatasetEnd(dataverseName, itemTypeDataverseName,
- itemTypeDataverseName + "." + itemTypeName, metaItemTypeDataverseName,
- metaItemTypeDataverseName + "." + metaItemTypeName, nodegroupName, compactionPolicy,
- dataverseName + "." + datasetName, defaultCompactionPolicy);
- }
- }
-
- private void validateIfResourceIsActiveInFeed(String dataverseName, String datasetName) throws AsterixException {
- StringBuilder builder = null;
- IActiveEntityEventsListener[] listeners = ActiveJobNotificationHandler.INSTANCE.getEventListeners();
- for (IActiveEntityEventsListener listener : listeners) {
- if (listener instanceof FeedEventsListener
- && ((FeedEventsListener) listener).isConnectedToDataset(datasetName)) {
- if (builder == null) {
- builder = new StringBuilder();
- }
- builder.append(listener.getEntityId() + "\n");
- }
- }
- if (builder != null) {
- throw new AsterixException("Dataset " + dataverseName + "." + datasetName + " is currently being "
- + "fed into by the following feed(s).\n" + builder.toString() + "\n" + "Operation not supported");
- }
- }
-
- private String getNodeGroupName(Identifier ngNameId, DatasetDecl dd, String dataverse) {
- if (ngNameId != null) {
- return ngNameId.getValue();
- }
- String hintValue = dd.getHints().get(DatasetNodegroupCardinalityHint.NAME);
- if (hintValue == null) {
- return MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME;
- } else {
- return dataverse + ":" + dd.getName().getValue();
- }
- }
-
- private String configureNodegroupForDataset(DatasetDecl dd, String dataverse, MetadataTransactionContext mdTxnCtx)
- throws AsterixException {
- int nodegroupCardinality;
- String nodegroupName;
- String hintValue = dd.getHints().get(DatasetNodegroupCardinalityHint.NAME);
- if (hintValue == null) {
- nodegroupName = MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME;
- return nodegroupName;
- } else {
- int numChosen = 0;
- boolean valid = DatasetHints.validate(DatasetNodegroupCardinalityHint.NAME,
- dd.getHints().get(DatasetNodegroupCardinalityHint.NAME)).first;
- if (!valid) {
- throw new AsterixException("Incorrect use of hint:" + DatasetNodegroupCardinalityHint.NAME);
- } else {
- nodegroupCardinality = Integer.parseInt(dd.getHints().get(DatasetNodegroupCardinalityHint.NAME));
- }
- List<String> nodeNames = AsterixAppContextInfo.getInstance().getMetadataProperties().getNodeNames();
- List<String> nodeNamesClone = new ArrayList<String>(nodeNames);
- String metadataNodeName = AsterixAppContextInfo.getInstance().getMetadataProperties().getMetadataNodeName();
- List<String> selectedNodes = new ArrayList<String>();
- selectedNodes.add(metadataNodeName);
- numChosen++;
- nodeNamesClone.remove(metadataNodeName);
-
- if (numChosen < nodegroupCardinality) {
- Random random = new Random();
- String[] nodes = nodeNamesClone.toArray(new String[] {});
- int[] b = new int[nodeNamesClone.size()];
- for (int i = 0; i < b.length; i++) {
- b[i] = i;
- }
-
- for (int i = 0; i < nodegroupCardinality - numChosen; i++) {
- int selected = i + random.nextInt(nodeNamesClone.size() - i);
- int selNodeIndex = b[selected];
- selectedNodes.add(nodes[selNodeIndex]);
- int temp = b[0];
- b[0] = b[selected];
- b[selected] = temp;
- }
- }
- nodegroupName = dataverse + ":" + dd.getName().getValue();
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodegroupName, selectedNodes));
- return nodegroupName;
- }
-
- }
-
- private void handleCreateIndexStatement(AqlMetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc) throws Exception {
- ProgressState progress = ProgressState.NO_PROGRESS;
- CreateIndexStatement stmtCreateIndex = (CreateIndexStatement) stmt;
- String dataverseName = getActiveDataverse(stmtCreateIndex.getDataverseName());
- String datasetName = stmtCreateIndex.getDatasetName().getValue();
- List<Integer> keySourceIndicators = stmtCreateIndex.getFieldSourceIndicators();
-
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- boolean bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- MetadataLockManager.INSTANCE.createIndexBegin(dataverseName, dataverseName + "." + datasetName);
-
- String indexName = null;
- JobSpecification spec = null;
- Dataset ds = null;
- // For external datasets
- ArrayList<ExternalFile> externalFilesSnapshot = null;
- boolean firstExternalDatasetIndex = false;
- boolean filesIndexReplicated = false;
- Index filesIndex = null;
- boolean datasetLocked = false;
- try {
- ds = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName);
- if (ds == null) {
- throw new AlgebricksException(
- "There is no dataset with this name " + datasetName + " in dataverse " + dataverseName);
- }
-
- indexName = stmtCreateIndex.getIndexName().getValue();
- Index idx = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName, indexName);
- Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- ds.getItemTypeDataverseName(), ds.getItemTypeName());
- ARecordType aRecordType = (ARecordType) dt.getDatatype();
- ARecordType metaRecordType = null;
- if (ds.hasMetaPart()) {
- Datatype metaDt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
- metaRecordType = (ARecordType) metaDt.getDatatype();
- }
-
- List<List<String>> indexFields = new ArrayList<List<String>>();
- List<IAType> indexFieldTypes = new ArrayList<IAType>();
- int keyIndex = 0;
- for (Pair<List<String>, TypeExpression> fieldExpr : stmtCreateIndex.getFieldExprs()) {
- IAType fieldType = null;
- ARecordType subType =
- KeyFieldTypeUtils.chooseSource(keySourceIndicators, keyIndex, aRecordType, metaRecordType);
- boolean isOpen = subType.isOpen();
- int i = 0;
- if (fieldExpr.first.size() > 1 && !isOpen) {
- for (; i < fieldExpr.first.size() - 1;) {
- subType = (ARecordType) subType.getFieldType(fieldExpr.first.get(i));
- i++;
- if (subType.isOpen()) {
- isOpen = true;
- break;
- } ;
- }
- }
- if (fieldExpr.second == null) {
- fieldType = subType.getSubFieldType(fieldExpr.first.subList(i, fieldExpr.first.size()));
- } else {
- if (!stmtCreateIndex.isEnforced()) {
- throw new AlgebricksException("Cannot create typed index on \"" + fieldExpr.first
- + "\" field without enforcing it's type");
- }
- if (!isOpen) {
- throw new AlgebricksException("Typed index on \"" + fieldExpr.first
- + "\" field could be created only for open datatype");
- }
- if (stmtCreateIndex.hasMetaField()) {
- throw new AlgebricksException("Typed open index can only be created on the record part");
- }
- Map<TypeSignature, IAType> typeMap =
- TypeTranslator.computeTypes(mdTxnCtx, fieldExpr.second, indexName, dataverseName);
- TypeSignature typeSignature = new TypeSignature(dataverseName, indexName);
- fieldType = typeMap.get(typeSignature);
- }
- if (fieldType == null) {
- throw new AlgebricksException(
- "Unknown type " + (fieldExpr.second == null ? fieldExpr.first : fieldExpr.second));
- }
-
- indexFields.add(fieldExpr.first);
- indexFieldTypes.add(fieldType);
- ++keyIndex;
- }
-
- ValidateUtil.validateKeyFields(aRecordType, metaRecordType, indexFields, keySourceIndicators,
- indexFieldTypes, stmtCreateIndex.getIndexType());
-
- if (idx != null) {
- if (stmtCreateIndex.getIfNotExists()) {
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return;
- } else {
- throw new AlgebricksException("An index with this name " + indexName + " already exists.");
- }
- }
-
- // Checks whether a user is trying to create an inverted secondary index on a dataset
- // with a variable-length primary key.
- // Currently, we do not support this. Therefore, as a temporary solution, we print an
- // error message and stop.
- if (stmtCreateIndex.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
- || stmtCreateIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX
- || stmtCreateIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
- || stmtCreateIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
- List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(ds);
- for (List<String> partitioningKey : partitioningKeys) {
- IAType keyType = aRecordType.getSubFieldType(partitioningKey);
- ITypeTraits typeTrait = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
-
- // If it is not a fixed length
- if (typeTrait.getFixedLength() < 0) {
- throw new AlgebricksException("The keyword or ngram index -" + indexName
- + " cannot be created on the dataset -" + datasetName
- + " due to its variable-length primary key field - " + partitioningKey);
- }
-
- }
- }
-
- if (ds.getDatasetType() == DatasetType.INTERNAL) {
- validateIfResourceIsActiveInFeed(dataverseName, datasetName);
- } else {
- // External dataset
- // Check if the dataset is indexible
- if (!ExternalIndexingOperations.isIndexible((ExternalDatasetDetails) ds.getDatasetDetails())) {
- throw new AlgebricksException(
- "dataset using " + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
- + " Adapter can't be indexed");
- }
- // Check if the name of the index is valid
- if (!ExternalIndexingOperations.isValidIndexName(datasetName, indexName)) {
- throw new AlgebricksException("external dataset index name is invalid");
- }
-
- // Check if the files index exist
- filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
- firstExternalDatasetIndex = (filesIndex == null);
- // Lock external dataset
- ExternalDatasetsRegistry.INSTANCE.buildIndexBegin(ds, firstExternalDatasetIndex);
- datasetLocked = true;
- if (firstExternalDatasetIndex) {
- // Verify that no one has created an index before we acquire the lock
- filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
- dataverseName, datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
- if (filesIndex != null) {
- ExternalDatasetsRegistry.INSTANCE.buildIndexEnd(ds, firstExternalDatasetIndex);
- firstExternalDatasetIndex = false;
- ExternalDatasetsRegistry.INSTANCE.buildIndexBegin(ds, firstExternalDatasetIndex);
- }
- }
- if (firstExternalDatasetIndex) {
- // Get snapshot from External File System
- externalFilesSnapshot = ExternalIndexingOperations.getSnapshotFromExternalFileSystem(ds);
- // Add an entry for the files index
- filesIndex = new Index(dataverseName, datasetName,
- ExternalIndexingOperations.getFilesIndexName(datasetName), IndexType.BTREE,
- ExternalIndexingOperations.FILE_INDEX_FIELD_NAMES, null,
- ExternalIndexingOperations.FILE_INDEX_FIELD_TYPES, false, false,
- IMetadataEntity.PENDING_ADD_OP);
- MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
- // Add files to the external files index
- for (ExternalFile file : externalFilesSnapshot) {
- MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
- }
- // This is the first index for the external dataset, replicate the files index
- spec = ExternalIndexingOperations.buildFilesIndexReplicationJobSpec(ds, externalFilesSnapshot,
- metadataProvider, true);
- if (spec == null) {
- throw new AsterixException(
- "Failed to create job spec for replicating Files Index For external dataset");
- }
- filesIndexReplicated = true;
- JobUtils.runJob(hcc, spec, true);
- }
- }
-
- // check whether there exists another enforced index on the same field
- if (stmtCreateIndex.isEnforced()) {
- List<Index> indexes = MetadataManager.INSTANCE
- .getDatasetIndexes(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName);
- for (Index index : indexes) {
- if (index.getKeyFieldNames().equals(indexFields)
- && !index.getKeyFieldTypes().equals(indexFieldTypes) && index.isEnforcingKeyFileds()) {
- throw new AsterixException("Cannot create index " + indexName + " , enforced index "
- + index.getIndexName() + " on field \"" + StringUtils.join(indexFields, ',')
- + "\" is already defined with type \"" + index.getKeyFieldTypes() + "\"");
- }
- }
- }
-
- // #. add a new index with PendingAddOp
- Index index = new Index(dataverseName, datasetName, indexName, stmtCreateIndex.getIndexType(), indexFields,
- keySourceIndicators, indexFieldTypes, stmtCreateIndex.getGramLength(), stmtCreateIndex.isEnforced(),
- false, IMetadataEntity.PENDING_ADD_OP);
- MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
-
- ARecordType enforcedType = null;
- if (stmtCreateIndex.isEnforced()) {
- enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType,
- Lists.newArrayList(index));
- }
-
- // #. prepare to create the index artifact in NC.
- CompiledCreateIndexStatement cis = new CompiledCreateIndexStatement(index.getIndexName(), dataverseName,
- index.getDatasetName(), index.getKeyFieldNames(), index.getKeyFieldTypes(),
- index.isEnforcingKeyFileds(), index.getGramLength(), index.getIndexType());
- spec = IndexOperations.buildSecondaryIndexCreationJobSpec(cis, aRecordType, metaRecordType,
- keySourceIndicators, enforcedType, metadataProvider);
- if (spec == null) {
- throw new AsterixException("Failed to create job spec for creating index '"
- + stmtCreateIndex.getDatasetName() + "." + stmtCreateIndex.getIndexName() + "'");
- }
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
-
- progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
-
- // #. create the index artifact in NC.
- JobUtils.runJob(hcc, spec, true);
-
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- // #. load data into the index in NC.
- cis = new CompiledCreateIndexStatement(index.getIndexName(), dataverseName, index.getDatasetName(),
- index.getKeyFieldNames(), index.getKeyFieldTypes(), index.isEnforcingKeyFileds(),
- index.getGramLength(), index.getIndexType());
-
- spec = IndexOperations.buildSecondaryIndexLoadingJobSpec(cis, aRecordType, metaRecordType,
- keySourceIndicators, enforcedType, metadataProvider);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
-
- JobUtils.runJob(hcc, spec, true);
-
- // #. begin new metadataTxn
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- // #. add another new index with PendingNoOp after deleting the index with PendingAddOp
- MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName,
- indexName);
- index.setPendingOp(IMetadataEntity.PENDING_NO_OP);
- MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
- // add another new files index with PendingNoOp after deleting the index with
- // PendingAddOp
- if (firstExternalDatasetIndex) {
- MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName,
- filesIndex.getIndexName());
- filesIndex.setPendingOp(IMetadataEntity.PENDING_NO_OP);
- MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
- // update transaction timestamp
- ((ExternalDatasetDetails) ds.getDatasetDetails()).setRefreshTimestamp(new Date());
- MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
- }
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
-
- } catch (Exception e) {
- if (bActiveTxn) {
- abort(e, e, mdTxnCtx);
- }
- // If files index was replicated for external dataset, it should be cleaned up on NC side
- if (filesIndexReplicated) {
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
- ExternalIndexingOperations.getFilesIndexName(datasetName));
- try {
- JobSpecification jobSpec =
- ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds, metadataProvider, ds);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- JobUtils.runJob(hcc, jobSpec, true);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- if (bActiveTxn) {
- abort(e, e2, mdTxnCtx);
- }
- }
- }
-
- if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
- // #. execute compensation operations
- // remove the index in NC
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName, indexName);
- try {
- JobSpecification jobSpec =
- IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds);
-
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- JobUtils.runJob(hcc, jobSpec, true);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- if (bActiveTxn) {
- abort(e, e2, mdTxnCtx);
- }
- }
-
- if (firstExternalDatasetIndex) {
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
- // Drop External Files from metadata
- MetadataManager.INSTANCE.dropDatasetExternalFiles(mdTxnCtx, ds);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is inconsistent state: pending files for("
- + dataverseName + "." + datasetName + ") couldn't be removed from the metadata", e);
- }
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
- // Drop the files index from metadata
- MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is inconsistent state: pending index(" + dataverseName
- + "." + datasetName + "." + ExternalIndexingOperations.getFilesIndexName(datasetName)
- + ") couldn't be removed from the metadata", e);
- }
- }
- // remove the record from the metadata.
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
- MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName, indexName);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is in inconsistent state: pending index(" + dataverseName
- + "." + datasetName + "." + indexName + ") couldn't be removed from the metadata", e);
- }
- }
- throw e;
- } finally {
- MetadataLockManager.INSTANCE.createIndexEnd(dataverseName, dataverseName + "." + datasetName);
- if (datasetLocked) {
- ExternalDatasetsRegistry.INSTANCE.buildIndexEnd(ds, firstExternalDatasetIndex);
- }
- }
- }
-
- private void handleCreateTypeStatement(AqlMetadataProvider metadataProvider, Statement stmt) throws Exception {
- TypeDecl stmtCreateType = (TypeDecl) stmt;
- String dataverseName = getActiveDataverse(stmtCreateType.getDataverseName());
- String typeName = stmtCreateType.getIdent().getValue();
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- MetadataLockManager.INSTANCE.createTypeBegin(dataverseName, dataverseName + "." + typeName);
- try {
-
- Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
- if (dv == null) {
- throw new AlgebricksException("Unknown dataverse " + dataverseName);
- }
- Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataverseName, typeName);
- if (dt != null) {
- if (!stmtCreateType.getIfNotExists()) {
- throw new AlgebricksException("A datatype with this name " + typeName + " already exists.");
- }
- } else {
- if (builtinTypeMap.get(typeName) != null) {
- throw new AlgebricksException("Cannot redefine builtin type " + typeName + ".");
- } else {
- Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx,
- stmtCreateType.getTypeDef(), stmtCreateType.getIdent().getValue(), dataverseName);
- TypeSignature typeSignature = new TypeSignature(dataverseName, typeName);
- IAType type = typeMap.get(typeSignature);
- MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(dataverseName, typeName, type, false));
- }
- }
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e) {
- abort(e, e, mdTxnCtx);
- throw e;
- } finally {
- MetadataLockManager.INSTANCE.createTypeEnd(dataverseName, dataverseName + "." + typeName);
- }
- }
-
- private void handleDataverseDropStatement(AqlMetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc) throws Exception {
- DataverseDropStatement stmtDelete = (DataverseDropStatement) stmt;
- String dataverseName = stmtDelete.getDataverseName().getValue();
-
- ProgressState progress = ProgressState.NO_PROGRESS;
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- boolean bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- MetadataLockManager.INSTANCE.acquireDataverseWriteLock(dataverseName);
- List<JobSpecification> jobsToExecute = new ArrayList<JobSpecification>();
- try {
- Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
- if (dv == null) {
- if (stmtDelete.getIfExists()) {
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return;
- } else {
- throw new AlgebricksException("There is no dataverse with this name " + dataverseName + ".");
- }
- }
- // # disconnect all feeds from any datasets in the dataverse.
- IActiveEntityEventsListener[] activeListeners = ActiveJobNotificationHandler.INSTANCE.getEventListeners();
- Identifier dvId = new Identifier(dataverseName);
- for (IActiveEntityEventsListener listener : activeListeners) {
- EntityId activeEntityId = listener.getEntityId();
- if (activeEntityId.getExtensionName().equals(Feed.EXTENSION_NAME)
- && activeEntityId.getDataverse().equals(dataverseName)) {
- FeedEventsListener feedEventListener = (FeedEventsListener) listener;
- FeedConnectionId[] connections = feedEventListener.getConnections();
- for (FeedConnectionId conn : connections) {
- disconnectFeedBeforeDelete(dvId, activeEntityId, conn, metadataProvider, hcc);
- }
- // prepare job to remove feed log storage
- jobsToExecute.add(FeedOperations.buildRemoveFeedStorageJob(
- MetadataManager.INSTANCE.getFeed(mdTxnCtx, dataverseName, activeEntityId.getEntityName())));
- }
- }
-
- // #. prepare jobs which will drop corresponding datasets with indexes.
- List<Dataset> datasets = MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverseName);
- for (int j = 0; j < datasets.size(); j++) {
- String datasetName = datasets.get(j).getDatasetName();
- DatasetType dsType = datasets.get(j).getDatasetType();
- if (dsType == DatasetType.INTERNAL) {
- List<Index> indexes =
- MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
- for (int k = 0; k < indexes.size(); k++) {
- if (indexes.get(k).isSecondaryIndex()) {
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
- indexes.get(k).getIndexName());
- jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider,
- datasets.get(j)));
- }
- }
-
- CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
- jobsToExecute.add(DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider));
- } else {
- // External dataset
- List<Index> indexes =
- MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
- for (int k = 0; k < indexes.size(); k++) {
- if (ExternalIndexingOperations.isFileIndex(indexes.get(k))) {
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
- indexes.get(k).getIndexName());
- jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
- metadataProvider, datasets.get(j)));
- } else {
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
- indexes.get(k).getIndexName());
- jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider,
- datasets.get(j)));
- }
- }
- ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(datasets.get(j));
- }
- }
- jobsToExecute.add(DataverseOperations.createDropDataverseJobSpec(dv, metadataProvider));
- // #. mark PendingDropOp on the dataverse record by
- // first, deleting the dataverse record from the DATAVERSE_DATASET
- // second, inserting the dataverse record with the PendingDropOp value into the
- // DATAVERSE_DATASET
- MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverseName);
- MetadataManager.INSTANCE.addDataverse(mdTxnCtx,
- new Dataverse(dataverseName, dv.getDataFormat(), IMetadataEntity.PENDING_DROP_OP));
-
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
-
- for (JobSpecification jobSpec : jobsToExecute) {
- JobUtils.runJob(hcc, jobSpec, true);
- }
-
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- // #. finally, delete the dataverse.
- MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverseName);
- if (activeDefaultDataverse != null && activeDefaultDataverse.getDataverseName() == dataverseName) {
- activeDefaultDataverse = null;
- }
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e) {
- if (bActiveTxn) {
- abort(e, e, mdTxnCtx);
- }
-
- if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
- if (activeDefaultDataverse != null && activeDefaultDataverse.getDataverseName() == dataverseName) {
- activeDefaultDataverse = null;
- }
-
- // #. execute compensation operations
- // remove the all indexes in NC
- try {
- for (JobSpecification jobSpec : jobsToExecute) {
- JobUtils.runJob(hcc, jobSpec, true);
- }
- } catch (Exception e2) {
- // do no throw exception since still the metadata needs to be compensated.
- e.addSuppressed(e2);
- }
-
- // remove the record from the metadata.
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- try {
- MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverseName);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is inconsistent state: pending dataverse(" + dataverseName
- + ") couldn't be removed from the metadata", e);
- }
- }
-
- throw e;
- } finally {
- MetadataLockManager.INSTANCE.releaseDataverseWriteLock(dataverseName);
- }
- }
-
- private void disconnectFeedBeforeDelete(Identifier dvId, EntityId activeEntityId, FeedConnectionId conn,
- AqlMetadataProvider metadataProvider, IHyracksClientConnection hcc) {
- DisconnectFeedStatement disStmt = new DisconnectFeedStatement(dvId,
- new Identifier(activeEntityId.getEntityName()), new Identifier(conn.getDatasetName()));
- try {
- handleDisconnectFeedStatement(metadataProvider, disStmt, hcc);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Disconnected feed " + activeEntityId.getEntityName() + " from dataset "
- + conn.getDatasetName());
- }
- } catch (Exception exception) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Unable to disconnect feed " + activeEntityId.getEntityName() + " from dataset "
- + conn.getDatasetName() + ". Encountered exception " + exception);
- }
- }
- }
-
- private void handleDatasetDropStatement(AqlMetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc) throws Exception {
- DropStatement stmtDelete = (DropStatement) stmt;
- String dataverseName = getActiveDataverse(stmtDelete.getDataverseName());
- String datasetName = stmtDelete.getDatasetName().getValue();
-
- ProgressState progress = ProgressState.NO_PROGRESS;
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- boolean bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- MetadataLockManager.INSTANCE.dropDatasetBegin(dataverseName, dataverseName + "." + datasetName);
- List<JobSpecification> jobsToExecute = new ArrayList<JobSpecification>();
- try {
-
- Dataset ds = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
- if (ds == null) {
- if (stmtDelete.getIfExists()) {
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return;
- } else {
- throw new AlgebricksException("There is no dataset with this name " + datasetName + " in dataverse "
- + dataverseName + ".");
- }
- }
-
- Map<FeedConnectionId, Pair<JobSpecification, Boolean>> disconnectJobList = new HashMap<>();
- if (ds.getDatasetType() == DatasetType.INTERNAL) {
- // prepare job spec(s) that would disconnect any active feeds involving the dataset.
- IActiveEntityEventsListener[] feedConnections =
- ActiveJobNotificationHandler.INSTANCE.getEventListeners();
- for (IActiveEntityEventsListener conn : feedConnections) {
- if (conn.getEntityId().getExtensionName().equals(Feed.EXTENSION_NAME)
- && ((FeedEventsListener) conn).isConnectedToDataset(datasetName)) {
- FeedConnectionId connectionId = new FeedConnectionId(conn.getEntityId(), datasetName);
- Pair<JobSpecification, Boolean> p =
- FeedOperations.buildDisconnectFeedJobSpec(metadataProvider, connectionId);
- disconnectJobList.put(connectionId, p);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Disconnecting feed " + connectionId.getFeedId().getEntityName()
- + " from dataset " + datasetName + " as dataset is being dropped");
- }
- // prepare job to remove feed log storage
- jobsToExecute.add(FeedOperations.buildRemoveFeedStorageJob(
- MetadataManager.INSTANCE.getFeed(mdTxnCtx, connectionId.getFeedId().getDataverse(),
- connectionId.getFeedId().getEntityName())));
- }
- }
-
- // #. prepare jobs to drop the datatset and the indexes in NC
- List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
- for (int j = 0; j < indexes.size(); j++) {
- if (indexes.get(j).isSecondaryIndex()) {
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
- indexes.get(j).getIndexName());
- jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
- }
- }
- CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
- jobsToExecute.add(DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider));
-
- // #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
- MetadataManager.INSTANCE.addDataset(mdTxnCtx,
- new Dataset(dataverseName, datasetName, ds.getItemTypeDataverseName(), ds.getItemTypeName(),
- ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName(), ds.getNodeGroupName(),
- ds.getCompactionPolicy(), ds.getCompactionPolicyProperties(), ds.getDatasetDetails(),
- ds.getHints(), ds.getDatasetType(), ds.getDatasetId(),
- IMetadataEntity.PENDING_DROP_OP));
-
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
-
- // # disconnect the feeds
- for (Pair<JobSpecification, Boolean> p : disconnectJobList.values()) {
- JobUtils.runJob(hcc, p.first, true);
- }
-
- // #. run the jobs
- for (JobSpecification jobSpec : jobsToExecute) {
- JobUtils.runJob(hcc, jobSpec, true);
- }
-
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- } else {
- // External dataset
- ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
- // #. prepare jobs to drop the datatset and the indexes in NC
- List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
- for (int j = 0; j < indexes.size(); j++) {
- if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
- indexes.get(j).getIndexName());
- jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
- } else {
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
- indexes.get(j).getIndexName());
- jobsToExecute
- .add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds, metadataProvider, ds));
- }
- }
-
- // #. mark the existing dataset as PendingDropOp
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
- MetadataManager.INSTANCE.addDataset(mdTxnCtx,
- new Dataset(dataverseName, datasetName, ds.getItemTypeDataverseName(), ds.getItemTypeName(),
- ds.getNodeGroupName(), ds.getCompactionPolicy(), ds.getCompactionPolicyProperties(),
- ds.getDatasetDetails(), ds.getHints(), ds.getDatasetType(), ds.getDatasetId(),
- IMetadataEntity.PENDING_DROP_OP));
-
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
- progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
-
- // #. run the jobs
- for (JobSpecification jobSpec : jobsToExecute) {
- JobUtils.runJob(hcc, jobSpec, true);
- }
- if (indexes.size() > 0) {
- ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
- }
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- }
-
- // #. finally, delete the dataset.
- MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
- // Drop the associated nodegroup
- String nodegroup = ds.getNodeGroupName();
- if (!nodegroup.equalsIgnoreCase(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)) {
- MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, dataverseName + ":" + datasetName);
- }
-
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e) {
- if (bActiveTxn) {
- abort(e, e, mdTxnCtx);
- }
-
- if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
- // #. execute compensation operations
- // remove the all indexes in NC
- try {
- for (JobSpecification jobSpec : jobsToExecute) {
- JobUtils.runJob(hcc, jobSpec, true);
- }
- } catch (Exception e2) {
- // do no throw exception since still the metadata needs to be compensated.
- e.addSuppressed(e2);
- }
-
- // remove the record from the metadata.
- mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- try {
- MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
- datasetName);
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- } catch (Exception e2) {
- e.addSuppressed(e2);
- abort(e, e2, mdTxnCtx);
- throw new IllegalStateException("System is inconsistent state: pending dataset(" + dataverseName
- + "." + datasetName + ") couldn't be removed from the metadata", e);
- }
- }
-
- throw e;
- } finally {
- MetadataLockManager.INSTANCE.dropDatasetEnd(dataverseName, dataverseName + "." + datasetName);
- }
- }
-
- private void handleIndexDropStatement(AqlMetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc) throws Exception {
-
- IndexDropStatement stmtIndexDrop = (IndexDropStatement) stmt;
- String datasetName = stmtIndexDrop.getDatasetName().getValue();
- String dataverseName = getActiveDataverse(stmtIndexDrop.getDataverseName());
- ProgressState progress = ProgressState.NO_PROGRESS;
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- boolean bActiveTxn = true;
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
-
- MetadataLockManager.INSTANCE.dropIndexBegin(dataverseName, dataverseName + "." + datasetName);
-
- String indexName = null;
- // For external index
- boolean dropFilesIndex = false;
- List<JobSpecification> jobsToExecute = new ArrayList<JobSpecification>();
- try {
-
- Dataset ds = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
- if (ds == null) {
- throw new AlgebricksException(
- "There is no dataset with this name " + datasetName + " in dataverse " + dataverseName);
- }
- IActiveEntityEventsListener[] listeners = ActiveJobNotificationHandler.INSTANCE.getEventListeners();
- StringBuilder builder = null;
- for (IActiveEntityEventsListener listener : listeners) {
- if (listener.getEntityId().getExtensionName().equals(Feed.EXTENSION_NAME)
- && ((FeedEventsListener) listener).isConnectedToDataset(datasetName)) {
- if (builder == null) {
- builder = new StringBuilder();
- }
- builder.append(new FeedConnectionId(listener.getEntityId(), datasetName) + "\n");
- }
- }
- if (builder != null) {
- throw new AsterixException(
- "Dataset" + datasetName + " is currently being fed into by the following feeds " + "."
- + builder.toString() + "\nOperation not supported.");
- }
-
- if (ds.getDatasetType() == DatasetType.INTERNAL) {
- indexName = stmtIndexDrop.getIndexName().getValue();
- Index index = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, indexName);
- if (index == null) {
- if (stmtIndexDrop.getIfExists()) {
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return;
- } else {
- throw new AlgebricksException("There is no index with this name " + indexName + ".");
- }
- }
- // #. prepare a job to drop the index in NC.
- CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName, indexName);
- jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
-
- // #. mark PendingDropOp on the existing index
- MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
- MetadataManager.INSTANCE.addIndex(mdTxnCtx,
- new Index(dataverseName, datasetName, indexName, index.getIndexType(), index.getKeyFieldNames(),
- index.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(),
- index.isEnforcingKeyFileds(), index.isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
-
- // #. commit the existing transaction before calling runJob.
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- bActiveTxn = false;
-
<TRUNCATED>
[12/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/IStatementExecutorExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/IStatementExecutorExtension.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/IStatementExecutorExtension.java
new file mode 100644
index 0000000..a4b2345
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/IStatementExecutorExtension.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.cc;
+
+import org.apache.asterix.common.api.IExtension;
+import org.apache.asterix.translator.IStatementExecutorFactory;
+
+/**
+ * An interface for extensions of {@code IStatementExecutor}
+ */
+public interface IStatementExecutorExtension extends IExtension {
+ @Override
+ default ExtensionKind getExtensionKind() {
+ return ExtensionKind.STATEMENT_EXECUTOR;
+ }
+
+ /**
+ * @return The extension implementation of the {@code IStatementExecutorFactory}
+ */
+ IStatementExecutorFactory getQueryTranslatorFactory();
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ActiveLifecycleListener.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ActiveLifecycleListener.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ActiveLifecycleListener.java
index c44ffc1..fe15ce8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ActiveLifecycleListener.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ActiveLifecycleListener.java
@@ -26,7 +26,8 @@ import java.util.logging.Logger;
import org.apache.asterix.active.ActiveEvent;
import org.apache.asterix.active.ActiveJobNotificationHandler;
-import org.apache.asterix.active.ActivePartitionMessage;
+import org.apache.asterix.active.EntityId;
+import org.apache.asterix.active.message.ActivePartitionMessage;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.job.IActivityClusterGraphGeneratorFactory;
import org.apache.hyracks.api.job.IJobLifecycleListener;
@@ -48,15 +49,17 @@ public class ActiveLifecycleListener implements IJobLifecycleListener {
@Override
public synchronized void notifyJobStart(JobId jobId) throws HyracksException {
- if (ActiveJobNotificationHandler.INSTANCE.isActiveJob(jobId)) {
- jobEventInbox.add(new ActiveEvent(jobId, ActiveEvent.EventKind.JOB_START));
+ EntityId entityId = ActiveJobNotificationHandler.INSTANCE.getEntity(jobId);
+ if (entityId != null) {
+ jobEventInbox.add(new ActiveEvent(jobId, ActiveEvent.EventKind.JOB_START, entityId));
}
}
@Override
public synchronized void notifyJobFinish(JobId jobId) throws HyracksException {
- if (ActiveJobNotificationHandler.INSTANCE.isActiveJob(jobId)) {
- jobEventInbox.add(new ActiveEvent(jobId, ActiveEvent.EventKind.JOB_FINISH));
+ EntityId entityId = ActiveJobNotificationHandler.INSTANCE.getEntity(jobId);
+ if (entityId != null) {
+ jobEventInbox.add(new ActiveEvent(jobId, ActiveEvent.EventKind.JOB_FINISH, entityId));
} else {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("NO NEED TO NOTIFY JOB FINISH!");
@@ -72,7 +75,7 @@ public class ActiveLifecycleListener implements IJobLifecycleListener {
public void receive(ActivePartitionMessage message) {
if (ActiveJobNotificationHandler.INSTANCE.isActiveJob(message.getJobId())) {
jobEventInbox.add(new ActiveEvent(message.getJobId(), ActiveEvent.EventKind.PARTITION_EVENT,
- message.getFeedId(), message.getPayload()));
+ message.getActiveRuntimeId().getEntityId(), message));
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
index 92ef062..95fe68c 100755
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalLibraryUtils.java
@@ -265,8 +265,8 @@ public class ExternalLibraryUtils {
String adapterFactoryClass = adapter.getFactoryClass().trim();
String adapterName = libraryName + "#" + adapter.getName().trim();
AdapterIdentifier aid = new AdapterIdentifier(dataverse, adapterName);
- DatasourceAdapter dsa = new DatasourceAdapter(aid, adapterFactoryClass,
- IDataSourceAdapter.AdapterType.EXTERNAL);
+ DatasourceAdapter dsa =
+ new DatasourceAdapter(aid, adapterFactoryClass, IDataSourceAdapter.AdapterType.EXTERNAL);
MetadataManager.INSTANCE.addAdapter(mdTxnCtx, dsa);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Installed adapter: " + adapterName);
@@ -334,8 +334,8 @@ public class ExternalLibraryUtils {
}
// get a reference to the specific library dir
- File libDir = new File(
- installDir.getAbsolutePath() + File.separator + dataverse + File.separator + libraryName);
+ File libDir =
+ new File(installDir.getAbsolutePath() + File.separator + dataverse + File.separator + libraryName);
FilenameFilter jarFileFilter = new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
index ba15fb1..b815602 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
@@ -22,9 +22,10 @@ import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
+import org.apache.asterix.app.translator.QueryTranslator;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.app.SessionConfig.OutputFormat;
import org.apache.asterix.compiler.provider.AqlCompilationProvider;
import org.apache.asterix.compiler.provider.ILangCompilationProvider;
import org.apache.asterix.external.feed.api.IFeedWork;
@@ -36,6 +37,7 @@ import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.om.util.AsterixAppContextInfo;
+import org.apache.asterix.translator.IStatementExecutor;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -68,6 +70,7 @@ public class FeedWorkCollection {
private static class SubscribeFeedWorkRunnable implements Runnable {
+ private static final DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory(null);
private final FeedConnectionRequest request;
private final String[] locations;
@@ -82,13 +85,13 @@ public class FeedWorkCollection {
//TODO(amoudi): route PrintWriter to log file
PrintWriter writer = new PrintWriter(System.err, true);
SessionConfig pc = new SessionConfig(writer, OutputFormat.ADM);
- DataverseDecl dataverseDecl =
- new DataverseDecl(new Identifier(request.getReceivingFeedId().getDataverse()));
+ DataverseDecl dataverseDecl = new DataverseDecl(
+ new Identifier(request.getReceivingFeedId().getDataverse()));
SubscribeFeedStatement subscribeStmt = new SubscribeFeedStatement(locations, request);
List<Statement> statements = new ArrayList<Statement>();
statements.add(dataverseDecl);
statements.add(subscribeStmt);
- QueryTranslator translator = new QueryTranslator(statements, pc, compilationProvider);
+ IStatementExecutor translator = qtFactory.create(statements, pc, compilationProvider);
translator.compileAndExecute(AsterixAppContextInfo.getInstance().getHcc(), null,
QueryTranslator.ResultDelivery.SYNC);
if (LOGGER.isEnabledFor(Level.INFO)) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java
new file mode 100644
index 0000000..3ebe873
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/AsterixNCAppRuntimeContext.java
@@ -0,0 +1,475 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.nc;
+
+import java.io.IOException;
+import java.rmi.RemoteException;
+import java.rmi.server.UnicastRemoteObject;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.asterix.active.ActiveManager;
+import org.apache.asterix.api.common.AsterixAppRuntimeContextProviderForRecovery;
+import org.apache.asterix.common.api.AsterixThreadExecutor;
+import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
+import org.apache.asterix.common.api.IDatasetLifecycleManager;
+import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.asterix.common.config.AsterixBuildProperties;
+import org.apache.asterix.common.config.AsterixCompilerProperties;
+import org.apache.asterix.common.config.AsterixExtensionProperties;
+import org.apache.asterix.common.config.AsterixExternalProperties;
+import org.apache.asterix.common.config.AsterixFeedProperties;
+import org.apache.asterix.common.config.AsterixMetadataProperties;
+import org.apache.asterix.common.config.AsterixPropertiesAccessor;
+import org.apache.asterix.common.config.AsterixReplicationProperties;
+import org.apache.asterix.common.config.AsterixStorageProperties;
+import org.apache.asterix.common.config.AsterixTransactionProperties;
+import org.apache.asterix.common.config.IAsterixPropertiesProvider;
+import org.apache.asterix.common.context.AsterixFileMapManager;
+import org.apache.asterix.common.context.DatasetLifecycleManager;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.common.replication.IRemoteRecoveryManager;
+import org.apache.asterix.common.replication.IReplicaResourcesManager;
+import org.apache.asterix.common.replication.IReplicationChannel;
+import org.apache.asterix.common.replication.IReplicationManager;
+import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
+import org.apache.asterix.common.transactions.IRecoveryManager;
+import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.transactions.ITransactionSubsystem;
+import org.apache.asterix.external.library.ExternalLibraryManager;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataNode;
+import org.apache.asterix.metadata.api.IAsterixStateProxy;
+import org.apache.asterix.metadata.api.IMetadataNode;
+import org.apache.asterix.metadata.bootstrap.MetadataBootstrap;
+import org.apache.asterix.metadata.bootstrap.MetadataIndexImmutableProperties;
+import org.apache.asterix.om.util.AsterixClusterProperties;
+import org.apache.asterix.replication.management.ReplicationChannel;
+import org.apache.asterix.replication.management.ReplicationManager;
+import org.apache.asterix.replication.recovery.RemoteRecoveryManager;
+import org.apache.asterix.replication.storage.ReplicaResourcesManager;
+import org.apache.asterix.transaction.management.resource.GlobalResourceIdFactoryProvider;
+import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepositoryFactory;
+import org.apache.asterix.transaction.management.service.transaction.TransactionSubsystem;
+import org.apache.hyracks.api.application.IApplicationConfig;
+import org.apache.hyracks.api.application.INCApplicationContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
+import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
+import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
+import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
+import org.apache.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
+import org.apache.hyracks.storage.common.buffercache.DelayPageCleanerPolicy;
+import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import org.apache.hyracks.storage.common.buffercache.IPageCleanerPolicy;
+import org.apache.hyracks.storage.common.buffercache.IPageReplacementStrategy;
+import org.apache.hyracks.storage.common.file.IFileMapManager;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
+import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
+import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
+import org.apache.hyracks.storage.common.file.IResourceIdFactory;
+
+public class AsterixNCAppRuntimeContext implements IAsterixAppRuntimeContext, IAsterixPropertiesProvider {
+ private static final Logger LOGGER = Logger.getLogger(AsterixNCAppRuntimeContext.class.getName());
+
+ private ILSMMergePolicyFactory metadataMergePolicyFactory;
+ private final INCApplicationContext ncApplicationContext;
+
+ private AsterixCompilerProperties compilerProperties;
+ private AsterixExternalProperties externalProperties;
+ private AsterixMetadataProperties metadataProperties;
+ private AsterixStorageProperties storageProperties;
+ private AsterixTransactionProperties txnProperties;
+ private AsterixFeedProperties feedProperties;
+ private AsterixBuildProperties buildProperties;
+ private AsterixReplicationProperties replicationProperties;
+
+ private AsterixThreadExecutor threadExecutor;
+ private IDatasetLifecycleManager datasetLifecycleManager;
+ private IFileMapManager fileMapManager;
+ private IBufferCache bufferCache;
+ private ITransactionSubsystem txnSubsystem;
+
+ private ILSMIOOperationScheduler lsmIOScheduler;
+ private PersistentLocalResourceRepository localResourceRepository;
+ private IResourceIdFactory resourceIdFactory;
+ private IIOManager ioManager;
+ private boolean isShuttingdown;
+
+ private ActiveManager activeManager;
+
+ private IReplicationChannel replicationChannel;
+ private IReplicationManager replicationManager;
+ private IRemoteRecoveryManager remoteRecoveryManager;
+ private IReplicaResourcesManager replicaResourcesManager;
+ private final int metadataRmiPort;
+
+ private final ILibraryManager libraryManager;
+ private final NCExtensionManager ncExtensionManager;
+
+ public AsterixNCAppRuntimeContext(INCApplicationContext ncApplicationContext, int metadataRmiPort)
+ throws AsterixException, InstantiationException, IllegalAccessException, ClassNotFoundException,
+ IOException {
+ this.ncApplicationContext = ncApplicationContext;
+ // Determine whether to use old-style asterix-configuration.xml or new-style configuration.
+ // QQQ strip this out eventually
+ AsterixPropertiesAccessor propertiesAccessor;
+ IApplicationConfig cfg = ncApplicationContext.getAppConfig();
+ // QQQ this is NOT a good way to determine whether the config is valid
+ if (cfg.getString("cc", "cluster.address") != null) {
+ propertiesAccessor = new AsterixPropertiesAccessor(cfg);
+ } else {
+ propertiesAccessor = new AsterixPropertiesAccessor();
+ }
+ compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
+ externalProperties = new AsterixExternalProperties(propertiesAccessor);
+ metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
+ storageProperties = new AsterixStorageProperties(propertiesAccessor);
+ txnProperties = new AsterixTransactionProperties(propertiesAccessor);
+ feedProperties = new AsterixFeedProperties(propertiesAccessor);
+ buildProperties = new AsterixBuildProperties(propertiesAccessor);
+ replicationProperties = new AsterixReplicationProperties(propertiesAccessor,
+ AsterixClusterProperties.INSTANCE.getCluster());
+ this.metadataRmiPort = metadataRmiPort;
+ libraryManager = new ExternalLibraryManager();
+ ncExtensionManager = new NCExtensionManager(
+ new AsterixExtensionProperties(propertiesAccessor).getExtensions());
+ }
+
+ @Override
+ public void initialize(boolean initialRun) throws IOException, ACIDException {
+ Logger.getLogger("org.apache").setLevel(externalProperties.getLogLevel());
+
+ threadExecutor = new AsterixThreadExecutor(ncApplicationContext.getThreadFactory());
+ fileMapManager = new AsterixFileMapManager();
+ ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+ IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
+ ioManager = ncApplicationContext.getRootContext().getIOManager();
+ IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator,
+ storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
+
+ AsynchronousScheduler.INSTANCE.init(ncApplicationContext.getThreadFactory());
+ lsmIOScheduler = AsynchronousScheduler.INSTANCE;
+
+ metadataMergePolicyFactory = new PrefixMergePolicyFactory();
+
+ ILocalResourceRepositoryFactory persistentLocalResourceRepositoryFactory =
+ new PersistentLocalResourceRepositoryFactory(ioManager, ncApplicationContext.getNodeId(),
+ metadataProperties);
+
+ localResourceRepository = (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory
+ .createRepository();
+
+ IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider =
+ new AsterixAppRuntimeContextProviderForRecovery(this);
+ txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider,
+ txnProperties);
+
+ IRecoveryManager recoveryMgr = txnSubsystem.getRecoveryManager();
+ SystemState systemState = recoveryMgr.getSystemState();
+ if (initialRun || systemState == SystemState.NEW_UNIVERSE) {
+ //delete any storage data before the resource factory is initialized
+ localResourceRepository.deleteStorageData(true);
+ }
+ initializeResourceIdFactory();
+
+ datasetLifecycleManager = new DatasetLifecycleManager(storageProperties, localResourceRepository,
+ MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager(),
+ ioManager.getIODevices().size());
+
+ isShuttingdown = false;
+
+ activeManager = new ActiveManager(ncApplicationContext.getNodeId(),
+ feedProperties.getMemoryComponentGlobalBudget(), compilerProperties.getFrameSize());
+
+ if (replicationProperties.isReplicationEnabled()) {
+ String nodeId = ncApplicationContext.getNodeId();
+
+ replicaResourcesManager = new ReplicaResourcesManager(localResourceRepository, metadataProperties);
+
+ replicationManager = new ReplicationManager(nodeId, replicationProperties, replicaResourcesManager,
+ txnSubsystem.getLogManager(), asterixAppRuntimeContextProvider);
+
+ //pass replication manager to replication required object
+ //LogManager to replicate logs
+ txnSubsystem.getLogManager().setReplicationManager(replicationManager);
+
+ //PersistentLocalResourceRepository to replicate metadata files and delete backups on drop index
+ localResourceRepository.setReplicationManager(replicationManager);
+
+ /**
+ * add the partitions that will be replicated in this node as inactive partitions
+ */
+ //get nodes which replicate to this node
+ Set<String> replicationClients = replicationProperties.getNodeReplicationClients(nodeId);
+ //remove the node itself
+ replicationClients.remove(nodeId);
+ for (String clientId : replicationClients) {
+ //get the partitions of each client
+ ClusterPartition[] clientPartitions = metadataProperties.getNodePartitions().get(clientId);
+ for (ClusterPartition partition : clientPartitions) {
+ localResourceRepository.addInactivePartition(partition.getPartitionId());
+ }
+ }
+
+ //initialize replication channel
+ replicationChannel = new ReplicationChannel(nodeId, replicationProperties, txnSubsystem.getLogManager(),
+ replicaResourcesManager, replicationManager, ncApplicationContext,
+ asterixAppRuntimeContextProvider);
+
+ remoteRecoveryManager = new RemoteRecoveryManager(replicationManager, this, replicationProperties);
+
+ bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager,
+ storageProperties.getBufferCacheMaxOpenFiles(), ncApplicationContext.getThreadFactory(),
+ replicationManager);
+ } else {
+ bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager,
+ storageProperties.getBufferCacheMaxOpenFiles(), ncApplicationContext.getThreadFactory());
+ }
+
+ /*
+ * The order of registration is important. The buffer cache must registered before recovery and transaction
+ * managers. Notes: registered components are stopped in reversed order
+ */
+ ILifeCycleComponentManager lccm = ncApplicationContext.getLifeCycleComponentManager();
+ lccm.register((ILifeCycleComponent) bufferCache);
+ /**
+ * LogManager must be stopped after RecoveryManager, DatasetLifeCycleManager, and ReplicationManager
+ * to process any logs that might be generated during stopping these components
+ */
+ lccm.register((ILifeCycleComponent) txnSubsystem.getLogManager());
+ /**
+ * ReplicationManager must be stopped after indexLifecycleManager and recovery manager
+ * so that any logs/files generated during closing datasets or checkpoints are sent to remote replicas
+ */
+ if (replicationManager != null) {
+ lccm.register(replicationManager);
+ }
+ lccm.register((ILifeCycleComponent) txnSubsystem.getRecoveryManager());
+ /**
+ * Stopping indexLifecycleManager will flush and close all datasets.
+ */
+ lccm.register((ILifeCycleComponent) datasetLifecycleManager);
+ lccm.register((ILifeCycleComponent) txnSubsystem.getTransactionManager());
+ lccm.register((ILifeCycleComponent) txnSubsystem.getLockManager());
+ }
+
+ @Override
+ public boolean isShuttingdown() {
+ return isShuttingdown;
+ }
+
+ @Override
+ public void setShuttingdown(boolean isShuttingdown) {
+ this.isShuttingdown = isShuttingdown;
+ }
+
+ @Override
+ public void deinitialize() throws HyracksDataException {
+ }
+
+ @Override
+ public IBufferCache getBufferCache() {
+ return bufferCache;
+ }
+
+ @Override
+ public IFileMapProvider getFileMapManager() {
+ return fileMapManager;
+ }
+
+ @Override
+ public ITransactionSubsystem getTransactionSubsystem() {
+ return txnSubsystem;
+ }
+
+ @Override
+ public IDatasetLifecycleManager getDatasetLifecycleManager() {
+ return datasetLifecycleManager;
+ }
+
+ @Override
+ public double getBloomFilterFalsePositiveRate() {
+ return storageProperties.getBloomFilterFalsePositiveRate();
+ }
+
+ @Override
+ public ILSMIOOperationScheduler getLSMIOScheduler() {
+ return lsmIOScheduler;
+ }
+
+ @Override
+ public ILocalResourceRepository getLocalResourceRepository() {
+ return localResourceRepository;
+ }
+
+ @Override
+ public IResourceIdFactory getResourceIdFactory() {
+ return resourceIdFactory;
+ }
+
+ @Override
+ public IIOManager getIOManager() {
+ return ioManager;
+ }
+
+ @Override
+ public AsterixStorageProperties getStorageProperties() {
+ return storageProperties;
+ }
+
+ @Override
+ public AsterixTransactionProperties getTransactionProperties() {
+ return txnProperties;
+ }
+
+ @Override
+ public AsterixCompilerProperties getCompilerProperties() {
+ return compilerProperties;
+ }
+
+ @Override
+ public AsterixMetadataProperties getMetadataProperties() {
+ return metadataProperties;
+ }
+
+ @Override
+ public AsterixExternalProperties getExternalProperties() {
+ return externalProperties;
+ }
+
+ @Override
+ public AsterixFeedProperties getFeedProperties() {
+ return feedProperties;
+ }
+
+ @Override
+ public AsterixBuildProperties getBuildProperties() {
+ return buildProperties;
+ }
+
+ @Override
+ public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID) {
+ return datasetLifecycleManager.getOperationTracker(datasetID);
+ }
+
+ @Override
+ public AsterixThreadExecutor getThreadExecutor() {
+ return threadExecutor;
+ }
+
+ @Override
+ public ILSMMergePolicyFactory getMetadataMergePolicyFactory() {
+ return metadataMergePolicyFactory;
+ }
+
+ @Override
+ public ActiveManager getActiveManager() {
+ return activeManager;
+ }
+
+ @Override
+ public AsterixReplicationProperties getReplicationProperties() {
+ return replicationProperties;
+ }
+
+ @Override
+ public IReplicationChannel getReplicationChannel() {
+ return replicationChannel;
+ }
+
+ @Override
+ public IReplicaResourcesManager getReplicaResourcesManager() {
+ return replicaResourcesManager;
+ }
+
+ @Override
+ public IRemoteRecoveryManager getRemoteRecoveryManager() {
+ return remoteRecoveryManager;
+ }
+
+ @Override
+ public IReplicationManager getReplicationManager() {
+ return replicationManager;
+ }
+
+ @Override
+ public ILibraryManager getLibraryManager() {
+ return libraryManager;
+ }
+
+ @Override
+ public void initializeResourceIdFactory() throws HyracksDataException {
+ resourceIdFactory = new GlobalResourceIdFactoryProvider(ncApplicationContext).createResourceIdFactory();
+ }
+
+ @Override
+ public void initializeMetadata(boolean newUniverse) throws Exception {
+ IAsterixStateProxy proxy;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bootstrapping metadata");
+ }
+ MetadataNode.INSTANCE.initialize(this, ncExtensionManager.getMetadataTupleTranslatorProvider(),
+ ncExtensionManager.getMetadataExtensions());
+
+ proxy = (IAsterixStateProxy) ncApplicationContext.getDistributedState();
+ if (proxy == null) {
+ throw new IllegalStateException("Metadata node cannot access distributed state");
+ }
+
+ // This is a special case, we just give the metadataNode directly.
+ // This way we can delay the registration of the metadataNode until
+ // it is completely initialized.
+ MetadataManager.instantiate(new MetadataManager(proxy, MetadataNode.INSTANCE));
+ MetadataBootstrap.startUniverse(this, ncApplicationContext, newUniverse);
+ MetadataBootstrap.startDDLRecovery();
+ ncExtensionManager.initializeMetadata();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Metadata node bound");
+ }
+ }
+
+ @Override
+ public void exportMetadataNodeStub() throws RemoteException {
+ IMetadataNode stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, metadataRmiPort);
+ ((IAsterixStateProxy) ncApplicationContext.getDistributedState()).setMetadataNode(stub);
+ }
+
+ @Override
+ public void unexportMetadataNodeStub() throws RemoteException {
+ UnicastRemoteObject.unexportObject(MetadataNode.INSTANCE, false);
+ }
+
+ public NCExtensionManager getNcExtensionManager() {
+ return ncExtensionManager;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
new file mode 100644
index 0000000..759bf09
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCExtensionManager.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.nc;
+
+import java.rmi.RemoteException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.app.cc.CompilerExtensionManager;
+import org.apache.asterix.common.api.IExtension;
+import org.apache.asterix.common.config.AsterixExtension;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.metadata.api.IMetadataExtension;
+import org.apache.asterix.metadata.entitytupletranslators.MetadataTupleTranslatorProvider;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * AsterixDB's implementation of {@code INCExtensionManager} which takes care of
+ * initializing extensions on Node Controllers
+ */
+public class NCExtensionManager {
+
+ private final MetadataTupleTranslatorProvider tupleTranslatorProvider;
+ private final List<IMetadataExtension> mdExtensions;
+
+ /**
+ * Initialize {@code CCExtensionManager} from configuration
+ *
+ * @param list
+ * @throws InstantiationException
+ * @throws IllegalAccessException
+ * @throws ClassNotFoundException
+ * @throws HyracksDataException
+ */
+ public NCExtensionManager(List<AsterixExtension> list)
+ throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
+ MetadataTupleTranslatorProvider ttp = null;
+ IMetadataExtension tupleTranslatorExtension = null;
+ mdExtensions = new ArrayList<>();
+ if (list != null) {
+ for (AsterixExtension extensionConf : list) {
+ IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
+ extension.configure(extensionConf.getArgs());
+ switch (extension.getExtensionKind()) {
+ case METADATA:
+ IMetadataExtension mde = (IMetadataExtension) extension;
+ mdExtensions.add(mde);
+ ttp = extendTupleTranslator(ttp, tupleTranslatorExtension, mde);
+ tupleTranslatorExtension = ttp == null ? null : mde;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ this.tupleTranslatorProvider = ttp == null ? new MetadataTupleTranslatorProvider() : ttp;
+ }
+
+ private MetadataTupleTranslatorProvider extendTupleTranslator(MetadataTupleTranslatorProvider ttp,
+ IMetadataExtension tupleTranslatorExtension, IMetadataExtension mde) throws HyracksDataException {
+ if (ttp != null) {
+ throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_EXTENSION_CONFLICT,
+ CompilerExtensionManager.ERROR_MESSAGE_COMPONENT_CONFLICT, tupleTranslatorExtension.getId(),
+ mde.getId(), IMetadataExtension.class.getSimpleName());
+ }
+ return mde.getMetadataTupleTranslatorProvider();
+ }
+
+ public List<IMetadataExtension> getMetadataExtensions() {
+ return mdExtensions;
+ }
+
+ public MetadataTupleTranslatorProvider getMetadataTupleTranslatorProvider() {
+ return tupleTranslatorProvider;
+ }
+
+ /**
+ * Called on bootstrap of metadata node allowing extensions to instantiate their Metadata artifacts
+ *
+ * @throws HyracksDataException
+ */
+ public void initializeMetadata() throws HyracksDataException {
+ if (mdExtensions != null) {
+ for (IMetadataExtension mdExtension : mdExtensions) {
+ try {
+ mdExtension.initializeMetadata();
+ } catch (RemoteException | ACIDException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultPrinter.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultPrinter.java
new file mode 100644
index 0000000..9b1b32b
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultPrinter.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.result;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.ByteBuffer;
+
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.utils.JSONUtil;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameTupleAccessor;
+import org.apache.hyracks.api.comm.VSizeFrame;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.nc.resources.memory.FrameManager;
+
+public class ResultPrinter {
+
+ // TODO(tillw): Should this be static?
+ private static FrameManager resultDisplayFrameMgr = new FrameManager(ResultReader.FRAME_SIZE);
+
+ private final SessionConfig conf;
+ private final Stats stats;
+ private final ARecordType recordType;
+
+ private boolean indentJSON;
+ private boolean quoteRecord;
+
+ // Whether we are wrapping the output sequence in an array
+ private boolean wrapArray = false;
+ // Whether this is the first instance being output
+ private boolean notFirst = false;
+
+ public ResultPrinter(SessionConfig conf, Stats stats, ARecordType recordType) {
+ this.conf = conf;
+ this.stats = stats;
+ this.recordType = recordType;
+ this.indentJSON = conf.is(SessionConfig.FORMAT_INDENT_JSON);
+ this.quoteRecord = conf.is(SessionConfig.FORMAT_QUOTE_RECORD);
+ }
+
+ private static void appendCSVHeader(Appendable app, ARecordType recordType) throws HyracksDataException {
+ try {
+ String[] fieldNames = recordType.getFieldNames();
+ boolean notfirst = false;
+ for (String name : fieldNames) {
+ if (notfirst) {
+ app.append(',');
+ }
+ notfirst = true;
+ app.append('"').append(name.replace("\"", "\"\"")).append('"');
+ }
+ app.append("\r\n");
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ private void printPrefix() throws HyracksDataException {
+ // If we're outputting CSV with a header, the HTML header was already
+ // output by displayCSVHeader(), so skip it here
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Results:</h4>");
+ conf.out().println("<pre>");
+ }
+
+ try {
+ conf.resultPrefix(new AlgebricksAppendable(conf.out()));
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+
+ if (conf.is(SessionConfig.FORMAT_WRAPPER_ARRAY)) {
+ conf.out().print("[ ");
+ wrapArray = true;
+ }
+
+ if (conf.fmt() == SessionConfig.OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER)) {
+ if (recordType == null) {
+ throw new HyracksDataException("Cannot print CSV with header without specifying output-record-type");
+ }
+ if (quoteRecord) {
+ StringWriter sw = new StringWriter();
+ appendCSVHeader(sw, recordType);
+ conf.out().print(JSONUtil.quoteAndEscape(sw.toString()));
+ conf.out().print("\n");
+ notFirst = true;
+ } else {
+ appendCSVHeader(conf.out(), recordType);
+ }
+ }
+ }
+
+ private void printPostfix() throws HyracksDataException {
+ conf.out().flush();
+ if (wrapArray) {
+ conf.out().println(" ]");
+ }
+ try {
+ conf.resultPostfix(new AlgebricksAppendable(conf.out()));
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
+ }
+ }
+
+ private void displayRecord(String result) {
+ String record = result;
+ if (indentJSON) {
+ // TODO(tillw): this is inefficient - do this during record generation
+ record = JSONUtil.indent(record, 2);
+ }
+ if (conf.fmt() == SessionConfig.OutputFormat.CSV) {
+ // TODO(tillw): this is inefficient as well
+ record = record + "\r\n";
+ }
+ if (quoteRecord) {
+ // TODO(tillw): this is inefficient as well
+ record = JSONUtil.quoteAndEscape(record);
+ }
+ conf.out().print(record);
+ stats.setCount(stats.getCount() + 1);
+ // TODO(tillw) fix this approximation
+ stats.setSize(stats.getSize() + record.length());
+ }
+
+ public void print(String record) throws HyracksDataException {
+ printPrefix();
+ // TODO(tillw) evil hack
+ quoteRecord = true;
+ displayRecord(record);
+ printPostfix();
+ }
+
+ public void print(ResultReader resultReader) throws HyracksDataException {
+ printPrefix();
+
+ final IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor();
+ final IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
+
+ while (resultReader.read(frame) > 0) {
+ final ByteBuffer frameBuffer = frame.getBuffer();
+ final byte[] frameBytes = frameBuffer.array();
+ fta.reset(frameBuffer);
+ final int last = fta.getTupleCount();
+ for (int tIndex = 0; tIndex < last; tIndex++) {
+ final int start = fta.getTupleStartOffset(tIndex);
+ int length = fta.getTupleEndOffset(tIndex) - start;
+ if (conf.fmt() == SessionConfig.OutputFormat.CSV
+ && ((length > 0) && (frameBytes[start + length - 1] == '\n'))) {
+ length--;
+ }
+ String result = new String(frameBytes, start, length, UTF_8);
+ if (wrapArray && notFirst) {
+ conf.out().print(", ");
+ }
+ notFirst = true;
+ displayRecord(result);
+ }
+ frameBuffer.clear();
+ }
+
+ printPostfix();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
new file mode 100644
index 0000000..99235be
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.result;
+
+import org.apache.asterix.om.util.AsterixAppContextInfo;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameTupleAccessor;
+import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
+import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
+
+public class ResultReader {
+ private final IHyracksDataset hyracksDataset;
+
+ private IHyracksDatasetReader reader;
+
+ private IFrameTupleAccessor frameTupleAccessor;
+
+ // Number of parallel result reader buffers
+ public static final int NUM_READERS = 1;
+
+ public static final int FRAME_SIZE = AsterixAppContextInfo.getInstance().getCompilerProperties().getFrameSize();
+
+ public ResultReader(IHyracksDataset hdc) {
+ hyracksDataset = hdc;
+ }
+
+ public void open(JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
+ reader = hyracksDataset.createReader(jobId, resultSetId);
+ frameTupleAccessor = new ResultFrameTupleAccessor();
+ }
+
+ public Status getStatus() {
+ return reader.getResultStatus();
+ }
+
+ public int read(IFrame frame) throws HyracksDataException {
+ return reader.read(frame);
+ }
+
+ public IFrameTupleAccessor getFrameTupleAccessor() {
+ return frameTupleAccessor;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultUtil.java
new file mode 100644
index 0000000..b5fd96e
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultUtil.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.result;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.AbstractMap;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.asterix.api.http.servlet.APIServlet;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
+import org.apache.http.ParseException;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class ResultUtil {
+ private static final Logger LOGGER = Logger.getLogger(ResultUtil.class.getName());
+ public static final Map<Character, String> HTML_ENTITIES = Collections.unmodifiableMap(Stream.of(
+ new AbstractMap.SimpleImmutableEntry<>('"', """), new AbstractMap.SimpleImmutableEntry<>('&', "&"),
+ new AbstractMap.SimpleImmutableEntry<>('<', "<"), new AbstractMap.SimpleImmutableEntry<>('>', ">"))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
+
+ private ResultUtil() {
+
+ }
+
+ /**
+ * escapes html entities in aString
+ *
+ * @param aString
+ * @return escaped String
+ */
+ public static String escapeHTML(String aString) {
+ String escaped = aString;
+ for (Entry<Character, String> entry : HTML_ENTITIES.entrySet()) {
+ if (escaped.indexOf(entry.getKey()) >= 0) {
+ escaped = escaped.replace(entry.getKey().toString(), entry.getValue());
+ }
+ }
+ return escaped;
+ }
+
+ public static void displayResults(ResultReader resultReader, SessionConfig conf, Stats stats,
+ ARecordType recordType) throws HyracksDataException {
+ new ResultPrinter(conf, stats, recordType).print(resultReader);
+ }
+
+ public static void displayResults(String record, SessionConfig conf, Stats stats, ARecordType recordType)
+ throws HyracksDataException {
+ new ResultPrinter(conf, stats, recordType).print(record);
+ }
+
+ public static JSONObject getErrorResponse(int errorCode, String errorMessage, String errorSummary,
+ String errorStackTrace) {
+ JSONObject errorResp = new JSONObject();
+ JSONArray errorArray = new JSONArray();
+ errorArray.put(errorCode);
+ errorArray.put(errorMessage);
+ try {
+ errorResp.put("error-code", errorArray);
+ if (!"".equals(errorSummary)) {
+ errorResp.put("summary", errorSummary);
+ } else {
+ //parse exception
+ errorResp.put("summary", errorMessage);
+ }
+ errorResp.put("stacktrace", errorStackTrace);
+ } catch (JSONException e) {
+ LOGGER.warn("Failed to build the result's JSON object", e);
+ // TODO(madhusudancs): Figure out what to do when JSONException occurs while building the results.
+ }
+ return errorResp;
+ }
+
+ public static void webUIErrorHandler(PrintWriter out, Exception e) {
+ String errorTemplate = readTemplateFile("/webui/errortemplate.html", "%s\n%s\n%s");
+
+ String errorOutput = String.format(errorTemplate, escapeHTML(extractErrorMessage(e)),
+ escapeHTML(extractErrorSummary(e)), escapeHTML(extractFullStackTrace(e)));
+ out.println(errorOutput);
+ }
+
+ public static void webUIParseExceptionHandler(PrintWriter out, Throwable e, String query) {
+ String errorTemplate = readTemplateFile("/webui/errortemplate_message.html", "<pre class=\"error\">%s\n</pre>");
+
+ String errorOutput = String.format(errorTemplate, buildParseExceptionMessage(e, query));
+ out.println(errorOutput);
+ }
+
+ public static void apiErrorHandler(PrintWriter out, Exception e) {
+ int errorCode = 99;
+ if (e instanceof ParseException) {
+ errorCode = 2;
+ } else if (e instanceof AlgebricksException) {
+ errorCode = 3;
+ } else if (e instanceof HyracksDataException) {
+ errorCode = 4;
+ }
+
+ JSONObject errorResp = ResultUtil.getErrorResponse(errorCode, extractErrorMessage(e), extractErrorSummary(e),
+ extractFullStackTrace(e));
+ out.write(errorResp.toString());
+ }
+
+ public static String buildParseExceptionMessage(Throwable e, String query) {
+ StringBuilder errorMessage = new StringBuilder();
+ String message = e.getMessage();
+ message = message.replace("<", "<");
+ message = message.replace(">", ">");
+ errorMessage.append("Error: " + message + "\n");
+ int pos = message.indexOf("line");
+ if (pos > 0) {
+ Pattern p = Pattern.compile("\\d+");
+ Matcher m = p.matcher(message);
+ if (m.find(pos)) {
+ int lineNo = Integer.parseInt(message.substring(m.start(), m.end()));
+ String[] lines = query.split("\n");
+ if (lineNo > lines.length) {
+ errorMessage.append("===> <BLANK LINE> \n");
+ } else {
+ String line = lines[lineNo - 1];
+ errorMessage.append("==> " + line);
+ }
+ }
+ }
+ return errorMessage.toString();
+ }
+
+ private static Throwable getRootCause(Throwable cause) {
+ Throwable currentCause = cause;
+ Throwable nextCause = cause.getCause();
+ while (nextCause != null && nextCause != currentCause) {
+ currentCause = nextCause;
+ nextCause = cause.getCause();
+ }
+ return currentCause;
+ }
+
+ /**
+ * Extract the message in the root cause of the stack trace:
+ *
+ * @param e
+ * @return error message string.
+ */
+ private static String extractErrorMessage(Throwable e) {
+ Throwable cause = getRootCause(e);
+ String fullyQualifiedExceptionClassName = cause.getClass().getName();
+ String[] hierarchySplits = fullyQualifiedExceptionClassName.split("\\.");
+ //try returning the class without package qualification
+ String exceptionClassName = hierarchySplits[hierarchySplits.length - 1];
+ String localizedMessage = cause.getLocalizedMessage();
+ if (localizedMessage == null) {
+ localizedMessage = "Internal error. Please check instance logs for further details.";
+ }
+ return localizedMessage + " [" + exceptionClassName + "]";
+ }
+
+ /**
+ * Extract the meaningful part of a stack trace:
+ * a. the causes in the stack trace hierarchy
+ * b. the top exception for each cause
+ *
+ * @param e
+ * @return the contacted message containing a and b.
+ */
+ private static String extractErrorSummary(Throwable e) {
+ StringBuilder errorMessageBuilder = new StringBuilder();
+ Throwable cause = e;
+ errorMessageBuilder.append(cause.getLocalizedMessage());
+ while (cause != null) {
+ StackTraceElement[] stackTraceElements = cause.getStackTrace();
+ errorMessageBuilder.append(stackTraceElements.length > 0 ? "\n caused by: " + stackTraceElements[0] : "");
+ cause = cause.getCause();
+ }
+ return errorMessageBuilder.toString();
+ }
+
+ /**
+ * Extract the full stack trace:
+ *
+ * @param e
+ * @return the string containing the full stack trace of the error.
+ */
+ public static String extractFullStackTrace(Throwable e) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(stringWriter);
+ e.printStackTrace(printWriter);
+ return stringWriter.toString();
+ }
+
+ /**
+ * Read the template file which is stored as a resource and return its content. If the file does not exist or is
+ * not readable return the default template string.
+ *
+ * @param path
+ * The path to the resource template file
+ * @param defaultTemplate
+ * The default template string if the template file does not exist or is not readable
+ * @return The template string to be used to render the output.
+ */
+ //TODO(till|amoudi|mblow|yingyi|ceej|imaxon): path is ignored completely!!
+ private static String readTemplateFile(String path, String defaultTemplate) {
+ String errorTemplate = defaultTemplate;
+ try {
+ String resourcePath = "/webui/errortemplate_message.html";
+ InputStream is = APIServlet.class.getResourceAsStream(resourcePath);
+ InputStreamReader isr = new InputStreamReader(is);
+ StringBuilder sb = new StringBuilder();
+ BufferedReader br = new BufferedReader(isr);
+ String line = br.readLine();
+
+ while (line != null) {
+ sb.append(line);
+ line = br.readLine();
+ }
+ errorTemplate = sb.toString();
+ } catch (IOException ioe) {
+ LOGGER.warn("Unable to read template error message file", ioe);
+ }
+ return errorTemplate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java
new file mode 100644
index 0000000..c802d08
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.translator;
+
+import java.util.List;
+
+import org.apache.asterix.app.cc.CompilerExtensionManager;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
+
+public class DefaultStatementExecutorFactory implements IStatementExecutorFactory {
+
+ protected final CompilerExtensionManager cExtensionManager;
+
+ public DefaultStatementExecutorFactory(CompilerExtensionManager cExtensionManager) {
+ this.cExtensionManager = cExtensionManager;
+ }
+
+ @Override
+ public QueryTranslator create(List<Statement> aqlStatements, SessionConfig conf,
+ ILangCompilationProvider compilationProvider) {
+ return new QueryTranslator(aqlStatements, conf, compilationProvider, cExtensionManager);
+ }
+
+}
[02/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
index db4300a..2ffaded 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/LoadableDataSource.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.utils.DatasetUtils;
@@ -31,8 +32,19 @@ import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
import org.apache.commons.lang.StringUtils;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobSpecification;
public class LoadableDataSource extends AqlDataSource {
@@ -112,4 +124,24 @@ public class LoadableDataSource extends AqlDataSource {
public Dataset getTargetDataset() {
return targetDataset;
}
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime(
+ AqlMetadataProvider aqlMetadataProvider, IDataSource<AqlSourceId> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+ throws AlgebricksException {
+ // This is a load into dataset operation
+ LoadableDataSource alds = (LoadableDataSource) dataSource;
+ List<List<String>> aldsPartitioningKeys = alds.getPartitioningKeys();
+ boolean isAldsPKAutoGenerated = ((InternalDatasetDetails) alds.getTargetDataset().getDatasetDetails())
+ .isAutogenerated();
+ ARecordType itemType = (ARecordType) alds.getLoadedType();
+ IAdapterFactory adapterFactory = aqlMetadataProvider.getConfiguredAdapterFactory(alds.getTargetDataset(),
+ alds.getAdapter(), alds.getAdapterProperties(), itemType, isAldsPKAutoGenerated, aldsPartitioningKeys,
+ null);
+ RecordDescriptor rDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
+ return aqlMetadataProvider.buildLoadableDatasetScan(jobSpec, adapterFactory, rDesc);
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index aba8f8d..baf141f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -25,14 +25,22 @@ import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
/**
* Metadata describing a dataset.
*/
public class Dataset implements IMetadataEntity<Dataset> {
- private static final long serialVersionUID = 1L;
+ /**
+ * Dataset related operations
+ */
+ public static final byte OP_READ = 0x00;
+ public static final byte OP_INSERT = 0x01;
+ public static final byte OP_DELETE = 0x02;
+ public static final byte OP_UPSERT = 0x03;
+ private static final long serialVersionUID = 1L;
private final String dataverseName;
// Enforced to be unique within a dataverse.
private final String datasetName;
@@ -84,6 +92,13 @@ public class Dataset implements IMetadataEntity<Dataset> {
this.hints = hints;
}
+ public Dataset(Dataset dataset) {
+ this(dataset.dataverseName, dataset.datasetName, dataset.itemTypeDataverseName, dataset.itemTypeName,
+ dataset.metaItemTypeDataverseName, dataset.metaItemTypeName, dataset.nodeGroupName,
+ dataset.compactionPolicy, dataset.compactionPolicyProperties, dataset.datasetDetails, dataset.hints,
+ dataset.datasetType, dataset.datasetId, dataset.pendingOp);
+ }
+
public String getDataverseName() {
return dataverseName;
}
@@ -176,6 +191,10 @@ public class Dataset implements IMetadataEntity<Dataset> {
return true;
}
+ public boolean allow(ProjectOperator project, byte operation) {
+ return !hasMetaPart();
+ }
+
@Override
public String toString() {
return dataverseName + "." + datasetName;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java
index 467e6f5..1b40484 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataverse.java
@@ -60,4 +60,9 @@ public class Dataverse implements IMetadataEntity<Dataverse> {
public Dataverse dropFromCache(MetadataCache cache) {
return cache.dropDataverse(this);
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ":" + dataverseName;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
index 3508554..bbb62d3 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
@@ -154,14 +154,14 @@ public class InternalDatasetDetails implements IDatasetDetails {
AMutableString aString = new AMutableString("");
AMutableInt8 aInt8 = new AMutableInt8((byte) 0);
@SuppressWarnings("unchecked")
- ISerializerDeserializer<ABoolean> booleanSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ABOOLEAN);
+ ISerializerDeserializer<ABoolean> booleanSerde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABOOLEAN);
@SuppressWarnings("unchecked")
- ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ ISerializerDeserializer<AString> stringSerde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
@SuppressWarnings("unchecked")
- ISerializerDeserializer<AInt8> int8Serde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.AINT8);
+ ISerializerDeserializer<AInt8> int8Serde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT8);
// write field 0
fieldValue.reset();
@@ -280,8 +280,8 @@ public class InternalDatasetDetails implements IDatasetDetails {
propertyRecordBuilder.reset(recordType);
AMutableString aString = new AMutableString("");
@SuppressWarnings("unchecked")
- ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ ISerializerDeserializer<AString> stringSerde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
// write field 0
fieldValue.reset();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
index 62a28e9..5ec25e6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/AbstractTupleTranslator.java
@@ -38,6 +38,7 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
* IMetadataEntityTupleTranslator.
*/
public abstract class AbstractTupleTranslator<T> implements IMetadataEntityTupleTranslator<T> {
+ private static final long serialVersionUID = 1L;
protected AMutableString aString = new AMutableString("");
@SuppressWarnings("unchecked")
protected ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
@@ -49,10 +50,10 @@ public abstract class AbstractTupleTranslator<T> implements IMetadataEntityTuple
protected ISerializerDeserializer<AInt32> int32Serde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.AINT32);
- protected final IARecordBuilder recordBuilder;
- protected final ArrayBackedValueStorage fieldValue;
- protected final ArrayTupleBuilder tupleBuilder;
- protected final ArrayTupleReference tuple;
+ protected final transient IARecordBuilder recordBuilder;
+ protected final transient ArrayBackedValueStorage fieldValue;
+ protected final transient ArrayTupleBuilder tupleBuilder;
+ protected final transient ArrayTupleReference tuple;
public AbstractTupleTranslator(boolean getTuple, int fieldCount) {
if (getTuple) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java
index e29b9dd..9d37ddb 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/CompactionPolicyTupleTranslator.java
@@ -51,7 +51,7 @@ public class CompactionPolicyTupleTranslator extends AbstractTupleTranslator<Com
private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.COMPACTION_POLICY_RECORDTYPE);
- public CompactionPolicyTupleTranslator(boolean getTuple) {
+ protected CompactionPolicyTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET.getFieldCount());
}
@@ -62,7 +62,7 @@ public class CompactionPolicyTupleTranslator extends AbstractTupleTranslator<Com
int recordLength = tuple.getFieldLength(COMPACTION_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord compactionPolicyRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord compactionPolicyRecord = recordSerDes.deserialize(in);
return createCompactionPolicyFromARecord(compactionPolicyRecord);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 01380c0..e97f449 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -73,6 +73,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
* Translates a Dataset metadata entity to an ITupleReference and vice versa.
*/
public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
+ private static final long serialVersionUID = 1L;
// Field indexes of serialized Dataset in a tuple.
// First key field.
public static final int DATASET_DATAVERSENAME_TUPLE_FIELD_INDEX = 0;
@@ -82,14 +83,14 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
public static final int DATASET_PAYLOAD_TUPLE_FIELD_INDEX = 2;
@SuppressWarnings("unchecked")
- private final ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
+ protected final ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.DATASET_RECORDTYPE);
- private final AMutableInt32 aInt32;
- protected final ISerializerDeserializer<AInt32> aInt32Serde;
- private final ArrayBackedValueStorage fieldName = new ArrayBackedValueStorage();
+ protected final transient AMutableInt32 aInt32;
+ protected final transient ISerializerDeserializer<AInt32> aInt32Serde;
+ protected final transient ArrayBackedValueStorage fieldName = new ArrayBackedValueStorage();
@SuppressWarnings("unchecked")
- public DatasetTupleTranslator(boolean getTuple) {
+ protected DatasetTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.DATASET_DATASET.getFieldCount());
aInt32 = new AMutableInt32(-1);
aInt32Serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
@@ -106,7 +107,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
return createDatasetFromARecord(datasetRecord);
}
- private Dataset createDatasetFromARecord(ARecord datasetRecord) throws IOException {
+ protected Dataset createDatasetFromARecord(ARecord datasetRecord) throws IOException {
String dataverseName = ((AString) datasetRecord
.getValueByPos(MetadataRecordTypes.DATASET_ARECORD_DATAVERSENAME_FIELD_INDEX)).getStringValue();
@@ -249,8 +250,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
if (metaTypeDataverseNameIndex >= 0) {
metaTypeDataverseName = ((AString) datasetRecord.getValueByPos(metaTypeDataverseNameIndex))
.getStringValue();
- int metaTypeNameIndex = datasetRecord.getType()
- .getFieldIndex(MetadataRecordTypes.FIELD_NAME_METATYPE_NAME);
+ int metaTypeNameIndex = datasetRecord.getType().getFieldIndex(MetadataRecordTypes.FIELD_NAME_METATYPE_NAME);
metaTypeName = ((AString) datasetRecord.getValueByPos(metaTypeNameIndex)).getStringValue();
}
@@ -374,6 +374,18 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
recordBuilder.addField(MetadataRecordTypes.DATASET_ARECORD_PENDINGOP_FIELD_INDEX, fieldValue);
// write open fields
+ writeOpenFields(dataset);
+
+ // write record
+ recordBuilder.write(tupleBuilder.getDataOutput(), true);
+ tupleBuilder.addFieldEndOffset();
+
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ return tuple;
+ }
+
+ protected void writeOpenFields(Dataset dataset) throws HyracksDataException {
+ // write open fields
if (dataset.hasMetaPart()) {
// write open field 1, the meta item type Dataverse name.
fieldName.reset();
@@ -393,16 +405,9 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
stringSerde.serialize(aString, fieldValue.getDataOutput());
recordBuilder.addField(fieldName, fieldValue);
}
-
- // write record
- recordBuilder.write(tupleBuilder.getDataOutput(), true);
- tupleBuilder.addFieldEndOffset();
-
- tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
- return tuple;
}
- private void writeDatasetDetailsRecordType(IARecordBuilder recordBuilder, Dataset dataset, DataOutput dataOutput)
+ protected void writeDatasetDetailsRecordType(IARecordBuilder recordBuilder, Dataset dataset, DataOutput dataOutput)
throws HyracksDataException {
dataset.getDatasetDetails().writeDatasetDetailsRecordType(fieldValue.getDataOutput());
@@ -417,7 +422,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
}
- private Map<String, String> getDatasetHints(ARecord datasetRecord) {
+ protected Map<String, String> getDatasetHints(ARecord datasetRecord) {
Map<String, String> hints = new HashMap<String, String>();
String key;
String value;
@@ -434,7 +439,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
}
@SuppressWarnings("unchecked")
- private void writeDatasetHintRecord(String name, String value, DataOutput out) throws HyracksDataException {
+ protected void writeDatasetHintRecord(String name, String value, DataOutput out) throws HyracksDataException {
IARecordBuilder propertyRecordBuilder = new RecordBuilder();
ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
propertyRecordBuilder.reset(MetadataRecordTypes.DATASET_HINTS_RECORDTYPE);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
index a824551..70c5813 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasourceAdapterTupleTranslator.java
@@ -52,7 +52,7 @@ public class DatasourceAdapterTupleTranslator extends AbstractTupleTranslator<Da
private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.DATASOURCE_ADAPTER_RECORDTYPE);
- public DatasourceAdapterTupleTranslator(boolean getTuple) {
+ protected DatasourceAdapterTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET.getFieldCount());
}
@@ -63,7 +63,7 @@ public class DatasourceAdapterTupleTranslator extends AbstractTupleTranslator<Da
int recordLength = tuple.getFieldLength(ADAPTER_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord adapterRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord adapterRecord = recordSerDes.deserialize(in);
return createAdapterFromARecord(adapterRecord);
}
@@ -71,8 +71,9 @@ public class DatasourceAdapterTupleTranslator extends AbstractTupleTranslator<Da
String dataverseName = ((AString) adapterRecord
.getValueByPos(MetadataRecordTypes.DATASOURCE_ADAPTER_ARECORD_DATAVERSENAME_FIELD_INDEX))
.getStringValue();
- String adapterName = ((AString) adapterRecord
- .getValueByPos(MetadataRecordTypes.DATASOURCE_ADAPTER_ARECORD_NAME_FIELD_INDEX)).getStringValue();
+ String adapterName =
+ ((AString) adapterRecord.getValueByPos(MetadataRecordTypes.DATASOURCE_ADAPTER_ARECORD_NAME_FIELD_INDEX))
+ .getStringValue();
String classname = ((AString) adapterRecord
.getValueByPos(MetadataRecordTypes.DATASOURCE_ADAPTER_ARECORD_CLASSNAME_FIELD_INDEX)).getStringValue();
IDataSourceAdapter.AdapterType adapterType = IDataSourceAdapter.AdapterType.valueOf(
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
index 0604fff..b294b77 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatatypeTupleTranslator.java
@@ -82,7 +82,7 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
private final MetadataNode metadataNode;
private final JobId jobId;
- public DatatypeTupleTranslator(JobId jobId, MetadataNode metadataNode, boolean getTuple) {
+ protected DatatypeTupleTranslator(JobId jobId, MetadataNode metadataNode, boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.DATATYPE_DATASET.getFieldCount());
this.jobId = jobId;
this.metadataNode = metadataNode;
@@ -100,15 +100,17 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
}
private Datatype createDataTypeFromARecord(ARecord datatypeRecord) throws MetadataException {
- String dataverseName = ((AString) datatypeRecord
- .getValueByPos(MetadataRecordTypes.DATATYPE_ARECORD_DATAVERSENAME_FIELD_INDEX)).getStringValue();
- String datatypeName = ((AString) datatypeRecord
- .getValueByPos(MetadataRecordTypes.DATATYPE_ARECORD_DATATYPENAME_FIELD_INDEX)).getStringValue();
+ String dataverseName =
+ ((AString) datatypeRecord.getValueByPos(MetadataRecordTypes.DATATYPE_ARECORD_DATAVERSENAME_FIELD_INDEX))
+ .getStringValue();
+ String datatypeName =
+ ((AString) datatypeRecord.getValueByPos(MetadataRecordTypes.DATATYPE_ARECORD_DATATYPENAME_FIELD_INDEX))
+ .getStringValue();
IAType type = AsterixBuiltinTypeMap.getBuiltinTypes().get(datatypeName);
if (type == null) {
// Derived Type
- ARecord derivedTypeRecord = (ARecord) datatypeRecord
- .getValueByPos(MetadataRecordTypes.DATATYPE_ARECORD_DERIVED_FIELD_INDEX);
+ ARecord derivedTypeRecord =
+ (ARecord) datatypeRecord.getValueByPos(MetadataRecordTypes.DATATYPE_ARECORD_DERIVED_FIELD_INDEX);
DerivedTypeTag tag = DerivedTypeTag.valueOf(
((AString) derivedTypeRecord.getValueByPos(MetadataRecordTypes.DERIVEDTYPE_ARECORD_TAG_FIELD_INDEX))
.getStringValue());
@@ -131,11 +133,11 @@ public class DatatypeTupleTranslator extends AbstractTupleTranslator<Datatype> {
String fieldTypeName;
while (cursor.next()) {
ARecord field = (ARecord) cursor.get();
- fieldNames[fieldId] = ((AString) field
- .getValueByPos(MetadataRecordTypes.FIELD_ARECORD_FIELDNAME_FIELD_INDEX))
+ fieldNames[fieldId] =
+ ((AString) field.getValueByPos(MetadataRecordTypes.FIELD_ARECORD_FIELDNAME_FIELD_INDEX))
.getStringValue();
- fieldTypeName = ((AString) field
- .getValueByPos(MetadataRecordTypes.FIELD_ARECORD_FIELDTYPE_FIELD_INDEX))
+ fieldTypeName =
+ ((AString) field.getValueByPos(MetadataRecordTypes.FIELD_ARECORD_FIELDTYPE_FIELD_INDEX))
.getStringValue();
boolean isNullable = ((ABoolean) field
.getValueByPos(MetadataRecordTypes.FIELD_ARECORD_ISNULLABLE_FIELD_INDEX)).getBoolean()
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java
index 74f85ab..086bb10 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java
@@ -48,7 +48,7 @@ public class DataverseTupleTranslator extends AbstractTupleTranslator<Dataverse>
// Payload field containing serialized Dataverse.
public static final int DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX = 1;
- private AMutableInt32 aInt32;
+ private transient AMutableInt32 aInt32;
protected ISerializerDeserializer<AInt32> aInt32Serde;
@SuppressWarnings("unchecked")
@@ -56,7 +56,7 @@ public class DataverseTupleTranslator extends AbstractTupleTranslator<Dataverse>
.getSerializerDeserializer(MetadataRecordTypes.DATAVERSE_RECORDTYPE);
@SuppressWarnings("unchecked")
- public DataverseTupleTranslator(boolean getTuple) {
+ protected DataverseTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.DATAVERSE_DATASET.getFieldCount());
aInt32 = new AMutableInt32(-1);
aInt32Serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java
index 717e61f..f4100d4 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/ExternalFileTupleTranslator.java
@@ -53,9 +53,9 @@ public class ExternalFileTupleTranslator extends AbstractTupleTranslator<Externa
// Payload field containing serialized ExternalFile.
public static final int EXTERNAL_FILE_PAYLOAD_TUPLE_FIELD_INDEX = 3;
- protected AMutableInt32 aInt32 = new AMutableInt32(0);
- protected AMutableDateTime aDateTime = new AMutableDateTime(0);
- protected AMutableInt64 aInt64 = new AMutableInt64(0);
+ protected transient AMutableInt32 aInt32 = new AMutableInt32(0);
+ protected transient AMutableDateTime aDateTime = new AMutableDateTime(0);
+ protected transient AMutableInt64 aInt64 = new AMutableInt64(0);
@SuppressWarnings("unchecked")
protected ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
@@ -70,7 +70,7 @@ public class ExternalFileTupleTranslator extends AbstractTupleTranslator<Externa
private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.EXTERNAL_FILE_RECORDTYPE);
- public ExternalFileTupleTranslator(boolean getTuple) {
+ protected ExternalFileTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET.getFieldCount());
}
@@ -81,7 +81,7 @@ public class ExternalFileTupleTranslator extends AbstractTupleTranslator<Externa
int recordLength = tuple.getFieldLength(EXTERNAL_FILE_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord externalFileRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord externalFileRecord = recordSerDes.deserialize(in);
return createExternalFileFromARecord(externalFileRecord);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
index 4839065..5b9c1c9 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedPolicyTupleTranslator.java
@@ -67,7 +67,7 @@ public class FeedPolicyTupleTranslator extends AbstractTupleTranslator<FeedPolic
protected ISerializerDeserializer<AInt32> aInt32Serde;
@SuppressWarnings("unchecked")
- public FeedPolicyTupleTranslator(boolean getTuple) {
+ protected FeedPolicyTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.FEED_POLICY_DATASET.getFieldCount());
aInt32Serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
}
@@ -173,8 +173,8 @@ public class FeedPolicyTupleTranslator extends AbstractTupleTranslator<FeedPolic
ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
propertyRecordBuilder.reset(MetadataRecordTypes.POLICY_PARAMS_RECORDTYPE);
AMutableString aString = new AMutableString("");
- ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ ISerializerDeserializer<AString> stringSerde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
// write field 0
fieldValue.reset();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java
index fc82255..09b44b4 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FeedTupleTranslator.java
@@ -67,10 +67,10 @@ public class FeedTupleTranslator extends AbstractTupleTranslator<Feed> {
public static final int FEED_PAYLOAD_TUPLE_FIELD_INDEX = 2;
@SuppressWarnings("unchecked")
- private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(MetadataRecordTypes.FEED_RECORDTYPE);
+ private ISerializerDeserializer<ARecord> recordSerDes =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(MetadataRecordTypes.FEED_RECORDTYPE);
- public FeedTupleTranslator(boolean getTuple) {
+ protected FeedTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.FEED_DATASET.getFieldCount());
}
@@ -87,8 +87,9 @@ public class FeedTupleTranslator extends AbstractTupleTranslator<Feed> {
private Feed createFeedFromARecord(ARecord feedRecord) {
Feed feed = null;
- String dataverseName = ((AString) feedRecord
- .getValueByPos(MetadataRecordTypes.FEED_ARECORD_DATAVERSE_NAME_FIELD_INDEX)).getStringValue();
+ String dataverseName =
+ ((AString) feedRecord.getValueByPos(MetadataRecordTypes.FEED_ARECORD_DATAVERSE_NAME_FIELD_INDEX))
+ .getStringValue();
String feedName = ((AString) feedRecord.getValueByPos(MetadataRecordTypes.FEED_ARECORD_FEED_NAME_FIELD_INDEX))
.getStringValue();
@@ -219,8 +220,8 @@ public class FeedTupleTranslator extends AbstractTupleTranslator<Feed> {
primaryDetailsRecordBuilder.reset(MetadataRecordTypes.PRIMARY_FEED_DETAILS_RECORDTYPE);
AMutableString aString = new AMutableString("");
- ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ ISerializerDeserializer<AString> stringSerde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
// write field 0
fieldValue.reset();
@@ -278,8 +279,8 @@ public class FeedTupleTranslator extends AbstractTupleTranslator<Feed> {
ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
propertyRecordBuilder.reset(MetadataRecordTypes.FEED_ADAPTER_CONFIGURATION_RECORDTYPE);
AMutableString aString = new AMutableString("");
- ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ ISerializerDeserializer<AString> stringSerde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
// write field 0
fieldValue.reset();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
index 72fe79d..f9e98e2 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
@@ -60,7 +60,7 @@ public class FunctionTupleTranslator extends AbstractTupleTranslator<Function> {
private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.FUNCTION_RECORDTYPE);
- public FunctionTupleTranslator(boolean getTuple) {
+ protected FunctionTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.FUNCTION_DATASET.getFieldCount());
}
@@ -71,15 +71,17 @@ public class FunctionTupleTranslator extends AbstractTupleTranslator<Function> {
int recordLength = frameTuple.getFieldLength(FUNCTION_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord functionRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord functionRecord = recordSerDes.deserialize(in);
return createFunctionFromARecord(functionRecord);
}
private Function createFunctionFromARecord(ARecord functionRecord) {
- String dataverseName = ((AString) functionRecord
- .getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_DATAVERSENAME_FIELD_INDEX)).getStringValue();
- String functionName = ((AString) functionRecord
- .getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTIONNAME_FIELD_INDEX)).getStringValue();
+ String dataverseName =
+ ((AString) functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_DATAVERSENAME_FIELD_INDEX))
+ .getStringValue();
+ String functionName =
+ ((AString) functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTIONNAME_FIELD_INDEX))
+ .getStringValue();
String arity = ((AString) functionRecord
.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_ARITY_FIELD_INDEX)).getStringValue();
@@ -99,8 +101,9 @@ public class FunctionTupleTranslator extends AbstractTupleTranslator<Function> {
String language = ((AString) functionRecord
.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX)).getStringValue();
- String functionKind = ((AString) functionRecord
- .getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX)).getStringValue();
+ String functionKind =
+ ((AString) functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX))
+ .getStringValue();
return new Function(dataverseName, functionName, Integer.parseInt(arity), params, returnType, definition,
language, functionKind);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index 5dee24e..bf27fc8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -76,15 +76,15 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
public static final String INDEX_ISENFORCED_FIELD_NAME = "IsEnforced";
public static final String INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME = "SearchKeySourceIndicator";
- private OrderedListBuilder listBuilder = new OrderedListBuilder();
- private OrderedListBuilder primaryKeyListBuilder = new OrderedListBuilder();
- private AOrderedListType stringList = new AOrderedListType(BuiltinType.ASTRING, null);
- private AOrderedListType int8List = new AOrderedListType(BuiltinType.AINT8, null);
- private ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
- private ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
- private List<List<String>> searchKey;
- private List<IAType> searchKeyType;
- private AMutableInt8 aInt8 = new AMutableInt8((byte) 0);
+ private transient OrderedListBuilder listBuilder = new OrderedListBuilder();
+ private transient OrderedListBuilder primaryKeyListBuilder = new OrderedListBuilder();
+ private transient AOrderedListType stringList = new AOrderedListType(BuiltinType.ASTRING, null);
+ private transient AOrderedListType int8List = new AOrderedListType(BuiltinType.AINT8, null);
+ private transient ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
+ private transient ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
+ private transient List<List<String>> searchKey;
+ private transient List<IAType> searchKeyType;
+ private transient AMutableInt8 aInt8 = new AMutableInt8((byte) 0);
@SuppressWarnings("unchecked")
private ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.AINT32);
@@ -97,7 +97,7 @@ public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
private final MetadataNode metadataNode;
private final JobId jobId;
- public IndexTupleTranslator(JobId jobId, MetadataNode metadataNode, boolean getTuple) {
+ protected IndexTupleTranslator(JobId jobId, MetadataNode metadataNode, boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.INDEX_DATASET.getFieldCount());
this.jobId = jobId;
this.metadataNode = metadataNode;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
index fb38638..d259212 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
@@ -52,7 +52,7 @@ public class LibraryTupleTranslator extends AbstractTupleTranslator<Library> {
private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.LIBRARY_RECORDTYPE);
- public LibraryTupleTranslator(boolean getTuple) {
+ protected LibraryTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.LIBRARY_DATASET.getFieldCount());
}
@@ -63,15 +63,17 @@ public class LibraryTupleTranslator extends AbstractTupleTranslator<Library> {
int recordLength = frameTuple.getFieldLength(LIBRARY_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord libraryRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord libraryRecord = recordSerDes.deserialize(in);
return createLibraryFromARecord(libraryRecord);
}
private Library createLibraryFromARecord(ARecord libraryRecord) {
- String dataverseName = ((AString) libraryRecord
- .getValueByPos(MetadataRecordTypes.LIBRARY_ARECORD_DATAVERSENAME_FIELD_INDEX)).getStringValue();
- String libraryName = ((AString) libraryRecord
- .getValueByPos(MetadataRecordTypes.LIBRARY_ARECORD_NAME_FIELD_INDEX)).getStringValue();
+ String dataverseName =
+ ((AString) libraryRecord.getValueByPos(MetadataRecordTypes.LIBRARY_ARECORD_DATAVERSENAME_FIELD_INDEX))
+ .getStringValue();
+ String libraryName =
+ ((AString) libraryRecord.getValueByPos(MetadataRecordTypes.LIBRARY_ARECORD_NAME_FIELD_INDEX))
+ .getStringValue();
return new Library(dataverseName, libraryName);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/MetadataTupleTranslatorProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/MetadataTupleTranslatorProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/MetadataTupleTranslatorProvider.java
new file mode 100644
index 0000000..20f04c4
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/MetadataTupleTranslatorProvider.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.entitytupletranslators;
+
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.metadata.MetadataNode;
+
+public class MetadataTupleTranslatorProvider {
+
+ public CompactionPolicyTupleTranslator getCompactionPolicyTupleTranslator(boolean getTuple) {
+ return new CompactionPolicyTupleTranslator(getTuple);
+ }
+
+ public DatasetTupleTranslator getDatasetTupleTranslator(boolean getTuple) {
+ return new DatasetTupleTranslator(getTuple);
+ }
+
+ public DatasourceAdapterTupleTranslator getAdapterTupleTranslator(boolean getTuple) {
+ return new DatasourceAdapterTupleTranslator(getTuple);
+ }
+
+ public DatatypeTupleTranslator getDataTypeTupleTranslator(JobId jobId, MetadataNode metadataNode,
+ boolean getTuple) {
+ return new DatatypeTupleTranslator(jobId, metadataNode, getTuple);
+ }
+
+ public DataverseTupleTranslator getDataverseTupleTranslator(boolean getTuple) {
+ return new DataverseTupleTranslator(getTuple);
+ }
+
+ public ExternalFileTupleTranslator getExternalFileTupleTranslator(boolean getTuple) {
+ return new ExternalFileTupleTranslator(getTuple);
+ }
+
+ public FeedPolicyTupleTranslator getFeedPolicyTupleTranslator(boolean getTuple) {
+ return new FeedPolicyTupleTranslator(getTuple);
+ }
+
+ public FeedTupleTranslator getFeedTupleTranslator(boolean getTuple) {
+ return new FeedTupleTranslator(getTuple);
+ }
+
+ public FunctionTupleTranslator getFunctionTupleTranslator(boolean getTuple) {
+ return new FunctionTupleTranslator(getTuple);
+ }
+
+ public IndexTupleTranslator getIndexTupleTranslator(JobId jobId, MetadataNode metadataNode, boolean getTuple) {
+ return new IndexTupleTranslator(jobId, metadataNode, getTuple);
+ }
+
+ public LibraryTupleTranslator getLibraryTupleTranslator(boolean getTuple) {
+ return new LibraryTupleTranslator(getTuple);
+ }
+
+ public NodeTupleTranslator getNodeTupleTranslator(boolean getTuple) {
+ return new NodeTupleTranslator(getTuple);
+ }
+
+ public NodeGroupTupleTranslator getNodeGroupTupleTranslator(boolean getTuple) {
+ return new NodeGroupTupleTranslator(getTuple);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java
index af83ee9..397021d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeGroupTupleTranslator.java
@@ -47,20 +47,21 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
*/
public class NodeGroupTupleTranslator extends AbstractTupleTranslator<NodeGroup> {
+ private static final long serialVersionUID = 1L;
// Field indexes of serialized NodeGroup in a tuple.
// First key field.
public static final int NODEGROUP_NODEGROUPNAME_TUPLE_FIELD_INDEX = 0;
// Payload field containing serialized NodeGroup.
public static final int NODEGROUP_PAYLOAD_TUPLE_FIELD_INDEX = 1;
- private UnorderedListBuilder listBuilder = new UnorderedListBuilder();
- private ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
+ private transient UnorderedListBuilder listBuilder = new UnorderedListBuilder();
+ private transient ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
private List<String> nodeNames;
@SuppressWarnings("unchecked")
private ISerializerDeserializer<ARecord> recordSerDes = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.NODEGROUP_RECORDTYPE);
- public NodeGroupTupleTranslator(boolean getTuple) {
+ protected NodeGroupTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.NODEGROUP_DATASET.getFieldCount());
}
@@ -71,7 +72,7 @@ public class NodeGroupTupleTranslator extends AbstractTupleTranslator<NodeGroup>
int recordLength = frameTuple.getFieldLength(NODEGROUP_PAYLOAD_TUPLE_FIELD_INDEX);
ByteArrayInputStream stream = new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
DataInput in = new DataInputStream(stream);
- ARecord nodeGroupRecord = (ARecord) recordSerDes.deserialize(in);
+ ARecord nodeGroupRecord = recordSerDes.deserialize(in);
String gpName = ((AString) nodeGroupRecord
.getValueByPos(MetadataRecordTypes.NODEGROUP_ARECORD_GROUPNAME_FIELD_INDEX)).getStringValue();
IACursor cursor = ((AUnorderedList) nodeGroupRecord
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java
index 55b61db..a54be65 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/NodeTupleTranslator.java
@@ -44,7 +44,7 @@ public class NodeTupleTranslator extends AbstractTupleTranslator<Node> {
// Payload field containing serialized Node.
public static final int NODE_PAYLOAD_TUPLE_FIELD_INDEX = 1;
- private AMutableInt64 aInt64 = new AMutableInt64(-1);
+ private transient AMutableInt64 aInt64 = new AMutableInt64(-1);
@SuppressWarnings("unchecked")
private ISerializerDeserializer<AInt64> int64Serde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.AINT64);
@@ -54,7 +54,7 @@ public class NodeTupleTranslator extends AbstractTupleTranslator<Node> {
// NonTaggedSerializerDeserializerProvider.INSTANCE
// .getSerializerDeserializer(recordType);
- public NodeTupleTranslator(boolean getTuple) {
+ protected NodeTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.NODE_DATASET.getFieldCount());
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java
index 8ef6732..57284b0 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/BuiltinFeedPolicies.java
@@ -21,9 +21,9 @@ package org.apache.asterix.metadata.feeds;
import java.util.HashMap;
import java.util.Map;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.asterix.metadata.entities.FeedPolicyEntity;
+import org.apache.asterix.metadata.utils.MetadataConstants;
public class BuiltinFeedPolicies {
@@ -152,7 +152,8 @@ public class BuiltinFeedPolicies {
policyParams.put(FeedPolicyAccessor.TIME_TRACKING, "true");
String description = "AdvancedFT 100% Discard during congestion";
- return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Spill", description, policyParams);
+ return new FeedPolicyEntity(MetadataConstants.METADATA_DATAVERSE_NAME, "AdvancedFT_Spill", description,
+ policyParams);
}
// AdvancedFT_Spill
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
index 12ab634..f91ab8e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/FeedMetadataUtil.java
@@ -25,12 +25,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.dataflow.AsterixLSMTreeInsertDeleteOperatorDescriptor;
import org.apache.asterix.common.exceptions.ACIDException;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -59,6 +57,7 @@ import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Feed;
import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
@@ -81,10 +80,8 @@ import org.apache.hyracks.api.dataflow.IConnectorDescriptor;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.common.data.partition.RandomPartitionComputerFactory;
import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import org.apache.hyracks.dataflow.std.connectors.MToNPartitioningWithMessageConnectorDescriptor;
@@ -96,11 +93,6 @@ public class FeedMetadataUtil {
private static final Logger LOGGER = Logger.getLogger(FeedMetadataUtil.class.getName());
- private static class LocationConstraint {
- int partition;
- String location;
- }
-
public static Dataset validateIfDatasetExists(String dataverse, String datasetName, MetadataTransactionContext ctx)
throws AsterixException {
Dataset dataset = MetadataManager.INSTANCE.getDataset(ctx, dataverse, datasetName);
@@ -128,8 +120,8 @@ public class FeedMetadataUtil {
MetadataTransactionContext ctx) throws AsterixException {
FeedPolicyEntity feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(ctx, dataverse, policyName);
if (feedPolicy == null) {
- feedPolicy =
- MetadataManager.INSTANCE.getFeedPolicy(ctx, MetadataConstants.METADATA_DATAVERSE_NAME, policyName);
+ feedPolicy = MetadataManager.INSTANCE.getFeedPolicy(ctx, MetadataConstants.METADATA_DATAVERSE_NAME,
+ policyName);
if (feedPolicy == null) {
throw new AsterixException("Unknown feed policy" + policyName);
}
@@ -170,11 +162,11 @@ public class FeedMetadataUtil {
boolean enableSubscriptionMode;
OperatorDescriptorId opId = null;
if (opDesc instanceof AlgebricksMetaOperatorDescriptor) {
- IPushRuntimeFactory[] runtimeFactories =
- ((AlgebricksMetaOperatorDescriptor) opDesc).getPipeline().getRuntimeFactories();
+ IPushRuntimeFactory[] runtimeFactories = ((AlgebricksMetaOperatorDescriptor) opDesc).getPipeline()
+ .getRuntimeFactories();
if (runtimeFactories[0] instanceof AssignRuntimeFactory && runtimeFactories.length > 1) {
- IConnectorDescriptor connectorDesc =
- spec.getOperatorInputMap().get(opDesc.getOperatorId()).get(0);
+ IConnectorDescriptor connectorDesc = spec.getOperatorInputMap().get(opDesc.getOperatorId())
+ .get(0);
IOperatorDescriptor sourceOp = spec.getProducer(connectorDesc);
if (sourceOp instanceof FeedCollectOperatorDescriptor) {
runtimeType = FeedRuntimeType.COMPUTE;
@@ -209,16 +201,16 @@ public class FeedMetadataUtil {
}
// make connections between operators
- for (Entry<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>>
- entry : spec.getConnectorOperatorMap().entrySet()) {
+ for (Entry<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>,
+ Pair<IOperatorDescriptor, Integer>>> entry : spec.getConnectorOperatorMap().entrySet()) {
IConnectorDescriptor connDesc = altered.getConnectorMap().get(connectorMapping.get(entry.getKey()));
Pair<IOperatorDescriptor, Integer> leftOp = entry.getValue().getLeft();
Pair<IOperatorDescriptor, Integer> rightOp = entry.getValue().getRight();
- IOperatorDescriptor leftOpDesc =
- altered.getOperatorMap().get(oldNewOID.get(leftOp.getLeft().getOperatorId()));
- IOperatorDescriptor rightOpDesc =
- altered.getOperatorMap().get(oldNewOID.get(rightOp.getLeft().getOperatorId()));
+ IOperatorDescriptor leftOpDesc = altered.getOperatorMap()
+ .get(oldNewOID.get(leftOp.getLeft().getOperatorId()));
+ IOperatorDescriptor rightOpDesc = altered.getOperatorMap()
+ .get(oldNewOID.get(rightOp.getLeft().getOperatorId()));
altered.connect(connDesc, leftOpDesc, leftOp.getRight(), rightOpDesc, rightOp.getRight());
}
@@ -246,9 +238,8 @@ public class FeedMetadataUtil {
operatorLocations.put(opDesc.getOperatorId(), locations);
}
String location = (String) ((ConstantExpression) cexpr).getValue();
- LocationConstraint lc = new LocationConstraint();
- lc.location = location;
- lc.partition = ((PartitionLocationExpression) lexpr).getPartition();
+ LocationConstraint lc = new LocationConstraint(location,
+ ((PartitionLocationExpression) lexpr).getPartition());
locations.add(lc);
break;
default:
@@ -299,125 +290,6 @@ public class FeedMetadataUtil {
}
- public static void increaseCardinality(JobSpecification spec, FeedRuntimeType compute, int requiredCardinality,
- List<String> newLocations) throws AsterixException {
- IOperatorDescriptor changingOpDesc = alterJobSpecForComputeCardinality(spec, requiredCardinality);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, changingOpDesc,
- nChooseK(requiredCardinality, newLocations));
-
- }
-
- public static void decreaseComputeCardinality(JobSpecification spec, FeedRuntimeType compute,
- int requiredCardinality, List<String> currentLocations) throws AsterixException {
- IOperatorDescriptor changingOpDesc = alterJobSpecForComputeCardinality(spec, requiredCardinality);
- String[] chosenLocations = nChooseK(requiredCardinality, currentLocations);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, changingOpDesc, chosenLocations);
- }
-
- private static IOperatorDescriptor alterJobSpecForComputeCardinality(
- JobSpecification spec, int requiredCardinality) throws AsterixException {
- Map<ConnectorDescriptorId, IConnectorDescriptor> connectors = spec.getConnectorMap();
- Map<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>>
- connectorOpMap = spec.getConnectorOperatorMap();
- IOperatorDescriptor sourceOp = null;
- IOperatorDescriptor targetOp = null;
- IConnectorDescriptor connDesc = null;
- for (Entry<ConnectorDescriptorId, Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>>
- entry : connectorOpMap.entrySet()) {
- ConnectorDescriptorId cid = entry.getKey();
- sourceOp = entry.getValue().getKey().getKey();
- if (sourceOp instanceof FeedCollectOperatorDescriptor) {
- targetOp = entry.getValue().getValue().getKey();
- if ((targetOp instanceof FeedMetaOperatorDescriptor)
- && (((FeedMetaOperatorDescriptor) targetOp).getRuntimeType()
- .equals(FeedRuntimeType.COMPUTE))) {
- connDesc = connectors.get(cid);
- break;
- } else {
- throw new AsterixException("Incorrect manipulation, feed does not have a compute stage");
- }
- }
- }
-
- Map<OperatorDescriptorId, List<IConnectorDescriptor>> operatorInputMap = spec.getOperatorInputMap();
- boolean removed = operatorInputMap.get(targetOp.getOperatorId()).remove(connDesc);
- if (!removed) {
- throw new AsterixException("Connector desc not found");
- }
- Map<OperatorDescriptorId, List<IConnectorDescriptor>> operatorOutputMap = spec.getOperatorOutputMap();
- removed = operatorOutputMap.get(sourceOp.getOperatorId()).remove(connDesc);
- if (!removed) {
- throw new AsterixException("Connector desc not found");
- }
- spec.getConnectorMap().remove(connDesc.getConnectorId());
- connectorOpMap.remove(connDesc.getConnectorId());
-
- ITuplePartitionComputerFactory tpcf = new RandomPartitionComputerFactory();
- MToNPartitioningConnectorDescriptor newConnector = new MToNPartitioningConnectorDescriptor(spec, tpcf);
- spec.getConnectorMap().put(newConnector.getConnectorId(), newConnector);
- spec.connect(newConnector, sourceOp, 0, targetOp, 0);
-
- // ==============================================================================
- Set<Constraint> userConstraints = spec.getUserConstraints();
- Constraint countConstraint = null;
- Constraint locationConstraint = null;
- List<LocationConstraint> locations = new ArrayList<LocationConstraint>();
- IOperatorDescriptor changingOpDesc = null;
-
- for (Constraint constraint : userConstraints) {
- LValueConstraintExpression lexpr = constraint.getLValue();
- ConstraintExpression cexpr = constraint.getRValue();
- OperatorDescriptorId opId;
- switch (lexpr.getTag()) {
- case PARTITION_COUNT: {
- opId = ((PartitionCountExpression) lexpr).getOperatorDescriptorId();
- IOperatorDescriptor opDesc = spec.getOperatorMap().get(opId);
- if (opDesc instanceof FeedMetaOperatorDescriptor) {
- FeedRuntimeType runtimeType = ((FeedMetaOperatorDescriptor) opDesc).getRuntimeType();
- if (runtimeType.equals(FeedRuntimeType.COMPUTE)) {
- countConstraint = constraint;
- changingOpDesc = opDesc;
- }
- }
- break;
- }
- case PARTITION_LOCATION:
- opId = ((PartitionLocationExpression) lexpr).getOperatorDescriptorId();
- IOperatorDescriptor opDesc = spec.getOperatorMap().get(opId);
- if (opDesc instanceof FeedMetaOperatorDescriptor) {
- FeedRuntimeType runtimeType = ((FeedMetaOperatorDescriptor) opDesc).getRuntimeType();
- if (runtimeType.equals(FeedRuntimeType.COMPUTE)) {
- locationConstraint = constraint;
- changingOpDesc = opDesc;
- String location = (String) ((ConstantExpression) cexpr).getValue();
- LocationConstraint lc = new LocationConstraint();
- lc.location = location;
- lc.partition = ((PartitionLocationExpression) lexpr).getPartition();
- locations.add(lc);
- }
- }
- break;
- default:
- break;
- }
- }
-
- userConstraints.remove(countConstraint);
- if (locationConstraint != null) {
- userConstraints.remove(locationConstraint);
- }
-
- return changingOpDesc;
- }
-
- private static String[] nChooseK(int k, List<String> locations) {
- String[] result = new String[k];
- for (int i = 0; i < k; i++) {
- result[i] = locations.get(i);
- }
- return result;
- }
-
private static boolean preProcessingRequired(FeedConnectionId connectionId) {
MetadataTransactionContext ctx = null;
Feed feed = null;
@@ -584,8 +456,7 @@ public class FeedMetadataUtil {
if (ExternalDataUtils.isChangeFeed(configuration)) {
getSerdesForPKs(serdes, configuration, metaType, adapterOutputType, i);
}
- feedProps = new Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType>(adapterFactory,
- new RecordDescriptor(serdes), adapterType);
+ feedProps = new Triple<>(adapterFactory, new RecordDescriptor(serdes), adapterType);
} catch (Exception e) {
throw new AlgebricksException("unable to create adapter", e);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/LocationConstraint.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/LocationConstraint.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/LocationConstraint.java
new file mode 100644
index 0000000..24d6c5a
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/feeds/LocationConstraint.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.feeds;
+
+public class LocationConstraint {
+ public final String location;
+ public final int partition;
+
+ public LocationConstraint(String location, int partition) {
+ this.location = location;
+ this.partition = partition;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
index d0af744..56439b6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtils.java
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.asterix.builders.IARecordBuilder;
import org.apache.asterix.builders.RecordBuilder;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.external.indexing.IndexingConstants;
@@ -257,8 +256,8 @@ public class DatasetUtils {
ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
propertyRecordBuilder.reset(recordType);
AMutableString aString = new AMutableString("");
- ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(BuiltinType.ASTRING);
+ ISerializerDeserializer<AString> stringSerde =
+ AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
// write field 0
fieldValue.reset();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
new file mode 100644
index 0000000..fbd2380
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataConstants.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.metadata.utils;
+
+public class MetadataConstants {
+ private MetadataConstants() {
+ }
+
+ // Name of the dataverse the metadata lives in.
+ public static final String METADATA_DATAVERSE_NAME = "Metadata";
+
+ // Name of the node group where metadata is stored on.
+ public static final String METADATA_NODEGROUP_NAME = "MetadataGroup";
+
+ // Name of the default nodegroup where internal/feed datasets will be partitioned
+ // if an explicit nodegroup is not specified at the time of creation of a dataset
+ public static final String METADATA_DEFAULT_NODEGROUP_NAME = "DEFAULT_NG_ALL_NODES";
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
index 601ce15..a31f2a6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SplitsAndConstraintsUtil.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.asterix.common.cluster.ClusterPartition;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
@@ -38,16 +37,19 @@ import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
public class SplitsAndConstraintsUtil {
+ private SplitsAndConstraintsUtil() {
+ }
+
private static FileSplit[] splitsForDataverse(String dataverseName) {
File relPathFile = new File(dataverseName);
- List<FileSplit> splits = new ArrayList<FileSplit>();
+ List<FileSplit> splits = new ArrayList<>();
// get all partitions
ClusterPartition[] clusterPartition = AsterixClusterProperties.INSTANCE.getClusterPartitons();
String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
for (int j = 0; j < clusterPartition.length; j++) {
- int nodeParitions = AsterixClusterProperties.INSTANCE
- .getNodePartitionsCount(clusterPartition[j].getNodeId());
- for (int i = 0; i < nodeParitions; i++) {
+ int nodePartitions =
+ AsterixClusterProperties.INSTANCE.getNodePartitionsCount(clusterPartition[j].getNodeId());
+ for (int i = 0; i < nodePartitions; i++) {
File f = new File(StoragePathUtil.prepareStoragePartitionPath(storageDirName,
clusterPartition[i].getPartitionId()) + File.separator + relPathFile);
splits.add(StoragePathUtil.getFileSplitForClusterPartition(clusterPartition[j], f));
@@ -59,17 +61,17 @@ public class SplitsAndConstraintsUtil {
public static FileSplit[] splitsForDataset(MetadataTransactionContext mdTxnCtx, String dataverseName,
String datasetName, String targetIdxName, boolean temp) throws AlgebricksException {
try {
- File relPathFile = new File(
- StoragePathUtil.prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
+ File relPathFile =
+ new File(StoragePathUtil.prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
- List<String> nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName())
- .getNodeNames();
+ List<String> nodeGroup =
+ MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName()).getNodeNames();
if (nodeGroup == null) {
throw new AlgebricksException("Couldn't find node group " + dataset.getNodeGroupName());
}
String storageDirName = AsterixClusterProperties.INSTANCE.getStorageDirectoryName();
- List<FileSplit> splits = new ArrayList<FileSplit>();
+ List<FileSplit> splits = new ArrayList<>();
for (String nd : nodeGroup) {
int numPartitions = AsterixClusterProperties.INSTANCE.getNodePartitionsCount(nd);
ClusterPartition[] nodePartitions = AsterixClusterProperties.INSTANCE.getNodePartitions(nd);
@@ -96,16 +98,16 @@ public class SplitsAndConstraintsUtil {
private static FileSplit[] splitsForFilesIndex(MetadataTransactionContext mdTxnCtx, String dataverseName,
String datasetName, String targetIdxName, boolean create) throws AlgebricksException {
try {
- File relPathFile = new File(
- StoragePathUtil.prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
+ File relPathFile =
+ new File(StoragePathUtil.prepareDataverseIndexName(dataverseName, datasetName, targetIdxName));
Dataset dataset = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
- List<String> nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName())
- .getNodeNames();
+ List<String> nodeGroup =
+ MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName()).getNodeNames();
if (nodeGroup == null) {
throw new AlgebricksException("Couldn't find node group " + dataset.getNodeGroupName());
}
- List<FileSplit> splits = new ArrayList<FileSplit>();
+ List<FileSplit> splits = new ArrayList<>();
for (String nodeId : nodeGroup) {
// get node partitions
ClusterPartition[] nodePartitions = AsterixClusterProperties.INSTANCE.getNodePartitions(nodeId);
@@ -130,15 +132,15 @@ public class SplitsAndConstraintsUtil {
}
}
- public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataverse(
- String dataverse) {
+ public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
+ splitProviderAndPartitionConstraintsForDataverse(String dataverse) {
FileSplit[] splits = splitsForDataverse(dataverse);
return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
- public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForFilesIndex(
- MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName, String targetIdxName,
- boolean create) throws AlgebricksException {
+ public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint>
+ splitProviderAndPartitionConstraintsForFilesIndex(MetadataTransactionContext mdTxnCtx, String dataverseName,
+ String datasetName, String targetIdxName, boolean create) throws AlgebricksException {
FileSplit[] splits = splitsForFilesIndex(mdTxnCtx, dataverseName, datasetName, targetIdxName, create);
return StoragePathUtil.splitProviderAndPartitionConstraints(splits);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
index 2671834..6f3bd30 100644
--- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
+++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
@@ -25,7 +25,6 @@ import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.metadata.MetadataException;
-import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
@@ -51,11 +50,9 @@ public class DatasetTupleTranslatorTest {
indicator == null ? null : Collections.singletonList(indicator),
Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
- Dataset dataset = new Dataset("test", "log", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES",
- "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115,
- 0);
-
- MetadataPrimaryIndexes.init();
+ Dataset dataset =
+ new Dataset("test", "log", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES", "prefix",
+ compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115, 0);
DatasetTupleTranslator dtTranslator = new DatasetTupleTranslator(true);
ITupleReference tuple = dtTranslator.getTupleFromMetadataEntity(dataset);
Dataset deserializedDataset = dtTranslator.getMetadataEntityFromTuple(tuple);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
index 02b3e4b..46bacdd 100644
--- a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
+++ b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
@@ -32,7 +32,6 @@ import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataNode;
-import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Datatype;
import org.apache.asterix.metadata.entities.Index;
@@ -62,17 +61,15 @@ public class IndexTupleTranslatorTest {
indicator == null ? null : Collections.singletonList(indicator),
Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
- Dataset dataset = new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES",
- "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115,
- 0);
+ Dataset dataset =
+ new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES", "prefix",
+ compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115, 0);
Index index = new Index("test", "d1", "i1", IndexType.BTREE,
Collections.singletonList(Collections.singletonList("row_id")),
indicator == null ? null : Collections.singletonList(indicator),
Collections.singletonList(BuiltinType.AINT64), -1, false, false, 0);
- MetadataPrimaryIndexes.init();
-
MetadataNode mockMetadataNode = mock(MetadataNode.class);
when(mockMetadataNode.getDatatype(any(), anyString(), anyString())).thenReturn(new Datatype("test", "d1",
new ARecordType("", new String[] { "row_id" }, new IAType[] { BuiltinType.AINT64 }, true), true));
[04/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java
index d387041..8201627 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataPrimaryIndexes.java
@@ -21,8 +21,8 @@ package org.apache.asterix.metadata.bootstrap;
import java.util.Arrays;
-import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.api.IMetadataIndex;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
@@ -30,99 +30,104 @@ import org.apache.asterix.om.types.IAType;
* Contains static primary-index descriptors of all metadata datasets.
*/
public class MetadataPrimaryIndexes {
-
- public static IMetadataIndex DATAVERSE_DATASET;
- public static IMetadataIndex DATASET_DATASET;
- public static IMetadataIndex DATATYPE_DATASET;
- public static IMetadataIndex INDEX_DATASET;
- public static IMetadataIndex NODE_DATASET;
- public static IMetadataIndex NODEGROUP_DATASET;
- public static IMetadataIndex FUNCTION_DATASET;
- public static IMetadataIndex DATASOURCE_ADAPTER_DATASET;
- public static IMetadataIndex LIBRARY_DATASET;
- public static IMetadataIndex FEED_DATASET;
- public static IMetadataIndex FEED_ACTIVITY_DATASET;
- public static IMetadataIndex FEED_POLICY_DATASET;
- public static IMetadataIndex COMPACTION_POLICY_DATASET;
- public static IMetadataIndex EXTERNAL_FILE_DATASET;
-
- /**
- * Create all metadata primary index descriptors. MetadataRecordTypes must
- * have been initialized before calling this init.
- * @throws MetadataException
- * If MetadataRecordTypes have not been initialized.
- */
- public static void init() throws MetadataException {
- // Make sure the MetadataRecordTypes have been initialized.
- if (MetadataRecordTypes.DATASET_RECORDTYPE == null) {
- throw new MetadataException(
- "Must initialize MetadataRecordTypes before initializing MetadataPrimaryIndexes");
- }
-
- DATAVERSE_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.DATAVERSE, 2,
- new IAType[] { BuiltinType.ASTRING }, (Arrays.asList(Arrays.asList("DataverseName"))), 0,
- MetadataRecordTypes.DATAVERSE_RECORDTYPE, true, new int[] { 0 });
-
- DATASET_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.DATASET, 3,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("DatasetName"))), 0,
- MetadataRecordTypes.DATASET_RECORDTYPE, true, new int[] { 0, 1 });
-
- DATATYPE_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.DATATYPE, 3,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("DatatypeName"))), 0,
- MetadataRecordTypes.DATATYPE_RECORDTYPE, true, new int[] { 0, 1 });
-
- INDEX_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.INDEX, 4,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("DatasetName"),
- Arrays.asList("IndexName"))),
- 0, MetadataRecordTypes.INDEX_RECORDTYPE, true, new int[] { 0, 1, 2 });
-
- NODE_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.NODE, 2, new IAType[] { BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("NodeName"))), 0, MetadataRecordTypes.NODE_RECORDTYPE, true,
- new int[] { 0 });
-
- NODEGROUP_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.NODEGROUP, 2,
- new IAType[] { BuiltinType.ASTRING }, (Arrays.asList(Arrays.asList("GroupName"))), 0,
- MetadataRecordTypes.NODEGROUP_RECORDTYPE, true, new int[] { 0 });
-
- FUNCTION_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.FUNCTION, 4,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("Name"), Arrays.asList("Arity"))), 0,
- MetadataRecordTypes.FUNCTION_RECORDTYPE, true, new int[] { 0, 1, 2 });
-
- DATASOURCE_ADAPTER_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.DATASOURCE_ADAPTER, 3,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("Name"))), 0,
- MetadataRecordTypes.DATASOURCE_ADAPTER_RECORDTYPE, true,
- new int[] { 0, 1 });
-
- FEED_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.FEED, 3,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("FeedName"))), 0,
- MetadataRecordTypes.FEED_RECORDTYPE, true, new int[] { 0, 1 });
-
- LIBRARY_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.LIBRARY, 3,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("Name"))), 0,
- MetadataRecordTypes.LIBRARY_RECORDTYPE, true, new int[] { 0, 1 });
-
- FEED_POLICY_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.FEED_POLICY, 3,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("PolicyName"))), 0,
- MetadataRecordTypes.FEED_POLICY_RECORDTYPE, true, new int[] { 0, 1 });
-
- COMPACTION_POLICY_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.COMPACTION_POLICY, 3,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("CompactionPolicy"))), 0,
- MetadataRecordTypes.COMPACTION_POLICY_RECORDTYPE, true,
- new int[] { 0, 1 });
-
- EXTERNAL_FILE_DATASET = new MetadataIndex(MetadataIndexImmutableProperties.EXTERNAL_FILE, 4,
- new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.AINT32 },
- (Arrays.asList(Arrays.asList("DataverseName"), Arrays.asList("DatasetName"),
- Arrays.asList("FileNumber"))),
- 0, MetadataRecordTypes.EXTERNAL_FILE_RECORDTYPE, true, new int[] { 0, 1, 2 });
+ public static final MetadataIndexImmutableProperties PROPERTIES_METADATA =
+ new MetadataIndexImmutableProperties(MetadataConstants.METADATA_DATAVERSE_NAME, 0, 0);
+ public static final MetadataIndexImmutableProperties PROPERTIES_DATAVERSE =
+ new MetadataIndexImmutableProperties("Dataverse", 1, 1);
+ public static final MetadataIndexImmutableProperties PROPERTIES_DATASET =
+ new MetadataIndexImmutableProperties("Dataset", 2, 2);
+ public static final MetadataIndexImmutableProperties PROPERTIES_DATATYPE =
+ new MetadataIndexImmutableProperties("Datatype", 3, 3);
+ public static final MetadataIndexImmutableProperties PROPERTIES_INDEX =
+ new MetadataIndexImmutableProperties("Index", 4, 4);
+ public static final MetadataIndexImmutableProperties PROPERTIES_NODE =
+ new MetadataIndexImmutableProperties("Node", 5, 5);
+ public static final MetadataIndexImmutableProperties PROPERTIES_NODEGROUP =
+ new MetadataIndexImmutableProperties("Nodegroup", 6, 6);
+ public static final MetadataIndexImmutableProperties PROPERTIES_FUNCTION =
+ new MetadataIndexImmutableProperties("Function", 7, 7);
+ public static final MetadataIndexImmutableProperties PROPERTIES_DATASOURCE_ADAPTER =
+ new MetadataIndexImmutableProperties("DatasourceAdapter", 8, 8);
+ public static final MetadataIndexImmutableProperties PROPERTIES_LIBRARY =
+ new MetadataIndexImmutableProperties("Library", 9, 9);
+ public static final MetadataIndexImmutableProperties PROPERTIES_FEED =
+ new MetadataIndexImmutableProperties("Feed", 10, 10);
+ public static final MetadataIndexImmutableProperties PROPERTIES_FEED_ACTIVITY_DATASET_ID =
+ new MetadataIndexImmutableProperties("FeedActivity", 11, 11);
+ public static final MetadataIndexImmutableProperties PROPERTIES_FEED_POLICY =
+ new MetadataIndexImmutableProperties("FeedPolicy", 12, 12);
+ public static final MetadataIndexImmutableProperties PROPERTIES_COMPACTION_POLICY =
+ new MetadataIndexImmutableProperties("CompactionPolicy", 13, 13);
+ public static final MetadataIndexImmutableProperties PROPERTIES_EXTERNAL_FILE =
+ new MetadataIndexImmutableProperties("ExternalFile", 14, 14);
+
+ public static final IMetadataIndex DATAVERSE_DATASET =
+ new MetadataIndex(PROPERTIES_DATAVERSE, 2, new IAType[] { BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME)), 0,
+ MetadataRecordTypes.DATAVERSE_RECORDTYPE, true, new int[] { 0 });
+ public static final IMetadataIndex DATASET_DATASET =
+ new MetadataIndex(PROPERTIES_DATASET, 3, new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATASET_NAME)),
+ 0, MetadataRecordTypes.DATASET_RECORDTYPE, true, new int[] { 0, 1 });
+ public static final IMetadataIndex DATATYPE_DATASET =
+ new MetadataIndex(PROPERTIES_DATATYPE, 3, new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATATYPE_NAME)),
+ 0, MetadataRecordTypes.DATATYPE_RECORDTYPE, true, new int[] { 0, 1 });
+ public static final IMetadataIndex INDEX_DATASET = new MetadataIndex(PROPERTIES_INDEX, 4,
+ new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATASET_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_INDEX_NAME)),
+ 0, MetadataRecordTypes.INDEX_RECORDTYPE, true, new int[] { 0, 1, 2 });
+ public static final IMetadataIndex NODE_DATASET =
+ new MetadataIndex(PROPERTIES_NODE, 2, new IAType[] { BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_NODE_NAME)), 0,
+ MetadataRecordTypes.NODE_RECORDTYPE, true, new int[] { 0 });
+ public static final IMetadataIndex NODEGROUP_DATASET =
+ new MetadataIndex(PROPERTIES_NODEGROUP, 2, new IAType[] { BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_GROUP_NAME)), 0,
+ MetadataRecordTypes.NODEGROUP_RECORDTYPE, true, new int[] { 0 });
+ public static final IMetadataIndex FUNCTION_DATASET = new MetadataIndex(PROPERTIES_FUNCTION, 4,
+ new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_ARITY)),
+ 0, MetadataRecordTypes.FUNCTION_RECORDTYPE, true, new int[] { 0, 1, 2 });
+ public static final IMetadataIndex DATASOURCE_ADAPTER_DATASET = new MetadataIndex(PROPERTIES_DATASOURCE_ADAPTER, 3,
+ new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_NAME)),
+ 0, MetadataRecordTypes.DATASOURCE_ADAPTER_RECORDTYPE, true, new int[] { 0, 1 });
+ public static final IMetadataIndex LIBRARY_DATASET =
+ new MetadataIndex(PROPERTIES_LIBRARY, 3, new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_NAME)),
+ 0, MetadataRecordTypes.LIBRARY_RECORDTYPE, true, new int[] { 0, 1 });
+ public static final IMetadataIndex FEED_DATASET =
+ new MetadataIndex(PROPERTIES_FEED, 3, new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_FEED_NAME)),
+ 0, MetadataRecordTypes.FEED_RECORDTYPE, true, new int[] { 0, 1 });
+ public static final IMetadataIndex FEED_ACTIVITY_DATASET = null;
+ public static final IMetadataIndex FEED_POLICY_DATASET =
+ new MetadataIndex(PROPERTIES_FEED_POLICY, 3, new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_POLICY_NAME)),
+ 0, MetadataRecordTypes.FEED_POLICY_RECORDTYPE, true, new int[] { 0, 1 });
+ public static final IMetadataIndex COMPACTION_POLICY_DATASET = new MetadataIndex(PROPERTIES_COMPACTION_POLICY, 3,
+ new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_COMPACTION_POLICY)),
+ 0, MetadataRecordTypes.COMPACTION_POLICY_RECORDTYPE, true, new int[] { 0, 1 });
+ public static final IMetadataIndex EXTERNAL_FILE_DATASET = new MetadataIndex(PROPERTIES_EXTERNAL_FILE, 4,
+ new IAType[] { BuiltinType.ASTRING, BuiltinType.ASTRING, BuiltinType.AINT32 },
+ Arrays.asList(Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATAVERSE_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_DATASET_NAME),
+ Arrays.asList(MetadataRecordTypes.FIELD_NAME_FILE_NUMBER)),
+ 0, MetadataRecordTypes.EXTERNAL_FILE_RECORDTYPE, true, new int[] { 0, 1, 2 });
+
+ private MetadataPrimaryIndexes() {
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
index 14ef049..a783c63 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java
@@ -510,7 +510,7 @@ public final class MetadataRecordTypes {
private MetadataRecordTypes() {
}
- private static ARecordType createRecordType(String recordTypeName, String[] fieldNames, IAType[] fieldTypes,
+ public static ARecordType createRecordType(String recordTypeName, String[] fieldNames, IAType[] fieldTypes,
boolean isOpen) {
ARecordType recordType = new ARecordType(recordTypeName, fieldNames, fieldTypes, isOpen);
if (recordTypeName != null) {
@@ -519,7 +519,7 @@ public final class MetadataRecordTypes {
return recordType;
}
- private static final ARecordType createPropertiesRecordType() {
+ public static final ARecordType createPropertiesRecordType() {
return createRecordType(
// RecordTypeName
null,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
index d809f4f..1272d03 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlDataSource.java
@@ -27,11 +27,15 @@ import java.util.Map;
import java.util.Set;
import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.ListSet;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
@@ -43,6 +47,9 @@ import org.apache.hyracks.algebricks.core.algebra.properties.OrderColumn;
import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
import org.apache.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.job.JobSpecification;
public abstract class AqlDataSource implements IDataSource<AqlSourceId> {
@@ -217,4 +224,10 @@ public abstract class AqlDataSource implements IDataSource<AqlSourceId> {
return new ArrayList<>(dataScanVariables.subList(0, dataScanVariables.size() - (hasMeta() ? 2 : 1)));
}
+ public abstract Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime(
+ AqlMetadataProvider aqlMetadataProvider, IDataSource<AqlSourceId> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+ throws AlgebricksException;
}
\ No newline at end of file
[08/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtension.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtension.java
new file mode 100644
index 0000000..7417bc6
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtension.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.config;
+
+import java.util.List;
+
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class AsterixExtension {
+ private final String className;
+ private final List<Pair<String, String>> args;
+
+ public AsterixExtension(String className, List<Pair<String, String>> args) {
+ this.className = className;
+ this.args = args;
+ }
+
+ public List<Pair<String, String>> getArgs() {
+ return args;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java
new file mode 100644
index 0000000..e8f2bcc
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixExtensionProperties.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.config;
+
+import java.util.List;
+
+public class AsterixExtensionProperties extends AbstractAsterixProperties {
+
+ public AsterixExtensionProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public List<AsterixExtension> getExtensions() {
+ return accessor.getExtensions();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
new file mode 100644
index 0000000..1576774
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixProperties.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.config;
+
+import java.io.File;
+
+public class AsterixProperties {
+ //---------------------------- Directories ---------------------------//
+ private static final String VAR = File.separator + "var";
+ private static final String LIB = VAR + File.separator + "lib";
+ private static final String ASTERIXDB = LIB + File.separator + "asterixdb";
+ //----------------------------- Sections -----------------------------//
+ public static final String SECTION_ASTERIX = "asterix";
+ public static final String SECTION_PREFIX_EXTENSION = "extension/";
+ public static final String SECTION_CC = "cc";
+ public static final String SECTION_PREFIX_NC = "nc/";
+ //---------------------------- Properties ---=------------------------//
+ public static final String PROPERTY_CLUSTER_ADDRESS = "cluster.address";
+ public static final String PROPERTY_INSTANCE_NAME = "instance";
+ public static final String DEFAULT_INSTANCE_NAME = "DEFAULT_INSTANCE";
+ public static final String PROPERTY_METADATA_PORT = "metadata.port";
+ public static final String PROPERTY_COREDUMP_DIR = "coredumpdir";
+ public static final String DEFAULT_COREDUMP_DIR = String.join(File.separator, ASTERIXDB, "coredump");
+ public static final String PROPERTY_TXN_LOG_DIR = "txnlogdir";
+ public static final String DEFAULT_TXN_LOG_DIR = String.join(File.separator, ASTERIXDB, "txn-log");
+ public static final String PROPERTY_IO_DEV = "iodevices";
+ public static final String DEFAULT_IO_DEV = String.join(File.separator, ASTERIXDB, "iodevice");
+ public static final String PROPERTY_STORAGE_DIR = "storagedir";
+ public static final String DEFAULT_STORAGE_DIR = "storage";
+ public static final String PROPERTY_CLASS = "class";
+
+ private AsterixProperties() {
+ }
+
+ public static final String getSectionId(String prefix, String section) {
+ return section.substring(prefix.length());
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java
index 7309f0c..ea1ee31 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/AsterixPropertiesAccessor.java
@@ -28,10 +28,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
@@ -40,14 +39,20 @@ import javax.xml.bind.Unmarshaller;
import org.apache.asterix.common.cluster.ClusterPartition;
import org.apache.asterix.common.configuration.AsterixConfiguration;
import org.apache.asterix.common.configuration.Coredump;
+import org.apache.asterix.common.configuration.Extension;
import org.apache.asterix.common.configuration.Property;
import org.apache.asterix.common.configuration.Store;
import org.apache.asterix.common.configuration.TransactionLogDir;
import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.utils.ConfigUtil;
+import org.apache.commons.lang3.mutable.MutableInt;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.application.IApplicationConfig;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
public class AsterixPropertiesAccessor {
- private static Logger LOGGER = Logger.getLogger(AsterixPropertiesAccessor.class.getName());
+ private static final Logger LOGGER = Logger.getLogger(AsterixPropertiesAccessor.class.getName());
private final String instanceName;
private final String metadataNodeName;
@@ -62,36 +67,22 @@ public class AsterixPropertiesAccessor {
private final Map<String, String> asterixBuildProperties = new HashMap<>();
private final Map<String, ClusterPartition[]> nodePartitionsMap;
private final SortedMap<Integer, ClusterPartition> clusterPartitions = new TreeMap<>();
+ // For extensions
+ private final List<AsterixExtension> extensions;
/**
* Constructor which reads asterix-configuration.xml, the old way.
+ *
* @throws AsterixException
+ * @throws IOException
*/
- public AsterixPropertiesAccessor() throws AsterixException {
+ public AsterixPropertiesAccessor() throws AsterixException, IOException {
String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY);
if (fileName == null) {
fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
}
-
- InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName);
- if (is == null) {
- try {
- fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
- is = new FileInputStream(fileName);
- } catch (FileNotFoundException fnf) {
- throw new AsterixException("Could not find configuration file " + fileName);
- }
- }
-
- AsterixConfiguration asterixConfiguration = null;
+ AsterixConfiguration asterixConfiguration = configure(fileName);
cfg = null;
- try {
- JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
- Unmarshaller unmarshaller = ctx.createUnmarshaller();
- asterixConfiguration = (AsterixConfiguration) unmarshaller.unmarshal(is);
- } catch (JAXBException e) {
- throw new AsterixException("Failed to read configuration file " + fileName);
- }
instanceName = asterixConfiguration.getInstanceName();
metadataNodeName = asterixConfiguration.getMetadataNode();
List<Store> configuredStores = asterixConfiguration.getStore();
@@ -117,7 +108,16 @@ public class AsterixPropertiesAccessor {
nodePartitionsMap.put(store.getNcId(), nodePartitions);
nodeNames.add(store.getNcId());
}
- asterixConfigurationParams = new HashMap<String, Property>();
+
+ // Get extensions
+ extensions = new ArrayList<>();
+ if (asterixConfiguration.getExtensions() != null) {
+ for (Extension ext : asterixConfiguration.getExtensions().getExtension()) {
+ extensions.add(ConfigUtil.toAsterixExtension(ext));
+ }
+ }
+
+ asterixConfigurationParams = new HashMap<>();
for (Property p : asterixConfiguration.getProperty()) {
asterixConfigurationParams.put(p.getName(), p);
}
@@ -130,56 +130,56 @@ public class AsterixPropertiesAccessor {
loadAsterixBuildProperties();
}
+ private AsterixConfiguration configure(String fileName) throws IOException, AsterixException {
+ try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName)) {
+ if (is != null) {
+ return configure(is, fileName);
+ }
+ }
+ try (FileInputStream is = new FileInputStream(fileName)) {
+ return configure(is, fileName);
+ } catch (FileNotFoundException fnf1) {
+ LOGGER.warn("Failed to get configuration file " + fileName + " as FileInputStream. FileNotFoundException");
+ LOGGER.warn("Attempting to get default configuration file " + GlobalConfig.DEFAULT_CONFIG_FILE_NAME
+ + " as FileInputStream");
+ try (FileInputStream fis = new FileInputStream(GlobalConfig.DEFAULT_CONFIG_FILE_NAME)) {
+ return configure(fis, GlobalConfig.DEFAULT_CONFIG_FILE_NAME);
+ } catch (FileNotFoundException fnf2) {
+ fnf1.addSuppressed(fnf2);
+ throw new AsterixException("Could not find configuration file " + fileName, fnf1);
+ }
+ }
+ }
+
+ private AsterixConfiguration configure(InputStream is, String fileName) throws AsterixException {
+ try {
+ JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
+ Unmarshaller unmarshaller = ctx.createUnmarshaller();
+ return (AsterixConfiguration) unmarshaller.unmarshal(is);
+ } catch (JAXBException e) {
+ throw new AsterixException("Failed to read configuration file " + fileName, e);
+ }
+ }
+
/**
* Constructor which wraps an IApplicationConfig.
*/
public AsterixPropertiesAccessor(IApplicationConfig cfg) throws AsterixException {
this.cfg = cfg;
- instanceName = cfg.getString("asterix", "instance", "DEFAULT_INSTANCE");
+ instanceName = cfg.getString(AsterixProperties.SECTION_ASTERIX, AsterixProperties.PROPERTY_INSTANCE_NAME,
+ AsterixProperties.DEFAULT_INSTANCE_NAME);
String mdNode = null;
nodePartitionsMap = new HashMap<>();
- int uniquePartitionId = 0;
-
+ MutableInt uniquePartitionId = new MutableInt(0);
+ extensions = new ArrayList<>();
// Iterate through each configured NC.
for (String section : cfg.getSections()) {
- if (!section.startsWith("nc/")) {
- continue;
- }
- String ncId = section.substring(3);
-
- // Here we figure out which is the metadata node. If any NCs
- // declare "metadata.port", use that one; otherwise just use the first.
- if (mdNode == null) {
- mdNode = ncId;
- }
- if (cfg.getString(section, "metadata.port") != null) {
- // QQQ But we don't actually *honor* metadata.port yet!
- mdNode = ncId;
- }
-
- // Now we assign the coredump and txnlog directories for this node.
- // QQQ Default values? Should they be specified here? Or should there
- // be a default.ini? Certainly wherever they are, they should be platform-dependent.
- coredumpConfig.put(ncId, cfg.getString(section, "coredumpdir", "/var/lib/asterixdb/coredump"));
- transactionLogDirs.put(ncId, cfg.getString(section, "txnlogdir", "/var/lib/asterixdb/txn-log"));
-
- // Now we create an array of ClusterPartitions for all the partitions
- // on this NC.
- String[] iodevices = cfg.getString(section, "iodevices", "/var/lib/asterixdb/iodevice").split(",");
- String storageSubdir = cfg.getString(section, "storagedir", "storage");
- String[] nodeStores = new String[iodevices.length];
- ClusterPartition[] nodePartitions = new ClusterPartition[iodevices.length];
- for (int i = 0; i < nodePartitions.length; i++) {
- // Construct final storage path from iodevice dir + storage subdir.
- nodeStores[i] = iodevices[i] + File.separator + storageSubdir;
- // Create ClusterPartition instances for this NC.
- ClusterPartition partition = new ClusterPartition(uniquePartitionId++, ncId, i);
- clusterPartitions.put(partition.getPartitionId(), partition);
- nodePartitions[i] = partition;
+ if (section.startsWith(AsterixProperties.SECTION_PREFIX_NC)) {
+ mdNode = configureNc(section, mdNode, uniquePartitionId);
+ } else if (section.startsWith(AsterixProperties.SECTION_PREFIX_EXTENSION)) {
+ String className = AsterixProperties.getSectionId(AsterixProperties.SECTION_PREFIX_EXTENSION, section);
+ configureExtension(className, section);
}
- stores.put(ncId, nodeStores);
- nodePartitionsMap.put(ncId, nodePartitions);
- nodeNames.add(ncId);
}
metadataNodeName = mdNode;
@@ -187,6 +187,58 @@ public class AsterixPropertiesAccessor {
loadAsterixBuildProperties();
}
+ private void configureExtension(String className, String section) {
+ Set<String> keys = cfg.getKeys(section);
+ List<Pair<String, String>> kvs = new ArrayList<>();
+ for (String key : keys) {
+ String value = cfg.getString(section, key);
+ kvs.add(new Pair<>(key, value));
+ }
+ extensions.add(new AsterixExtension(className, kvs));
+ }
+
+ private String configureNc(String section, String mdNode, MutableInt uniquePartitionId) {
+ String ncId = AsterixProperties.getSectionId(AsterixProperties.SECTION_PREFIX_NC, section);
+ String newMetadataNode = mdNode;
+
+ // Here we figure out which is the metadata node. If any NCs
+ // declare "metadata.port", use that one; otherwise just use the first.
+ if (mdNode == null || cfg.getString(section, AsterixProperties.PROPERTY_METADATA_PORT) != null) {
+ // QQQ But we don't actually *honor* metadata.port yet!
+ newMetadataNode = ncId;
+ }
+
+ // Now we assign the coredump and txnlog directories for this node.
+ // QQQ Default values? Should they be specified here? Or should there
+ // be a default.ini? Certainly wherever they are, they should be platform-dependent.
+ coredumpConfig.put(ncId, cfg.getString(section, AsterixProperties.PROPERTY_COREDUMP_DIR,
+ AsterixProperties.DEFAULT_COREDUMP_DIR));
+ transactionLogDirs.put(ncId,
+ cfg.getString(section, AsterixProperties.PROPERTY_TXN_LOG_DIR, AsterixProperties.DEFAULT_TXN_LOG_DIR));
+
+ // Now we create an array of ClusterPartitions for all the partitions
+ // on this NC.
+ String[] iodevices = cfg.getString(section, AsterixProperties.PROPERTY_IO_DEV,
+ AsterixProperties.DEFAULT_IO_DEV).split(",");
+ String storageSubdir = cfg.getString(section, AsterixProperties.PROPERTY_STORAGE_DIR,
+ AsterixProperties.DEFAULT_STORAGE_DIR);
+ String[] nodeStores = new String[iodevices.length];
+ ClusterPartition[] nodePartitions = new ClusterPartition[iodevices.length];
+ for (int i = 0; i < nodePartitions.length; i++) {
+ // Construct final storage path from iodevice dir + storage subdir.
+ nodeStores[i] = iodevices[i] + File.separator + storageSubdir;
+ // Create ClusterPartition instances for this NC.
+ ClusterPartition partition = new ClusterPartition(uniquePartitionId.getValue(), ncId, i);
+ uniquePartitionId.increment();
+ clusterPartitions.put(partition.getPartitionId(), partition);
+ nodePartitions[i] = partition;
+ }
+ stores.put(ncId, nodeStores);
+ nodePartitionsMap.put(ncId, nodePartitions);
+ nodeNames.add(ncId);
+ return newMetadataNode;
+ }
+
private void loadAsterixBuildProperties() throws AsterixException {
Properties gitProperties = new Properties();
try {
@@ -242,14 +294,14 @@ public class AsterixPropertiesAccessor {
try {
return interpreter.interpret(value);
} catch (IllegalArgumentException e) {
- if (LOGGER.isLoggable(Level.SEVERE)) {
- StringBuilder msg =
- new StringBuilder("Invalid property value '" + value + "' for property '" + property + "'.\n");
+ if (LOGGER.isEnabledFor(Level.ERROR)) {
+ StringBuilder msg = new StringBuilder(
+ "Invalid property value '" + value + "' for property '" + property + "'.\n");
if (p != null) {
msg.append("See the description: \n" + p.getDescription() + "\n");
}
msg.append("Default = " + defaultValue);
- LOGGER.severe(msg.toString());
+ LOGGER.error(msg.toString());
}
throw e;
}
@@ -271,4 +323,8 @@ public class AsterixPropertiesAccessor {
public SortedMap<Integer, ClusterPartition> getClusterPartitions() {
return clusterPartitions;
}
+
+ public List<AsterixExtension> getExtensions() {
+ return extensions;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataConstants.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataConstants.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataConstants.java
deleted file mode 100644
index 943e385..0000000
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataConstants.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.common.config;
-
-public class MetadataConstants {
-
- // Name of the dataverse the metadata lives in.
- public final static String METADATA_DATAVERSE_NAME = "Metadata";
-
- // Name of the node group where metadata is stored on.
- public final static String METADATA_NODEGROUP_NAME = "MetadataGroup";
-
- // Name of the default nodegroup where internal/feed datasets will be partitioned
- // if an explicit nodegroup is not specified at the time of creation of a dataset
- public static final String METADATA_DEFAULT_NODEGROUP_NAME = "DEFAULT_NG_ALL_NODES";
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 66a01e0..a56d6f6 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -21,6 +21,8 @@ package org.apache.asterix.common.exceptions;
public class ErrorCode {
public static final String ASTERIX = "ASX";
public static final int ERROR_CASTING_FIELD = 0;
+ public static final int ERROR_EXTENSION_CONFLICT = 1;
+ public static final int ERROR_PARSE_ERROR = 2;
private ErrorCode() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/IApplicationMessage.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/IApplicationMessage.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/IApplicationMessage.java
index 7940680..9742a6c 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/IApplicationMessage.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/IApplicationMessage.java
@@ -36,7 +36,8 @@ public interface IApplicationMessage extends IMessage {
COMPLETE_FAILBACK_REQUEST,
COMPLETE_FAILBACK_RESPONSE,
REPLICA_EVENT,
- ACTIVE_ENTITY_MESSAGE
+ ACTIVE_ENTITY_TO_CC_MESSAGE,
+ ACTIVE_MANAGER_MESSAGE
}
public abstract ApplicationMessageType getMessageType();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ConfigUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ConfigUtil.java
new file mode 100644
index 0000000..5db1693
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ConfigUtil.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.util.ArrayList;
+
+import org.apache.asterix.common.config.AsterixExtension;
+import org.apache.asterix.common.configuration.Extension;
+import org.apache.asterix.common.configuration.Property;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class ConfigUtil {
+
+ private ConfigUtil() {
+ }
+
+ public static AsterixExtension toAsterixExtension(Extension ext) {
+ String className = ext.getExtensionClassName();
+ ArrayList<Pair<String, String>> args = new ArrayList<>();
+ for (Property property : ext.getProperty()) {
+ args.add(new Pair<>(property.getName(), property.getValue()));
+ }
+ return new AsterixExtension(className, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd b/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd
index bb99319..d6a35cd 100644
--- a/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd
+++ b/asterixdb/asterix-common/src/main/resources/schema/asterix-conf.xsd
@@ -56,6 +56,9 @@
<xs:element
name="txnLogDirPath"
type="xs:string" />
+ <xs:element
+ name="extensionClassName"
+ type="xs:string" />
<!-- definition of complex elements -->
<xs:element name="store">
@@ -95,6 +98,29 @@
</xs:complexType>
</xs:element>
+ <xs:element name="extension">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="mg:extensionClassName"
+ maxOccurs="1"
+ minOccurs="1"/>
+ <xs:element ref="mg:property"
+ minOccurs="0"
+ maxOccurs="unbounded" />
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+
+ <xs:element name="extensions">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element ref="mg:extension"
+ minOccurs="0"
+ maxOccurs="unbounded">
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
<xs:element name="asterixConfiguration">
<xs:complexType>
@@ -117,6 +143,9 @@
<xs:element
ref="mg:transactionLogDir"
maxOccurs="unbounded" />
+ <xs:element ref="mg:extensions"
+ minOccurs="0"
+ maxOccurs="1" />
<xs:element
ref="mg:property"
minOccurs="0"
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriberRuntime.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriberRuntime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriberRuntime.java
deleted file mode 100644
index 1752054..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/api/ISubscriberRuntime.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.feed.api;
-
-import java.util.Map;
-
-public interface ISubscriberRuntime {
-
- public Map<String, String> getFeedPolicy();
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
index 6159908..d4e3641 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/dataflow/FeedRuntimeInputHandler.java
@@ -76,7 +76,7 @@ public class FeedRuntimeInputHandler extends AbstractUnaryInputUnaryOutputOperat
this.spiller =
fpa.spillToDiskOnCongestion() ? new FrameSpiller(ctx,
connectionId.getFeedId() + "_" + connectionId.getDatasetName() + "_"
- + runtimeId.getFeedRuntimeType() + "_" + runtimeId.getPartition(),
+ + runtimeId.getRuntimeName() + "_" + runtimeId.getPartition(),
fpa.getMaxSpillOnDisk()) : null;
this.exceptionHandler = new FeedExceptionHandler(ctx, fta);
this.fpa = fpa;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
index 40d2500..c40fed6 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
@@ -158,7 +158,7 @@ public class FeedEventsListener implements IActiveEntityEventsListener {
}
private void handleIntakePartitionStarts(ActiveEvent message, ActiveJob jobInfo) {
- if (feedPipeline.get(message.getFeedId()).first.decrementAndGet() == 0) {
+ if (feedPipeline.get(message.getEntityId()).first.decrementAndGet() == 0) {
((FeedIntakeInfo) jobInfo).getIntakeFeedJoint().setState(State.ACTIVE);
jobInfo.setState(ActivityState.ACTIVE);
notifyFeedEventSubscribers(FeedLifecycleEvent.FEED_INTAKE_STARTED);
@@ -339,7 +339,7 @@ public class FeedEventsListener implements IActiveEntityEventsListener {
return locations;
}
- private void notifyFeedEventSubscribers(FeedLifecycleEvent event) {
+ private synchronized void notifyFeedEventSubscribers(FeedLifecycleEvent event) {
if (subscribers != null && !subscribers.isEmpty()) {
for (IFeedLifecycleEventSubscriber subscriber : subscribers) {
subscriber.handleFeedEvent(event);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java
index 9c7a319..6c04b2d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterExecutor.java
@@ -19,8 +19,8 @@
package org.apache.asterix.external.feed.runtime;
import org.apache.asterix.external.dataset.adapter.FeedAdapter;
-import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
import org.apache.asterix.external.util.ExternalDataExceptionUtils;
+import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.log4j.Logger;
/**
@@ -30,13 +30,12 @@ public class AdapterExecutor implements Runnable {
private static final Logger LOGGER = Logger.getLogger(AdapterExecutor.class.getName());
- private final DistributeFeedFrameWriter writer; // A writer that sends frames to multiple receivers (that can
+ private final IFrameWriter writer; // A writer that sends frames to multiple receivers (that can
// increase or decrease at any time)
- private final FeedAdapter adapter; // The adapter
+ private final FeedAdapter adapter; // The adapter
private final AdapterRuntimeManager adapterManager;// The runtime manager <-- two way visibility -->
- public AdapterExecutor(int partition, DistributeFeedFrameWriter writer, FeedAdapter adapter,
- AdapterRuntimeManager adapterManager) {
+ public AdapterExecutor(IFrameWriter writer, FeedAdapter adapter, AdapterRuntimeManager adapterManager) {
this.writer = writer;
this.adapter = adapter;
this.adapterManager = adapterManager;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java
index f2f0747..424f2dc 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/AdapterRuntimeManager.java
@@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.asterix.active.EntityId;
import org.apache.asterix.external.dataset.adapter.FeedAdapter;
-import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
+import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.log4j.Logger;
/**
@@ -34,27 +34,26 @@ public class AdapterRuntimeManager {
private static final Logger LOGGER = Logger.getLogger(AdapterRuntimeManager.class.getName());
- private final EntityId feedId; // (dataverse-feed)
+ private final EntityId feedId; // (dataverse-feed)
- private final FeedAdapter feedAdapter; // The adapter
+ private final FeedAdapter feedAdapter; // The adapter
- private final AdapterExecutor adapterExecutor; // The executor for the adapter
+ private final AdapterExecutor adapterExecutor; // The executor for the adapter
- private final int partition; // The partition number
+ private final int partition; // The partition number
- private final ExecutorService executorService; // Executor service to run/shutdown the adapter executor
+ private final ExecutorService executorService; // Executor service to run/shutdown the adapter executor
- private IngestionRuntime ingestionRuntime; // Runtime representing the ingestion stage of a feed
+ private IngestionRuntime ingestionRuntime; // Runtime representing the ingestion stage of a feed
private volatile boolean done = false;
private volatile boolean failed = false;
- public AdapterRuntimeManager(EntityId feedId, FeedAdapter feedAdapter, DistributeFeedFrameWriter writer,
- int partition) {
- this.feedId = feedId;
+ public AdapterRuntimeManager(EntityId entityId, FeedAdapter feedAdapter, IFrameWriter writer, int partition) {
+ this.feedId = entityId;
this.feedAdapter = feedAdapter;
this.partition = partition;
- this.adapterExecutor = new AdapterExecutor(partition, writer, feedAdapter, this);
+ this.adapterExecutor = new AdapterExecutor(writer, feedAdapter, this);
this.executorService = Executors.newSingleThreadExecutor();
}
@@ -82,7 +81,6 @@ public class AdapterRuntimeManager {
// stop() returned false, we try to force shutdown
executorService.shutdownNow();
}
-
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java
index 294642e..821a0b1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/CollectionRuntime.java
@@ -20,13 +20,13 @@ package org.apache.asterix.external.feed.runtime;
import java.util.Map;
-import org.apache.asterix.active.ActiveRuntime;
import org.apache.asterix.active.ActiveRuntimeId;
+import org.apache.asterix.active.IActiveRuntime;
import org.apache.asterix.external.feed.api.ISubscribableRuntime;
-import org.apache.asterix.external.feed.api.ISubscriberRuntime;
import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
/**
* Represents the feed runtime that collects feed tuples from another feed.
@@ -34,18 +34,19 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
* intake job. For a secondary feed, tuples are collected from the intake/compute
* runtime associated with the source feed.
*/
-public class CollectionRuntime extends ActiveRuntime implements ISubscriberRuntime {
+public class CollectionRuntime implements IActiveRuntime {
- private final FeedConnectionId connectionId; // [Dataverse - Feed - Dataset]
- private final ISubscribableRuntime sourceRuntime; // Runtime that provides the data
- private final Map<String, String> feedPolicy; // Policy associated with the feed
- private final FeedFrameCollector frameCollector; // Collector that can be plugged into a frame distributor
+ private final FeedConnectionId connectionId; // [Dataverse - Feed - Dataset]
+ private final ISubscribableRuntime sourceRuntime; // Runtime that provides the data
+ private final Map<String, String> feedPolicy; // Policy associated with the feed
+ private final FeedFrameCollector frameCollector; // Collector that can be plugged into a frame distributor
private final IHyracksTaskContext ctx;
+ private final ActiveRuntimeId runtimeId;
public CollectionRuntime(FeedConnectionId connectionId, ActiveRuntimeId runtimeId,
ISubscribableRuntime sourceRuntime, Map<String, String> feedPolicy, IHyracksTaskContext ctx,
FeedFrameCollector frameCollector) {
- super(runtimeId);
+ this.runtimeId = runtimeId;
this.connectionId = connectionId;
this.sourceRuntime = sourceRuntime;
this.feedPolicy = feedPolicy;
@@ -68,7 +69,6 @@ public class CollectionRuntime extends ActiveRuntime implements ISubscriberRunti
|| frameCollector.getState().equals(FeedFrameCollector.State.HANDOVER);
}
- @Override
public Map<String, String> getFeedPolicy() {
return feedPolicy;
}
@@ -88,4 +88,13 @@ public class CollectionRuntime extends ActiveRuntime implements ISubscriberRunti
public IHyracksTaskContext getCtx() {
return ctx;
}
+
+ @Override
+ public ActiveRuntimeId getRuntimeId() {
+ return runtimeId;
+ }
+
+ @Override
+ public void stop() throws HyracksDataException, InterruptedException {
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
index 9661890..34237c4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
@@ -22,7 +22,7 @@ import java.util.logging.Level;
import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.active.EntityId;
-import org.apache.asterix.external.feed.api.ISubscriberRuntime;
+import org.apache.asterix.active.IActiveRuntime;
import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
import org.apache.hyracks.api.comm.VSizeFrame;
@@ -37,9 +37,9 @@ public class IngestionRuntime extends SubscribableRuntime {
private final IHyracksTaskContext ctx;
private int numSubscribers = 0;
- public IngestionRuntime(EntityId feedId, ActiveRuntimeId runtimeId, DistributeFeedFrameWriter feedWriter,
+ public IngestionRuntime(EntityId entityId, ActiveRuntimeId runtimeId, DistributeFeedFrameWriter feedWriter,
AdapterRuntimeManager adaptorRuntimeManager, IHyracksTaskContext ctx) {
- super(feedId, runtimeId, feedWriter);
+ super(entityId, runtimeId, feedWriter);
this.adapterRuntimeManager = adaptorRuntimeManager;
this.ctx = ctx;
}
@@ -53,7 +53,7 @@ public class IngestionRuntime extends SubscribableRuntime {
TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx);
TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE,
TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx), collectionRuntime.getCtx());
- adapterRuntimeManager.start();
+ start();
}
numSubscribers++;
if (LOGGER.isLoggable(Level.INFO)) {
@@ -65,7 +65,7 @@ public class IngestionRuntime extends SubscribableRuntime {
public synchronized void unsubscribe(CollectionRuntime collectionRuntime) throws InterruptedException {
numSubscribers--;
if (numSubscribers == 0) {
- adapterRuntimeManager.stop();
+ stop();
}
subscribers.remove(collectionRuntime);
}
@@ -75,7 +75,7 @@ public class IngestionRuntime extends SubscribableRuntime {
}
public void terminate() {
- for (ISubscriberRuntime subscriber : subscribers) {
+ for (IActiveRuntime subscriber : subscribers) {
try {
unsubscribe((CollectionRuntime) subscriber);
} catch (Exception e) {
@@ -86,4 +86,12 @@ public class IngestionRuntime extends SubscribableRuntime {
}
}
+ public void start() {
+ adapterRuntimeManager.start();
+ }
+
+ @Override
+ public void stop() throws InterruptedException {
+ adapterRuntimeManager.stop();
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java
index 423e599..fb70fdb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/SubscribableRuntime.java
@@ -22,25 +22,30 @@ import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
-import org.apache.asterix.active.ActiveRuntime;
import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.active.EntityId;
+import org.apache.asterix.active.IActiveRuntime;
import org.apache.asterix.external.feed.api.ISubscribableRuntime;
-import org.apache.asterix.external.feed.api.ISubscriberRuntime;
import org.apache.asterix.external.feed.dataflow.DistributeFeedFrameWriter;
-public abstract class SubscribableRuntime extends ActiveRuntime implements ISubscribableRuntime {
+public abstract class SubscribableRuntime implements ISubscribableRuntime {
protected static final Logger LOGGER = Logger.getLogger(SubscribableRuntime.class.getName());
protected final EntityId feedId;
- protected final List<ISubscriberRuntime> subscribers;
+ protected final List<IActiveRuntime> subscribers;
protected final DistributeFeedFrameWriter dWriter;
+ protected final ActiveRuntimeId runtimeId;
public SubscribableRuntime(EntityId feedId, ActiveRuntimeId runtimeId, DistributeFeedFrameWriter dWriter) {
- super(runtimeId);
+ this.runtimeId = runtimeId;
this.feedId = feedId;
this.dWriter = dWriter;
- this.subscribers = new ArrayList<ISubscriberRuntime>();
+ this.subscribers = new ArrayList<>();
+ }
+
+ @Override
+ public ActiveRuntimeId getRuntimeId() {
+ return runtimeId;
}
public EntityId getFeedId() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
index 533d119..7acb1f8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
@@ -23,6 +23,7 @@ import java.io.Serializable;
import java.util.Date;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import org.apache.commons.lang3.ObjectUtils;
public class ExternalFile implements Serializable, Comparable<ExternalFile> {
@@ -118,21 +119,21 @@ public class ExternalFile implements Serializable, Comparable<ExternalFile> {
}
@Override
+ public int hashCode() {
+ return ObjectUtils.hashCodeMulti(dataverseName, datasetName, fileName);
+ }
+
+ @Override
public boolean equals(Object obj) {
- if (obj == null)
- return false;
- if (obj == this)
+ if (obj == this) {
return true;
- if (!(obj instanceof ExternalFile))
+ }
+ if (!(obj instanceof ExternalFile)) {
return false;
+ }
ExternalFile anotherFile = (ExternalFile) obj;
- if (fileNumber != anotherFile.fileNumber)
- return false;
- if (!dataverseName.equals(anotherFile.dataverseName))
- return false;
- if (!datasetName.equals(anotherFile.datasetName))
- return false;
- return true;
+ return fileNumber == anotherFile.fileNumber && ObjectUtils.equals(dataverseName, anotherFile.getDataverseName())
+ && ObjectUtils.equals(datasetName, anotherFile.getDatasetName());
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
index 84c2cb4..266669b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorDescriptor.java
@@ -78,9 +78,9 @@ public class FeedCollectOperatorDescriptor extends AbstractSingleActivityOperato
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
throws HyracksDataException {
ActiveManager feedManager = (ActiveManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject()).getFeedManager();
+ .getApplicationContext().getApplicationObject()).getActiveManager();
ActiveRuntimeId sourceRuntimeId = new ActiveRuntimeId(sourceFeedId, subscriptionLocation.toString(), partition);
- ISubscribableRuntime sourceRuntime = (ISubscribableRuntime) feedManager.getSubscribableRuntime(sourceRuntimeId);
+ ISubscribableRuntime sourceRuntime = (ISubscribableRuntime) feedManager.getRuntime(sourceRuntimeId);
return new FeedCollectOperatorNodePushable(ctx, connectionId, feedPolicyProperties, partition, sourceRuntime);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
index 231fe99..ee9a186 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedCollectOperatorNodePushable.java
@@ -21,8 +21,8 @@ package org.apache.asterix.external.operators;
import java.util.Map;
import org.apache.asterix.active.ActiveManager;
-import org.apache.asterix.active.ActivePartitionMessage;
import org.apache.asterix.active.ActiveRuntimeId;
+import org.apache.asterix.active.message.ActivePartitionMessage;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.external.feed.api.ISubscribableRuntime;
import org.apache.asterix.external.feed.dataflow.FeedFrameCollector;
@@ -46,7 +46,7 @@ public class FeedCollectOperatorNodePushable extends AbstractUnaryOutputSourceOp
private final FeedConnectionId connectionId;
private final Map<String, String> feedPolicy;
private final FeedPolicyAccessor policyAccessor;
- private final ActiveManager feedManager;
+ private final ActiveManager activeManager;
private final ISubscribableRuntime sourceRuntime;
private final IHyracksTaskContext ctx;
private CollectionRuntime collectRuntime;
@@ -59,8 +59,8 @@ public class FeedCollectOperatorNodePushable extends AbstractUnaryOutputSourceOp
this.sourceRuntime = sourceRuntime;
this.feedPolicy = feedPolicy;
this.policyAccessor = new FeedPolicyAccessor(feedPolicy);
- this.feedManager = (ActiveManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
- .getApplicationObject()).getFeedManager();
+ this.activeManager = (ActiveManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
+ .getApplicationObject()).getActiveManager();
}
@Override
@@ -72,20 +72,19 @@ public class FeedCollectOperatorNodePushable extends AbstractUnaryOutputSourceOp
FrameTupleAccessor tAccessor = new FrameTupleAccessor(recordDesc);
if (policyAccessor.bufferingEnabled()) {
writer = new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, writer, policyAccessor, tAccessor,
- feedManager.getFramePool());
+ activeManager.getFramePool());
} else {
writer = new SyncFeedRuntimeInputHandler(ctx, writer, tAccessor);
}
collectRuntime = new CollectionRuntime(connectionId, runtimeId, sourceRuntime, feedPolicy, ctx,
new FeedFrameCollector(policyAccessor, writer, connectionId));
- feedManager.getActiveRuntimeRegistry().registerRuntime(collectRuntime);
+ activeManager.registerRuntime(collectRuntime);
sourceRuntime.subscribe(collectRuntime);
// Notify CC that Collection started
- ctx.sendApplicationMessageToCC(
- new ActivePartitionMessage(connectionId.getFeedId(), ctx.getJobletContext().getJobId(), null),
- null);
+ ctx.sendApplicationMessageToCC(new ActivePartitionMessage(runtimeId, ctx.getJobletContext().getJobId(),
+ ActivePartitionMessage.ACTIVE_RUNTIME_REGISTERED), null);
collectRuntime.waitTillCollectionOver();
- feedManager.getActiveRuntimeRegistry().deregisterRuntime(collectRuntime.getRuntimeId());
+ activeManager.deregisterRuntime(collectRuntime.getRuntimeId());
} catch (Exception e) {
throw new HyracksDataException(e);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
index ffa451b..7c8fe14 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedIntakeOperatorNodePushable.java
@@ -19,9 +19,9 @@
package org.apache.asterix.external.operators;
import org.apache.asterix.active.ActiveManager;
-import org.apache.asterix.active.ActivePartitionMessage;
import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.active.EntityId;
+import org.apache.asterix.active.message.ActivePartitionMessage;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.dataset.adapter.FeedAdapter;
@@ -62,7 +62,7 @@ public class FeedIntakeOperatorNodePushable extends AbstractUnaryOutputSourceOpe
@Override
public void initialize() throws HyracksDataException {
ActiveManager feedManager = (ActiveManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext()
- .getApplicationContext().getApplicationObject()).getFeedManager();
+ .getApplicationContext().getApplicationObject()).getActiveManager();
AdapterRuntimeManager adapterRuntimeManager = null;
DistributeFeedFrameWriter frameDistributor = null;
IngestionRuntime ingestionRuntime = null;
@@ -80,8 +80,8 @@ public class FeedIntakeOperatorNodePushable extends AbstractUnaryOutputSourceOpe
ingestionRuntime = new IngestionRuntime(feedId, runtimeId, frameDistributor, adapterRuntimeManager, ctx);
feedManager.registerRuntime(ingestionRuntime);
// Notify FeedJobNotificationHandler that this provider is ready to receive subscription requests.
- ctx.sendApplicationMessageToCC(new ActivePartitionMessage(feedId, ctx.getJobletContext().getJobId(), null),
- null);
+ ctx.sendApplicationMessageToCC(new ActivePartitionMessage(runtimeId, ctx.getJobletContext().getJobId(),
+ ActivePartitionMessage.ACTIVE_RUNTIME_REGISTERED), null);
// open the distributor
open = true;
frameDistributor.open();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java
index 5f92327..1dce6be 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMessageOperatorNodePushable.java
@@ -63,7 +63,7 @@ public class FeedMessageOperatorNodePushable extends AbstractUnaryOutputSourceOp
this.partition = partition;
IAsterixAppRuntimeContext runtimeCtx =
(IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext().getApplicationObject();
- this.feedManager = (ActiveManager) runtimeCtx.getFeedManager();
+ this.feedManager = (ActiveManager) runtimeCtx.getActiveManager();
}
@Override
@@ -98,8 +98,7 @@ public class FeedMessageOperatorNodePushable extends AbstractUnaryOutputSourceOp
EntityId sourceFeedId = endFeedMessage.getSourceFeedId();
ActiveRuntimeId subscribableRuntimeId =
new ActiveRuntimeId(sourceFeedId, FeedRuntimeType.INTAKE.toString(), partition);
- ISubscribableRuntime feedRuntime =
- (ISubscribableRuntime) feedManager.getSubscribableRuntime(subscribableRuntimeId);
+ ISubscribableRuntime feedRuntime = (ISubscribableRuntime) feedManager.getRuntime(subscribableRuntimeId);
AdapterRuntimeManager adapterRuntimeManager = ((IngestionRuntime) feedRuntime).getAdapterRuntimeManager();
adapterRuntimeManager.stop();
if (LOGGER.isLoggable(Level.INFO)) {
@@ -129,8 +128,7 @@ public class FeedMessageOperatorNodePushable extends AbstractUnaryOutputSourceOp
}
runtimeId = new ActiveRuntimeId(endFeedMessage.getSourceFeedId(), runtimeType.toString(), partition);
- CollectionRuntime feedRuntime =
- (CollectionRuntime) feedManager.getActiveRuntimeRegistry().getRuntime(runtimeId);
+ CollectionRuntime feedRuntime = (CollectionRuntime) feedManager.getRuntime(runtimeId);
if (feedRuntime != null) {
feedRuntime.getSourceRuntime().unsubscribe(feedRuntime);
}
@@ -150,11 +148,10 @@ public class FeedMessageOperatorNodePushable extends AbstractUnaryOutputSourceOp
ActiveRuntimeId feedSubscribableRuntimeId = new ActiveRuntimeId(connectionId.getFeedId(),
FeedRuntimeType.COMPUTE.toString(), partition);
ISubscribableRuntime feedRuntime =
- (ISubscribableRuntime) feedManager.getSubscribableRuntime(feedSubscribableRuntimeId);
+ (ISubscribableRuntime) feedManager.getRuntime(feedSubscribableRuntimeId);
runtimeId = new ActiveRuntimeId(endFeedMessage.getSourceFeedId(),
FeedRuntimeType.COMPUTE_COLLECT.toString(), partition);
- CollectionRuntime feedCollectionRuntime =
- (CollectionRuntime) feedManager.getActiveRuntimeRegistry().getRuntime(runtimeId);
+ CollectionRuntime feedCollectionRuntime = (CollectionRuntime) feedManager.getRuntime(runtimeId);
feedRuntime.unsubscribe(feedCollectionRuntime);
break;
default:
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
index 37a42a7..ff996aa 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaComputeNodePushable.java
@@ -24,7 +24,6 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.active.ActiveManager;
-import org.apache.asterix.active.ActiveRuntime;
import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler;
@@ -62,12 +61,6 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
private FeedPolicyEnforcer policyEnforcer;
/**
- * The Feed Runtime instance associated with the operator. Feed Runtime
- * captures the state of the operator while the feed is active.
- */
- private ActiveRuntime feedRuntime;
-
- /**
* A unique identifier for the feed instance. A feed instance represents
* the flow of data from a feed to a dataset.
**/
@@ -112,7 +105,7 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
this.partition = partition;
this.connectionId = feedConnectionId;
this.feedManager = (ActiveManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
- .getApplicationObject()).getFeedManager();
+ .getApplicationObject()).getActiveManager();
this.message = new VSizeFrame(ctx);
TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
this.opDesc = feedMetaOperatorDescriptor;
@@ -142,8 +135,6 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
} else {
writer = new SyncFeedRuntimeInputHandler(ctx, coreOperator, fta);
}
- feedRuntime = new ActiveRuntime(runtimeId);
- feedManager.getActiveRuntimeRegistry().registerRuntime(feedRuntime);
}
@Override
@@ -164,19 +155,11 @@ public class FeedMetaComputeNodePushable extends AbstractUnaryInputUnaryOutputOp
@Override
public void close() throws HyracksDataException {
- try {
- deregister();
- } finally {
- if (opened) {
- writer.close();
- }
+ if (opened) {
+ writer.close();
}
}
- private void deregister() {
- feedManager.getActiveRuntimeRegistry().deregisterRuntime(feedRuntime.getRuntimeId());
- }
-
@Override
public void flush() throws HyracksDataException {
writer.flush();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
index 95bebad..2c3b62f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
@@ -24,7 +24,6 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.asterix.active.ActiveManager;
-import org.apache.asterix.active.ActiveRuntime;
import org.apache.asterix.active.ActiveRuntimeId;
import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
import org.apache.asterix.common.dataflow.AsterixLSMInsertDeleteOperatorNodePushable;
@@ -34,7 +33,6 @@ import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.policy.FeedPolicyEnforcer;
import org.apache.asterix.external.util.FeedUtils;
import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType;
-import org.apache.hyracks.api.comm.IFrameWriter;
import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IActivity;
@@ -60,12 +58,6 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper
private final FeedPolicyEnforcer policyEnforcer;
/**
- * The Feed Runtime instance associated with the operator. Feed Runtime
- * captures the state of the operator while the feed is active.
- */
- private ActiveRuntime feedRuntime;
-
- /**
* A unique identifier for the feed instance. A feed instance represents
* the flow of data from a feed to a dataset.
**/
@@ -106,7 +98,7 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper
this.partition = partition;
this.connectionId = feedConnectionId;
this.feedManager = (ActiveManager) ((IAsterixAppRuntimeContext) ctx.getJobletContext().getApplicationContext()
- .getApplicationObject()).getFeedManager();
+ .getApplicationObject()).getActiveManager();
this.targetId = targetId;
this.message = new VSizeFrame(ctx);
TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, message, ctx);
@@ -116,8 +108,8 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper
@Override
public void open() throws HyracksDataException {
- ActiveRuntimeId runtimeId =
- new ActiveRuntimeId(connectionId.getFeedId(), runtimeType.toString() + "." + targetId, partition);
+ ActiveRuntimeId runtimeId = new ActiveRuntimeId(connectionId.getFeedId(),
+ runtimeType.toString() + "." + targetId, partition);
try {
initializeNewFeedRuntime(runtimeId);
insertOperator.open();
@@ -135,7 +127,6 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper
(AsterixLSMInsertDeleteOperatorNodePushable) insertOperator;
if (!indexOp.isPrimary()) {
writer = insertOperator;
- setupBasicRuntime(writer);
return;
}
}
@@ -145,14 +136,6 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper
} else {
writer = new SyncFeedRuntimeInputHandler(ctx, insertOperator, fta);
}
- setupBasicRuntime(writer);
- }
-
- private void setupBasicRuntime(IFrameWriter frameWriter) throws Exception {
- ActiveRuntimeId runtimeId =
- new ActiveRuntimeId(connectionId.getFeedId(), runtimeType.toString() + "." + targetId, partition);
- feedRuntime = new ActiveRuntime(runtimeId);
- feedManager.getActiveRuntimeRegistry().registerRuntime(feedRuntime);
}
@Override
@@ -173,15 +156,7 @@ public class FeedMetaStoreNodePushable extends AbstractUnaryInputUnaryOutputOper
@Override
public void close() throws HyracksDataException {
- try {
- writer.close();
- } finally {
- deregister();
- }
- }
-
- private void deregister() {
- feedManager.getActiveRuntimeRegistry().deregisterRuntime(feedRuntime.getRuntimeId());
+ writer.close();
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
index 65f61f2..5edc521 100644
--- a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
+++ b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
@@ -112,8 +112,8 @@ class AqlQueryRewriter implements IQueryRewriter {
buildOtherUdfs(topExpr.getBody(), otherFDecls, funIds);
declaredFunctions.addAll(otherFDecls);
if (!declaredFunctions.isEmpty()) {
- AQLInlineUdfsVisitor visitor = new AQLInlineUdfsVisitor(context, new AQLRewriterFactory(),
- declaredFunctions, metadataProvider);
+ AQLInlineUdfsVisitor visitor =
+ new AQLInlineUdfsVisitor(context, new AQLRewriterFactory(), declaredFunctions, metadataProvider);
while (topExpr.accept(visitor, declaredFunctions)) {
// loop until no more changes
}
@@ -154,9 +154,9 @@ class AqlQueryRewriter implements IQueryRewriter {
FunctionDecl functionDecl = functionParser.getFunctionDecl(function);
if (functionDecl != null) {
if (functionDecls.contains(functionDecl)) {
- throw new AsterixException("Recursive invocation "
- + functionDecls.get(functionDecls.size() - 1).getSignature() + " <==> "
- + functionDecl.getSignature());
+ throw new AsterixException(
+ "Recursive invocation " + functionDecls.get(functionDecls.size() - 1).getSignature()
+ + " <==> " + functionDecl.getSignature());
}
functionDecls.add(functionDecl);
buildOtherUdfs(functionDecl.getFuncBody(), functionDecls, declaredFunctions);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
index 836de6a..d4bf6bc 100644
--- a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
+++ b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
@@ -204,4 +204,9 @@ public class SubscribeFeedStatement implements Statement {
public String[] getLocations() {
return locations;
}
+
+ @Override
+ public byte getCategory() {
+ return Category.PROCEDURE;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
index 9cffda0..f46f13d 100644
--- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -43,7 +43,6 @@ import org.apache.asterix.common.annotations.TypeDataGen;
import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.aql.clause.DistinctClause;
@@ -106,7 +105,7 @@ import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
@@ -130,6 +129,7 @@ import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
@@ -872,7 +872,7 @@ Statement DropStatement() throws ParseException:
(
<DATASET> pairId = QualifiedName() ifExists = IfExists()
{
- stmt = new DropStatement(pairId.first, pairId.second, ifExists);
+ stmt = new DropDatasetStatement(pairId.first, pairId.second, ifExists);
}
| <INDEX> tripleId = DoubleQualifiedName() ifExists = IfExists()
{
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Expression.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Expression.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Expression.java
index b1df329..7da34fe 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Expression.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Expression.java
@@ -38,7 +38,6 @@ public interface Expression extends ILangExpression {
UNION_EXPRESSION,
SELECT_EXPRESSION,
PRIMARY_EXPRESSION,
- VALUE_EXPRESSION,
INDEPENDENT_SUBQUERY,
CASE_EXPRESSION
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
index fcc1080..b12195f 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
@@ -34,6 +34,28 @@ public interface Statement extends ILangExpression {
*/
public byte getKind();
+ /**
+ * get a byte representing the statement category.
+ * Each category describes the type of modifications this statement does.
+ *
+ * @return kind byte
+ */
+ public byte getCategory();
+
+ public class Category {
+ /** no modifications */
+ public static final byte QUERY = 0x01;
+ /** modify data */
+ public static final byte UPDATE = 0x02;
+ /** modify metadata */
+ public static final byte DDL = 0x04;
+ /** modify anything */
+ public static final byte PROCEDURE = 0x08;
+
+ private Category() {
+ }
+ }
+
public class Kind {
public static final byte DATASET_DECL = 0x00;
public static final byte DATAVERSE_DECL = 0x01;
@@ -70,6 +92,7 @@ public interface Statement extends ILangExpression {
public static final byte COMPACT = 0x20;
public static final byte EXTERNAL_DATASET_REFRESH = 0x21;
public static final byte RUN = 0x22;
+ public static final byte EXTENSION = 0x23;
private Kind() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CompactStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CompactStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CompactStatement.java
index 912ac6f..94aa1ac 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CompactStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CompactStatement.java
@@ -50,4 +50,9 @@ public class CompactStatement implements Statement {
public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
return visitor.visit(this, arg);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ConnectFeedStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ConnectFeedStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ConnectFeedStatement.java
index 48bc59e..fd1ca5e 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ConnectFeedStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/ConnectFeedStatement.java
@@ -102,4 +102,9 @@ public class ConnectFeedStatement implements Statement {
return feedName;
}
+ @Override
+ public byte getCategory() {
+ return Category.UPDATE;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateDataverseStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateDataverseStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateDataverseStatement.java
index c6dff3f..a475051 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateDataverseStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateDataverseStatement.java
@@ -57,4 +57,9 @@ public class CreateDataverseStatement implements Statement {
public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
return visitor.visit(this, arg);
}
+
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedPolicyStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedPolicyStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedPolicyStatement.java
index dfc6c7e..75fc556 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedPolicyStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedPolicyStatement.java
@@ -87,4 +87,9 @@ public class CreateFeedPolicyStatement implements Statement {
return description;
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedStatement.java
index c4f9efb..54b6a94 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFeedStatement.java
@@ -57,4 +57,8 @@ public abstract class CreateFeedStatement implements Statement {
@Override
public abstract <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException;
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
index 02a2b0c..5a75f43 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
@@ -75,4 +75,9 @@ public class CreateFunctionStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
index fd787ab..7f36c45 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
@@ -139,4 +139,9 @@ public class CreateIndexStatement implements Statement {
return visitor.visit(this, arg);
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
index b2f3fe1..bf27988 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
@@ -21,11 +21,11 @@ package org.apache.asterix.lang.common.statement;
import java.util.Map;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+import org.apache.asterix.metadata.utils.MetadataConstants;
public class DatasetDecl implements Statement {
protected final Identifier name;
@@ -153,4 +153,9 @@ public class DatasetDecl implements Statement {
return dataverse;
}
+ @Override
+ public byte getCategory() {
+ return Category.DDL;
+ }
+
}
[11/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
new file mode 100644
index 0000000..24c678c
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -0,0 +1,3122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.translator;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.rmi.RemoteException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.asterix.active.ActiveJobNotificationHandler;
+import org.apache.asterix.active.ActivityState;
+import org.apache.asterix.active.EntityId;
+import org.apache.asterix.active.IActiveEntityEventsListener;
+import org.apache.asterix.algebra.extension.IExtensionStatement;
+import org.apache.asterix.api.common.APIFramework;
+import org.apache.asterix.api.http.servlet.APIServlet;
+import org.apache.asterix.app.cc.CompilerExtensionManager;
+import org.apache.asterix.app.external.ExternalIndexingOperations;
+import org.apache.asterix.app.external.FeedJoint;
+import org.apache.asterix.app.external.FeedOperations;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.config.AsterixExternalProperties;
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
+import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.external.api.IAdapterFactory;
+import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.api.IFeed.FeedType;
+import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeedJoint.FeedJointType;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.api.IFeedLifecycleEventSubscriber.FeedLifecycleEvent;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.feed.management.FeedConnectionRequest;
+import org.apache.asterix.external.feed.management.FeedEventsListener;
+import org.apache.asterix.external.feed.management.FeedJointKey;
+import org.apache.asterix.external.feed.management.FeedLifecycleEventSubscriber;
+import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
+import org.apache.asterix.external.feed.watch.FeedActivity.FeedActivityDetails;
+import org.apache.asterix.external.feed.watch.FeedConnectJobInfo;
+import org.apache.asterix.external.feed.watch.FeedIntakeInfo;
+import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType;
+import org.apache.asterix.file.DatasetOperations;
+import org.apache.asterix.file.DataverseOperations;
+import org.apache.asterix.file.IndexOperations;
+import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
+import org.apache.asterix.lang.aql.statement.SubscribeFeedStatement;
+import org.apache.asterix.lang.common.base.IRewriterFactory;
+import org.apache.asterix.lang.common.base.IStatementRewriter;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.expression.TypeExpression;
+import org.apache.asterix.lang.common.statement.CompactStatement;
+import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
+import org.apache.asterix.lang.common.statement.CreateIndexStatement;
+import org.apache.asterix.lang.common.statement.CreatePrimaryFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateSecondaryFeedStatement;
+import org.apache.asterix.lang.common.statement.DatasetDecl;
+import org.apache.asterix.lang.common.statement.DataverseDecl;
+import org.apache.asterix.lang.common.statement.DataverseDropStatement;
+import org.apache.asterix.lang.common.statement.DeleteStatement;
+import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
+import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.FeedDropStatement;
+import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.FunctionDropStatement;
+import org.apache.asterix.lang.common.statement.IDatasetDetailsDecl;
+import org.apache.asterix.lang.common.statement.IndexDropStatement;
+import org.apache.asterix.lang.common.statement.InsertStatement;
+import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.LoadStatement;
+import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
+import org.apache.asterix.lang.common.statement.NodegroupDecl;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.statement.RefreshExternalDatasetStatement;
+import org.apache.asterix.lang.common.statement.RunStatement;
+import org.apache.asterix.lang.common.statement.SetStatement;
+import org.apache.asterix.lang.common.statement.TypeDecl;
+import org.apache.asterix.lang.common.statement.TypeDropStatement;
+import org.apache.asterix.lang.common.statement.WriteStatement;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.metadata.IDatasetDetails;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.api.IMetadataEntity;
+import org.apache.asterix.metadata.dataset.hints.DatasetHints;
+import org.apache.asterix.metadata.dataset.hints.DatasetHints.DatasetNodegroupCardinalityHint;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.metadata.entities.CompactionPolicy;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Datatype;
+import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
+import org.apache.asterix.metadata.entities.Feed;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
+import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.metadata.entities.NodeGroup;
+import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
+import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
+import org.apache.asterix.metadata.utils.DatasetUtils;
+import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
+import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
+import org.apache.asterix.metadata.utils.MetadataConstants;
+import org.apache.asterix.metadata.utils.MetadataLockManager;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeSignature;
+import org.apache.asterix.om.util.AsterixAppContextInfo;
+import org.apache.asterix.om.util.AsterixClusterProperties;
+import org.apache.asterix.optimizer.rules.IntroduceSecondaryIndexInsertDeleteRule;
+import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory;
+import org.apache.asterix.translator.AbstractLangTranslator;
+import org.apache.asterix.translator.CompiledStatements.CompiledConnectFeedStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledCreateIndexStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledDatasetDropStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledDeleteStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledIndexCompactStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledIndexDropStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledSubscribeFeedStatement;
+import org.apache.asterix.translator.CompiledStatements.CompiledUpsertStatement;
+import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.TypeTranslator;
+import org.apache.asterix.translator.util.ValidateUtil;
+import org.apache.asterix.util.FlushDatasetUtils;
+import org.apache.asterix.util.JobUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.mutable.MutableBoolean;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
+import org.apache.hyracks.algebricks.data.IAWriterFactory;
+import org.apache.hyracks.algebricks.data.IResultSerializerFactoryProvider;
+import org.apache.hyracks.algebricks.runtime.serializer.ResultSerializerFactoryProvider;
+import org.apache.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.dataflow.std.file.FileSplit;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import com.google.common.collect.Lists;
+
+/*
+ * Provides functionality for executing a batch of Query statements (queries included)
+ * sequentially.
+ */
+public class QueryTranslator extends AbstractLangTranslator implements IStatementExecutor {
+
+ private static final Logger LOGGER = Logger.getLogger(QueryTranslator.class.getName());
+
+ protected enum ProgressState {
+ NO_PROGRESS,
+ ADDED_PENDINGOP_RECORD_TO_METADATA
+ }
+
+ public static final boolean IS_DEBUG_MODE = false;// true
+ protected final List<Statement> statements;
+ protected final SessionConfig sessionConfig;
+ protected Dataverse activeDefaultDataverse;
+ protected final List<FunctionDecl> declaredFunctions;
+ protected final APIFramework apiFramework;
+ protected final IRewriterFactory rewriterFactory;
+
+ public QueryTranslator(List<Statement> aqlStatements, SessionConfig conf,
+ ILangCompilationProvider compliationProvider, CompilerExtensionManager ccExtensionManager) {
+ this.statements = aqlStatements;
+ this.sessionConfig = conf;
+ this.declaredFunctions = getDeclaredFunctions(aqlStatements);
+ this.apiFramework = new APIFramework(compliationProvider, ccExtensionManager);
+ this.rewriterFactory = compliationProvider.getRewriterFactory();
+ }
+
+ protected List<FunctionDecl> getDeclaredFunctions(List<Statement> statements) {
+ List<FunctionDecl> functionDecls = new ArrayList<>();
+ for (Statement st : statements) {
+ if (st.getKind() == Statement.Kind.FUNCTION_DECL) {
+ functionDecls.add((FunctionDecl) st);
+ }
+ }
+ return functionDecls;
+ }
+
+ /**
+ * Compiles and submits for execution a list of AQL statements.
+ *
+ * @param hcc
+ * A Hyracks client connection that is used to submit a jobspec to Hyracks.
+ * @param hdc
+ * A Hyracks dataset client object that is used to read the results.
+ * @param resultDelivery
+ * True if the results should be read asynchronously or false if we should wait for results to be read.
+ * @return A List<QueryResult> containing a QueryResult instance corresponding to each submitted query.
+ * @throws Exception
+ */
+ @Override
+ public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery)
+ throws Exception {
+ compileAndExecute(hcc, hdc, resultDelivery, new Stats());
+ }
+
+ @Override
+ public void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery,
+ Stats stats) throws Exception {
+ int resultSetIdCounter = 0;
+ FileSplit outputFile = null;
+ IAWriterFactory writerFactory = PrinterBasedWriterFactory.INSTANCE;
+ IResultSerializerFactoryProvider resultSerializerFactoryProvider = ResultSerializerFactoryProvider.INSTANCE;
+ Map<String, String> config = new HashMap<>();
+ /* Since the system runs a large number of threads, when HTTP requests don't return, it becomes difficult to
+ * find the thread running the request to determine where it has stopped.
+ * Setting the thread name helps make that easier
+ */
+ String threadName = Thread.currentThread().getName();
+ Thread.currentThread().setName(QueryTranslator.class.getSimpleName());
+ try {
+ for (Statement stmt : statements) {
+ if (sessionConfig.is(SessionConfig.FORMAT_HTML)) {
+ sessionConfig.out().println(APIServlet.HTML_STATEMENT_SEPARATOR);
+ }
+ validateOperation(activeDefaultDataverse, stmt);
+ rewriteStatement(stmt); // Rewrite the statement's AST.
+ AqlMetadataProvider metadataProvider = new AqlMetadataProvider(activeDefaultDataverse);
+ metadataProvider.setWriterFactory(writerFactory);
+ metadataProvider.setResultSerializerFactoryProvider(resultSerializerFactoryProvider);
+ metadataProvider.setOutputFile(outputFile);
+ metadataProvider.setConfig(config);
+ switch (stmt.getKind()) {
+ case Statement.Kind.SET:
+ handleSetStatement(stmt, config);
+ break;
+ case Statement.Kind.DATAVERSE_DECL:
+ activeDefaultDataverse = handleUseDataverseStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.CREATE_DATAVERSE:
+ handleCreateDataverseStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.DATASET_DECL:
+ handleCreateDatasetStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.CREATE_INDEX:
+ handleCreateIndexStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.TYPE_DECL:
+ handleCreateTypeStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.NODEGROUP_DECL:
+ handleCreateNodeGroupStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.DATAVERSE_DROP:
+ handleDataverseDropStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.DATASET_DROP:
+ handleDatasetDropStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.INDEX_DROP:
+ handleIndexDropStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.TYPE_DROP:
+ handleTypeDropStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.NODEGROUP_DROP:
+ handleNodegroupDropStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.CREATE_FUNCTION:
+ handleCreateFunctionStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.FUNCTION_DROP:
+ handleFunctionDropStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.LOAD:
+ handleLoadStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.INSERT:
+ case Statement.Kind.UPSERT:
+ handleInsertUpsertStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.DELETE:
+ handleDeleteStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.CREATE_PRIMARY_FEED:
+ case Statement.Kind.CREATE_SECONDARY_FEED:
+ handleCreateFeedStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.DROP_FEED:
+ handleDropFeedStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.DROP_FEED_POLICY:
+ handleDropFeedPolicyStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.CONNECT_FEED:
+ handleConnectFeedStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.DISCONNECT_FEED:
+ handleDisconnectFeedStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.SUBSCRIBE_FEED:
+ handleSubscribeFeedStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.CREATE_FEED_POLICY:
+ handleCreateFeedPolicyStatement(metadataProvider, stmt);
+ break;
+ case Statement.Kind.QUERY:
+ metadataProvider.setResultSetId(new ResultSetId(resultSetIdCounter++));
+ metadataProvider.setResultAsyncMode(resultDelivery == ResultDelivery.ASYNC
+ || resultDelivery == ResultDelivery.ASYNC_DEFERRED);
+ handleQuery(metadataProvider, (Query) stmt, hcc, hdc, resultDelivery, stats);
+ break;
+ case Statement.Kind.COMPACT:
+ handleCompactStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.EXTERNAL_DATASET_REFRESH:
+ handleExternalDatasetRefreshStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.WRITE:
+ Pair<IAWriterFactory, FileSplit> result = handleWriteStatement(stmt);
+ writerFactory = (result.first != null) ? result.first : writerFactory;
+ outputFile = result.second;
+ break;
+ case Statement.Kind.RUN:
+ handleRunStatement(metadataProvider, stmt, hcc);
+ break;
+ case Statement.Kind.FUNCTION_DECL:
+ // No op
+ break;
+ case Statement.Kind.EXTENSION:
+ ((IExtensionStatement) stmt).handle(this, metadataProvider, hcc);
+ break;
+ default:
+ throw new AsterixException("Unknown function");
+ }
+ }
+ } finally {
+ Thread.currentThread().setName(threadName);
+ }
+ }
+
+ protected void handleSetStatement(Statement stmt, Map<String, String> config) {
+ SetStatement ss = (SetStatement) stmt;
+ String pname = ss.getPropName();
+ String pvalue = ss.getPropValue();
+ config.put(pname, pvalue);
+ }
+
+ protected Pair<IAWriterFactory, FileSplit> handleWriteStatement(Statement stmt)
+ throws InstantiationException, IllegalAccessException, ClassNotFoundException {
+ WriteStatement ws = (WriteStatement) stmt;
+ File f = new File(ws.getFileName());
+ FileSplit outputFile = new FileSplit(ws.getNcName().getValue(), new FileReference(f));
+ IAWriterFactory writerFactory = null;
+ if (ws.getWriterClassName() != null) {
+ writerFactory = (IAWriterFactory) Class.forName(ws.getWriterClassName()).newInstance();
+ }
+ return new Pair<>(writerFactory, outputFile);
+ }
+
+ protected Dataverse handleUseDataverseStatement(AqlMetadataProvider metadataProvider, Statement stmt)
+ throws Exception {
+ DataverseDecl dvd = (DataverseDecl) stmt;
+ String dvName = dvd.getDataverseName().getValue();
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ MetadataLockManager.INSTANCE.acquireDataverseReadLock(dvName);
+ try {
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dvName);
+ if (dv == null) {
+ throw new MetadataException("Unknown dataverse " + dvName);
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ return dv;
+ } catch (Exception e) {
+ abort(e, e, mdTxnCtx);
+ throw new MetadataException(e);
+ } finally {
+ MetadataLockManager.INSTANCE.releaseDataverseReadLock(dvName);
+ }
+ }
+
+ protected void handleCreateDataverseStatement(AqlMetadataProvider metadataProvider, Statement stmt)
+ throws Exception {
+
+ CreateDataverseStatement stmtCreateDataverse = (CreateDataverseStatement) stmt;
+ String dvName = stmtCreateDataverse.getDataverseName().getValue();
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+ MetadataLockManager.INSTANCE.acquireDataverseReadLock(dvName);
+ try {
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(), dvName);
+ if (dv != null) {
+ if (stmtCreateDataverse.getIfNotExists()) {
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ return;
+ } else {
+ throw new AlgebricksException("A dataverse with this name " + dvName + " already exists.");
+ }
+ }
+ MetadataManager.INSTANCE.addDataverse(metadataProvider.getMetadataTxnContext(),
+ new Dataverse(dvName, stmtCreateDataverse.getFormat(), IMetadataEntity.PENDING_NO_OP));
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e) {
+ abort(e, e, mdTxnCtx);
+ throw e;
+ } finally {
+ MetadataLockManager.INSTANCE.releaseDataverseReadLock(dvName);
+ }
+ }
+
+ protected void validateCompactionPolicy(String compactionPolicy, Map<String, String> compactionPolicyProperties,
+ MetadataTransactionContext mdTxnCtx, boolean isExternalDataset) throws AsterixException, Exception {
+ CompactionPolicy compactionPolicyEntity = MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx,
+ MetadataConstants.METADATA_DATAVERSE_NAME, compactionPolicy);
+ if (compactionPolicyEntity == null) {
+ throw new AsterixException("Unknown compaction policy: " + compactionPolicy);
+ }
+ String compactionPolicyFactoryClassName = compactionPolicyEntity.getClassName();
+ ILSMMergePolicyFactory mergePolicyFactory = (ILSMMergePolicyFactory) Class
+ .forName(compactionPolicyFactoryClassName).newInstance();
+ if (isExternalDataset && mergePolicyFactory.getName().compareTo("correlated-prefix") == 0) {
+ throw new AsterixException("The correlated-prefix merge policy cannot be used with external dataset.");
+ }
+ if (compactionPolicyProperties == null) {
+ if (mergePolicyFactory.getName().compareTo("no-merge") != 0) {
+ throw new AsterixException("Compaction policy properties are missing.");
+ }
+ } else {
+ for (Map.Entry<String, String> entry : compactionPolicyProperties.entrySet()) {
+ if (!mergePolicyFactory.getPropertiesNames().contains(entry.getKey())) {
+ throw new AsterixException("Invalid compaction policy property: " + entry.getKey());
+ }
+ }
+ for (String p : mergePolicyFactory.getPropertiesNames()) {
+ if (!compactionPolicyProperties.containsKey(p)) {
+ throw new AsterixException("Missing compaction policy property: " + p);
+ }
+ }
+ }
+ }
+
+ protected void handleCreateDatasetStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc) throws AsterixException, Exception {
+ MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS);
+ DatasetDecl dd = (DatasetDecl) stmt;
+ String dataverseName = getActiveDataverse(dd.getDataverse());
+ String datasetName = dd.getName().getValue();
+ DatasetType dsType = dd.getDatasetType();
+ String itemTypeDataverseName = dd.getItemTypeDataverse().getValue();
+ String itemTypeName = dd.getItemTypeName().getValue();
+ String metaItemTypeDataverseName = dd.getMetaItemTypeDataverse().getValue();
+ String metaItemTypeName = dd.getMetaItemTypeName().getValue();
+ Identifier ngNameId = dd.getNodegroupName();
+ String nodegroupName = getNodeGroupName(ngNameId, dd, dataverseName);
+ String compactionPolicy = dd.getCompactionPolicy();
+ Map<String, String> compactionPolicyProperties = dd.getCompactionPolicyProperties();
+ boolean defaultCompactionPolicy = compactionPolicy == null;
+ boolean temp = dd.getDatasetDetailsDecl().isTemp();
+
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ boolean bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+ MetadataLockManager.INSTANCE.createDatasetBegin(dataverseName, itemTypeDataverseName,
+ itemTypeDataverseName + "." + itemTypeName, metaItemTypeDataverseName,
+ metaItemTypeDataverseName + "." + metaItemTypeName, nodegroupName, compactionPolicy,
+ dataverseName + "." + datasetName, defaultCompactionPolicy);
+ Dataset dataset = null;
+ try {
+
+ IDatasetDetails datasetDetails = null;
+ Dataset ds = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName);
+ if (ds != null) {
+ if (dd.getIfNotExists()) {
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ return;
+ } else {
+ throw new AlgebricksException("A dataset with this name " + datasetName + " already exists.");
+ }
+ }
+ Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
+ itemTypeDataverseName, itemTypeName);
+ if (dt == null) {
+ throw new AlgebricksException(": type " + itemTypeName + " could not be found.");
+ }
+ String ngName = ngNameId != null ? ngNameId.getValue()
+ : configureNodegroupForDataset(dd, dataverseName, mdTxnCtx);
+
+ if (compactionPolicy == null) {
+ compactionPolicy = GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME;
+ compactionPolicyProperties = GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES;
+ } else {
+ validateCompactionPolicy(compactionPolicy, compactionPolicyProperties, mdTxnCtx, false);
+ }
+ switch (dd.getDatasetType()) {
+ case INTERNAL:
+ IAType itemType = dt.getDatatype();
+ if (itemType.getTypeTag() != ATypeTag.RECORD) {
+ throw new AlgebricksException("Dataset type has to be a record type.");
+ }
+
+ IAType metaItemType = null;
+ if (metaItemTypeDataverseName != null && metaItemTypeName != null) {
+ metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
+ }
+ if (metaItemType != null && metaItemType.getTypeTag() != ATypeTag.RECORD) {
+ throw new AlgebricksException("Dataset meta type has to be a record type.");
+ }
+ ARecordType metaRecType = (ARecordType) metaItemType;
+
+ List<List<String>> partitioningExprs = ((InternalDetailsDecl) dd.getDatasetDetailsDecl())
+ .getPartitioningExprs();
+ List<Integer> keySourceIndicators = ((InternalDetailsDecl) dd.getDatasetDetailsDecl())
+ .getKeySourceIndicators();
+ boolean autogenerated = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated();
+ ARecordType aRecordType = (ARecordType) itemType;
+ List<IAType> partitioningTypes = ValidateUtil.validatePartitioningExpressions(aRecordType,
+ metaRecType, partitioningExprs, keySourceIndicators, autogenerated);
+
+ List<String> filterField = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getFilterField();
+ if (filterField != null) {
+ ValidateUtil.validateFilterField(aRecordType, filterField);
+ }
+ if (compactionPolicy == null && filterField != null) {
+ // If the dataset has a filter and the user didn't specify a merge
+ // policy, then we will pick the
+ // correlated-prefix as the default merge policy.
+ compactionPolicy = GlobalConfig.DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME;
+ compactionPolicyProperties = GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES;
+ }
+ datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
+ InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
+ keySourceIndicators, partitioningTypes, autogenerated, filterField, temp);
+ break;
+ case EXTERNAL:
+ String adapter = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl()).getAdapter();
+ Map<String, String> properties = ((ExternalDetailsDecl) dd.getDatasetDetailsDecl())
+ .getProperties();
+
+ datasetDetails = new ExternalDatasetDetails(adapter, properties, new Date(),
+ ExternalDatasetTransactionState.COMMIT);
+ break;
+ default:
+ throw new AsterixException("Unknown datatype " + dd.getDatasetType());
+ }
+
+ // #. initialize DatasetIdFactory if it is not initialized.
+ if (!DatasetIdFactory.isInitialized()) {
+ DatasetIdFactory.initialize(MetadataManager.INSTANCE.getMostRecentDatasetId());
+ }
+
+ // #. add a new dataset with PendingAddOp
+ dataset = new Dataset(dataverseName, datasetName, itemTypeDataverseName, itemTypeName,
+ metaItemTypeDataverseName, metaItemTypeName, ngName, compactionPolicy, compactionPolicyProperties,
+ datasetDetails, dd.getHints(), dsType, DatasetIdFactory.generateDatasetId(),
+ IMetadataEntity.PENDING_ADD_OP);
+ MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset);
+
+ if (dd.getDatasetType() == DatasetType.INTERNAL) {
+ Dataverse dataverse = MetadataManager.INSTANCE.getDataverse(metadataProvider.getMetadataTxnContext(),
+ dataverseName);
+ JobSpecification jobSpec = DatasetOperations.createDatasetJobSpec(dataverse, datasetName,
+ metadataProvider);
+
+ // #. make metadataTxn commit before calling runJob.
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ progress.setValue(ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA);
+
+ // #. runJob
+ JobUtils.runJob(hcc, jobSpec, true);
+
+ // #. begin new metadataTxn
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ }
+
+ // #. add a new dataset with PendingNoOp after deleting the dataset with PendingAddOp
+ MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName);
+ dataset.setPendingOp(IMetadataEntity.PENDING_NO_OP);
+ MetadataManager.INSTANCE.addDataset(metadataProvider.getMetadataTxnContext(), dataset);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e) {
+ if (bActiveTxn) {
+ abort(e, e, mdTxnCtx);
+ }
+
+ if (progress.getValue() == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
+
+ // #. execute compensation operations
+ // remove the index in NC
+ // [Notice]
+ // As long as we updated(and committed) metadata, we should remove any effect of the job
+ // because an exception occurs during runJob.
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
+ try {
+ JobSpecification jobSpec = DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ JobUtils.runJob(hcc, jobSpec, true);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ if (bActiveTxn) {
+ abort(e, e2, mdTxnCtx);
+ }
+ }
+
+ // remove the record from the metadata.
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+ MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx);
+ throw new IllegalStateException("System is inconsistent state: pending dataset(" + dataverseName
+ + "." + datasetName + ") couldn't be removed from the metadata", e);
+ }
+ }
+
+ throw e;
+ } finally {
+ MetadataLockManager.INSTANCE.createDatasetEnd(dataverseName, itemTypeDataverseName,
+ itemTypeDataverseName + "." + itemTypeName, metaItemTypeDataverseName,
+ metaItemTypeDataverseName + "." + metaItemTypeName, nodegroupName, compactionPolicy,
+ dataverseName + "." + datasetName, defaultCompactionPolicy);
+ }
+ }
+
+ protected void validateIfResourceIsActiveInFeed(String dataverseName, String datasetName) throws AsterixException {
+ StringBuilder builder = null;
+ IActiveEntityEventsListener[] listeners = ActiveJobNotificationHandler.INSTANCE.getEventListeners();
+ for (IActiveEntityEventsListener listener : listeners) {
+ if (listener instanceof FeedEventsListener
+ && ((FeedEventsListener) listener).isConnectedToDataset(datasetName)) {
+ if (builder == null) {
+ builder = new StringBuilder();
+ }
+ builder.append(listener.getEntityId() + "\n");
+ }
+ }
+ if (builder != null) {
+ throw new AsterixException("Dataset " + dataverseName + "." + datasetName + " is currently being "
+ + "fed into by the following feed(s).\n" + builder.toString() + "\n" + "Operation not supported");
+ }
+ }
+
+ protected String getNodeGroupName(Identifier ngNameId, DatasetDecl dd, String dataverse) {
+ if (ngNameId != null) {
+ return ngNameId.getValue();
+ }
+ String hintValue = dd.getHints().get(DatasetNodegroupCardinalityHint.NAME);
+ if (hintValue == null) {
+ return MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME;
+ } else {
+ return dataverse + ":" + dd.getName().getValue();
+ }
+ }
+
+ protected String configureNodegroupForDataset(DatasetDecl dd, String dataverse,
+ MetadataTransactionContext mdTxnCtx)
+ throws AsterixException {
+ int nodegroupCardinality;
+ String nodegroupName;
+ String hintValue = dd.getHints().get(DatasetNodegroupCardinalityHint.NAME);
+ if (hintValue == null) {
+ nodegroupName = MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME;
+ return nodegroupName;
+ } else {
+ int numChosen = 0;
+ boolean valid = DatasetHints.validate(DatasetNodegroupCardinalityHint.NAME,
+ dd.getHints().get(DatasetNodegroupCardinalityHint.NAME)).first;
+ if (!valid) {
+ throw new AsterixException("Incorrect use of hint:" + DatasetNodegroupCardinalityHint.NAME);
+ } else {
+ nodegroupCardinality = Integer.parseInt(dd.getHints().get(DatasetNodegroupCardinalityHint.NAME));
+ }
+ List<String> nodeNames = AsterixAppContextInfo.getInstance().getMetadataProperties().getNodeNames();
+ List<String> nodeNamesClone = new ArrayList<>(nodeNames);
+ String metadataNodeName = AsterixAppContextInfo.getInstance().getMetadataProperties().getMetadataNodeName();
+ List<String> selectedNodes = new ArrayList<>();
+ selectedNodes.add(metadataNodeName);
+ numChosen++;
+ nodeNamesClone.remove(metadataNodeName);
+
+ if (numChosen < nodegroupCardinality) {
+ Random random = new Random();
+ String[] nodes = nodeNamesClone.toArray(new String[] {});
+ int[] b = new int[nodeNamesClone.size()];
+ for (int i = 0; i < b.length; i++) {
+ b[i] = i;
+ }
+
+ for (int i = 0; i < nodegroupCardinality - numChosen; i++) {
+ int selected = i + random.nextInt(nodeNamesClone.size() - i);
+ int selNodeIndex = b[selected];
+ selectedNodes.add(nodes[selNodeIndex]);
+ int temp = b[0];
+ b[0] = b[selected];
+ b[selected] = temp;
+ }
+ }
+ nodegroupName = dataverse + ":" + dd.getName().getValue();
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodegroupName, selectedNodes));
+ return nodegroupName;
+ }
+
+ }
+
+ protected void handleCreateIndexStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc) throws Exception {
+ ProgressState progress = ProgressState.NO_PROGRESS;
+ CreateIndexStatement stmtCreateIndex = (CreateIndexStatement) stmt;
+ String dataverseName = getActiveDataverse(stmtCreateIndex.getDataverseName());
+ String datasetName = stmtCreateIndex.getDatasetName().getValue();
+ List<Integer> keySourceIndicators = stmtCreateIndex.getFieldSourceIndicators();
+
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ boolean bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+ MetadataLockManager.INSTANCE.createIndexBegin(dataverseName, dataverseName + "." + datasetName);
+
+ String indexName = null;
+ JobSpecification spec = null;
+ Dataset ds = null;
+ // For external datasets
+ ArrayList<ExternalFile> externalFilesSnapshot = null;
+ boolean firstExternalDatasetIndex = false;
+ boolean filesIndexReplicated = false;
+ Index filesIndex = null;
+ boolean datasetLocked = false;
+ try {
+ ds = MetadataManager.INSTANCE.getDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName);
+ if (ds == null) {
+ throw new AlgebricksException(
+ "There is no dataset with this name " + datasetName + " in dataverse " + dataverseName);
+ }
+
+ indexName = stmtCreateIndex.getIndexName().getValue();
+ Index idx = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName, indexName);
+ Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
+ ds.getItemTypeDataverseName(), ds.getItemTypeName());
+ ARecordType aRecordType = (ARecordType) dt.getDatatype();
+ ARecordType metaRecordType = null;
+ if (ds.hasMetaPart()) {
+ Datatype metaDt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
+ ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
+ metaRecordType = (ARecordType) metaDt.getDatatype();
+ }
+
+ List<List<String>> indexFields = new ArrayList<>();
+ List<IAType> indexFieldTypes = new ArrayList<>();
+ int keyIndex = 0;
+ for (Pair<List<String>, TypeExpression> fieldExpr : stmtCreateIndex.getFieldExprs()) {
+ IAType fieldType = null;
+ ARecordType subType = KeyFieldTypeUtils.chooseSource(keySourceIndicators, keyIndex, aRecordType,
+ metaRecordType);
+ boolean isOpen = subType.isOpen();
+ int i = 0;
+ if (fieldExpr.first.size() > 1 && !isOpen) {
+ while (i < fieldExpr.first.size() - 1 && !isOpen) {
+ subType = (ARecordType) subType.getFieldType(fieldExpr.first.get(i));
+ i++;
+ isOpen = subType.isOpen();
+ }
+ }
+ if (fieldExpr.second == null) {
+ fieldType = subType.getSubFieldType(fieldExpr.first.subList(i, fieldExpr.first.size()));
+ } else {
+ if (!stmtCreateIndex.isEnforced()) {
+ throw new AlgebricksException("Cannot create typed index on \"" + fieldExpr.first
+ + "\" field without enforcing it's type");
+ }
+ if (!isOpen) {
+ throw new AlgebricksException("Typed index on \"" + fieldExpr.first
+ + "\" field could be created only for open datatype");
+ }
+ if (stmtCreateIndex.hasMetaField()) {
+ throw new AlgebricksException("Typed open index can only be created on the record part");
+ }
+ Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx, fieldExpr.second,
+ indexName, dataverseName);
+ TypeSignature typeSignature = new TypeSignature(dataverseName, indexName);
+ fieldType = typeMap.get(typeSignature);
+ }
+ if (fieldType == null) {
+ throw new AlgebricksException(
+ "Unknown type " + (fieldExpr.second == null ? fieldExpr.first : fieldExpr.second));
+ }
+
+ indexFields.add(fieldExpr.first);
+ indexFieldTypes.add(fieldType);
+ ++keyIndex;
+ }
+
+ ValidateUtil.validateKeyFields(aRecordType, metaRecordType, indexFields, keySourceIndicators,
+ indexFieldTypes, stmtCreateIndex.getIndexType());
+
+ if (idx != null) {
+ if (stmtCreateIndex.getIfNotExists()) {
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ return;
+ } else {
+ throw new AlgebricksException("An index with this name " + indexName + " already exists.");
+ }
+ }
+
+ // Checks whether a user is trying to create an inverted secondary index on a dataset
+ // with a variable-length primary key.
+ // Currently, we do not support this. Therefore, as a temporary solution, we print an
+ // error message and stop.
+ if (stmtCreateIndex.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
+ || stmtCreateIndex.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX
+ || stmtCreateIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+ || stmtCreateIndex.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
+ List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(ds);
+ for (List<String> partitioningKey : partitioningKeys) {
+ IAType keyType = aRecordType.getSubFieldType(partitioningKey);
+ ITypeTraits typeTrait = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
+
+ // If it is not a fixed length
+ if (typeTrait.getFixedLength() < 0) {
+ throw new AlgebricksException("The keyword or ngram index -" + indexName
+ + " cannot be created on the dataset -" + datasetName
+ + " due to its variable-length primary key field - " + partitioningKey);
+ }
+
+ }
+ }
+
+ if (ds.getDatasetType() == DatasetType.INTERNAL) {
+ validateIfResourceIsActiveInFeed(dataverseName, datasetName);
+ } else {
+ // External dataset
+ // Check if the dataset is indexible
+ if (!ExternalIndexingOperations.isIndexible((ExternalDatasetDetails) ds.getDatasetDetails())) {
+ throw new AlgebricksException(
+ "dataset using " + ((ExternalDatasetDetails) ds.getDatasetDetails()).getAdapter()
+ + " Adapter can't be indexed");
+ }
+ // Check if the name of the index is valid
+ if (!ExternalIndexingOperations.isValidIndexName(datasetName, indexName)) {
+ throw new AlgebricksException("external dataset index name is invalid");
+ }
+
+ // Check if the files index exist
+ filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+ firstExternalDatasetIndex = filesIndex == null;
+ // Lock external dataset
+ ExternalDatasetsRegistry.INSTANCE.buildIndexBegin(ds, firstExternalDatasetIndex);
+ datasetLocked = true;
+ if (firstExternalDatasetIndex) {
+ // Verify that no one has created an index before we acquire the lock
+ filesIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(),
+ dataverseName, datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+ if (filesIndex != null) {
+ ExternalDatasetsRegistry.INSTANCE.buildIndexEnd(ds, firstExternalDatasetIndex);
+ firstExternalDatasetIndex = false;
+ ExternalDatasetsRegistry.INSTANCE.buildIndexBegin(ds, firstExternalDatasetIndex);
+ }
+ }
+ if (firstExternalDatasetIndex) {
+ // Get snapshot from External File System
+ externalFilesSnapshot = ExternalIndexingOperations.getSnapshotFromExternalFileSystem(ds);
+ // Add an entry for the files index
+ filesIndex = new Index(dataverseName, datasetName,
+ ExternalIndexingOperations.getFilesIndexName(datasetName), IndexType.BTREE,
+ ExternalIndexingOperations.FILE_INDEX_FIELD_NAMES, null,
+ ExternalIndexingOperations.FILE_INDEX_FIELD_TYPES, false, false,
+ IMetadataEntity.PENDING_ADD_OP);
+ MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
+ // Add files to the external files index
+ for (ExternalFile file : externalFilesSnapshot) {
+ MetadataManager.INSTANCE.addExternalFile(mdTxnCtx, file);
+ }
+ // This is the first index for the external dataset, replicate the files index
+ spec = ExternalIndexingOperations.buildFilesIndexReplicationJobSpec(ds, externalFilesSnapshot,
+ metadataProvider, true);
+ if (spec == null) {
+ throw new AsterixException(
+ "Failed to create job spec for replicating Files Index For external dataset");
+ }
+ filesIndexReplicated = true;
+ JobUtils.runJob(hcc, spec, true);
+ }
+ }
+
+ // check whether there exists another enforced index on the same field
+ if (stmtCreateIndex.isEnforced()) {
+ List<Index> indexes = MetadataManager.INSTANCE
+ .getDatasetIndexes(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName);
+ for (Index index : indexes) {
+ if (index.getKeyFieldNames().equals(indexFields)
+ && !index.getKeyFieldTypes().equals(indexFieldTypes) && index.isEnforcingKeyFileds()) {
+ throw new AsterixException("Cannot create index " + indexName + " , enforced index "
+ + index.getIndexName() + " on field \"" + StringUtils.join(indexFields, ',')
+ + "\" is already defined with type \"" + index.getKeyFieldTypes() + "\"");
+ }
+ }
+ }
+
+ // #. add a new index with PendingAddOp
+ Index index = new Index(dataverseName, datasetName, indexName, stmtCreateIndex.getIndexType(), indexFields,
+ keySourceIndicators, indexFieldTypes, stmtCreateIndex.getGramLength(), stmtCreateIndex.isEnforced(),
+ false, IMetadataEntity.PENDING_ADD_OP);
+ MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
+
+ ARecordType enforcedType = null;
+ if (stmtCreateIndex.isEnforced()) {
+ enforcedType = IntroduceSecondaryIndexInsertDeleteRule.createEnforcedType(aRecordType,
+ Lists.newArrayList(index));
+ }
+
+ // #. prepare to create the index artifact in NC.
+ CompiledCreateIndexStatement cis = new CompiledCreateIndexStatement(index.getIndexName(), dataverseName,
+ index.getDatasetName(), index.getKeyFieldNames(), index.getKeyFieldTypes(),
+ index.isEnforcingKeyFileds(), index.getGramLength(), index.getIndexType());
+ spec = IndexOperations.buildSecondaryIndexCreationJobSpec(cis, aRecordType, metaRecordType,
+ keySourceIndicators, enforcedType, metadataProvider);
+ if (spec == null) {
+ throw new AsterixException("Failed to create job spec for creating index '"
+ + stmtCreateIndex.getDatasetName() + "." + stmtCreateIndex.getIndexName() + "'");
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+
+ progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+
+ // #. create the index artifact in NC.
+ JobUtils.runJob(hcc, spec, true);
+
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+ // #. load data into the index in NC.
+ cis = new CompiledCreateIndexStatement(index.getIndexName(), dataverseName, index.getDatasetName(),
+ index.getKeyFieldNames(), index.getKeyFieldTypes(), index.isEnforcingKeyFileds(),
+ index.getGramLength(), index.getIndexType());
+
+ spec = IndexOperations.buildSecondaryIndexLoadingJobSpec(cis, aRecordType, metaRecordType,
+ keySourceIndicators, enforcedType, metadataProvider);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+
+ JobUtils.runJob(hcc, spec, true);
+
+ // #. begin new metadataTxn
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+ // #. add another new index with PendingNoOp after deleting the index with PendingAddOp
+ MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName,
+ indexName);
+ index.setPendingOp(IMetadataEntity.PENDING_NO_OP);
+ MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
+ // add another new files index with PendingNoOp after deleting the index with
+ // PendingAddOp
+ if (firstExternalDatasetIndex) {
+ MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName, datasetName,
+ filesIndex.getIndexName());
+ filesIndex.setPendingOp(IMetadataEntity.PENDING_NO_OP);
+ MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
+ // update transaction timestamp
+ ((ExternalDatasetDetails) ds.getDatasetDetails()).setRefreshTimestamp(new Date());
+ MetadataManager.INSTANCE.updateDataset(mdTxnCtx, ds);
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+
+ } catch (Exception e) {
+ if (bActiveTxn) {
+ abort(e, e, mdTxnCtx);
+ }
+ // If files index was replicated for external dataset, it should be cleaned up on NC side
+ if (filesIndexReplicated) {
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+ ExternalIndexingOperations.getFilesIndexName(datasetName));
+ try {
+ JobSpecification jobSpec = ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+ metadataProvider, ds);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ JobUtils.runJob(hcc, jobSpec, true);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ if (bActiveTxn) {
+ abort(e, e2, mdTxnCtx);
+ }
+ }
+ }
+
+ if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
+ // #. execute compensation operations
+ // remove the index in NC
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName, indexName);
+ try {
+ JobSpecification jobSpec = IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider,
+ ds);
+
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ JobUtils.runJob(hcc, jobSpec, true);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ if (bActiveTxn) {
+ abort(e, e2, mdTxnCtx);
+ }
+ }
+
+ if (firstExternalDatasetIndex) {
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+ // Drop External Files from metadata
+ MetadataManager.INSTANCE.dropDatasetExternalFiles(mdTxnCtx, ds);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx);
+ throw new IllegalStateException("System is inconsistent state: pending files for("
+ + dataverseName + "." + datasetName + ") couldn't be removed from the metadata", e);
+ }
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+ // Drop the files index from metadata
+ MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName, ExternalIndexingOperations.getFilesIndexName(datasetName));
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx);
+ throw new IllegalStateException("System is inconsistent state: pending index(" + dataverseName
+ + "." + datasetName + "." + ExternalIndexingOperations.getFilesIndexName(datasetName)
+ + ") couldn't be removed from the metadata", e);
+ }
+ }
+ // remove the record from the metadata.
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+ MetadataManager.INSTANCE.dropIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName, indexName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx);
+ throw new IllegalStateException("System is in inconsistent state: pending index(" + dataverseName
+ + "." + datasetName + "." + indexName + ") couldn't be removed from the metadata", e);
+ }
+ }
+ throw e;
+ } finally {
+ MetadataLockManager.INSTANCE.createIndexEnd(dataverseName, dataverseName + "." + datasetName);
+ if (datasetLocked) {
+ ExternalDatasetsRegistry.INSTANCE.buildIndexEnd(ds, firstExternalDatasetIndex);
+ }
+ }
+ }
+
+ protected void handleCreateTypeStatement(AqlMetadataProvider metadataProvider, Statement stmt) throws Exception {
+ TypeDecl stmtCreateType = (TypeDecl) stmt;
+ String dataverseName = getActiveDataverse(stmtCreateType.getDataverseName());
+ String typeName = stmtCreateType.getIdent().getValue();
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ MetadataLockManager.INSTANCE.createTypeBegin(dataverseName, dataverseName + "." + typeName);
+ try {
+
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ throw new AlgebricksException("Unknown dataverse " + dataverseName);
+ }
+ Datatype dt = MetadataManager.INSTANCE.getDatatype(mdTxnCtx, dataverseName, typeName);
+ if (dt != null) {
+ if (!stmtCreateType.getIfNotExists()) {
+ throw new AlgebricksException("A datatype with this name " + typeName + " already exists.");
+ }
+ } else {
+ if (builtinTypeMap.get(typeName) != null) {
+ throw new AlgebricksException("Cannot redefine builtin type " + typeName + ".");
+ } else {
+ Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx,
+ stmtCreateType.getTypeDef(), stmtCreateType.getIdent().getValue(), dataverseName);
+ TypeSignature typeSignature = new TypeSignature(dataverseName, typeName);
+ IAType type = typeMap.get(typeSignature);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(dataverseName, typeName, type, false));
+ }
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e) {
+ abort(e, e, mdTxnCtx);
+ throw e;
+ } finally {
+ MetadataLockManager.INSTANCE.createTypeEnd(dataverseName, dataverseName + "." + typeName);
+ }
+ }
+
+ protected void handleDataverseDropStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc) throws Exception {
+ DataverseDropStatement stmtDelete = (DataverseDropStatement) stmt;
+ String dataverseName = stmtDelete.getDataverseName().getValue();
+
+ ProgressState progress = ProgressState.NO_PROGRESS;
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ boolean bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ MetadataLockManager.INSTANCE.acquireDataverseWriteLock(dataverseName);
+ List<JobSpecification> jobsToExecute = new ArrayList<>();
+ try {
+ Dataverse dv = MetadataManager.INSTANCE.getDataverse(mdTxnCtx, dataverseName);
+ if (dv == null) {
+ if (stmtDelete.getIfExists()) {
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ return;
+ } else {
+ throw new AlgebricksException("There is no dataverse with this name " + dataverseName + ".");
+ }
+ }
+ // # disconnect all feeds from any datasets in the dataverse.
+ IActiveEntityEventsListener[] activeListeners = ActiveJobNotificationHandler.INSTANCE.getEventListeners();
+ Identifier dvId = new Identifier(dataverseName);
+ for (IActiveEntityEventsListener listener : activeListeners) {
+ EntityId activeEntityId = listener.getEntityId();
+ if (activeEntityId.getExtensionName().equals(Feed.EXTENSION_NAME)
+ && activeEntityId.getDataverse().equals(dataverseName)) {
+ FeedEventsListener feedEventListener = (FeedEventsListener) listener;
+ FeedConnectionId[] connections = feedEventListener.getConnections();
+ for (FeedConnectionId conn : connections) {
+ disconnectFeedBeforeDelete(dvId, activeEntityId, conn, metadataProvider, hcc);
+ }
+ // prepare job to remove feed log storage
+ jobsToExecute.add(FeedOperations.buildRemoveFeedStorageJob(
+ MetadataManager.INSTANCE.getFeed(mdTxnCtx, dataverseName, activeEntityId.getEntityName())));
+ }
+ }
+
+ // #. prepare jobs which will drop corresponding datasets with indexes.
+ List<Dataset> datasets = MetadataManager.INSTANCE.getDataverseDatasets(mdTxnCtx, dataverseName);
+ for (int j = 0; j < datasets.size(); j++) {
+ String datasetName = datasets.get(j).getDatasetName();
+ DatasetType dsType = datasets.get(j).getDatasetType();
+ if (dsType == DatasetType.INTERNAL) {
+ List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
+ datasetName);
+ for (int k = 0; k < indexes.size(); k++) {
+ if (indexes.get(k).isSecondaryIndex()) {
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+ indexes.get(k).getIndexName());
+ jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider,
+ datasets.get(j)));
+ }
+ }
+
+ CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
+ jobsToExecute.add(DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider));
+ } else {
+ // External dataset
+ List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
+ datasetName);
+ for (int k = 0; k < indexes.size(); k++) {
+ if (ExternalIndexingOperations.isFileIndex(indexes.get(k))) {
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+ indexes.get(k).getIndexName());
+ jobsToExecute.add(ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds,
+ metadataProvider, datasets.get(j)));
+ } else {
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+ indexes.get(k).getIndexName());
+ jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider,
+ datasets.get(j)));
+ }
+ }
+ ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(datasets.get(j));
+ }
+ }
+ jobsToExecute.add(DataverseOperations.createDropDataverseJobSpec(dv, metadataProvider));
+ // #. mark PendingDropOp on the dataverse record by
+ // first, deleting the dataverse record from the DATAVERSE_DATASET
+ // second, inserting the dataverse record with the PendingDropOp value into the
+ // DATAVERSE_DATASET
+ MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverseName);
+ MetadataManager.INSTANCE.addDataverse(mdTxnCtx,
+ new Dataverse(dataverseName, dv.getDataFormat(), IMetadataEntity.PENDING_DROP_OP));
+
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ bActiveTxn = false;
+ progress = ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA;
+
+ for (JobSpecification jobSpec : jobsToExecute) {
+ JobUtils.runJob(hcc, jobSpec, true);
+ }
+
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+ // #. finally, delete the dataverse.
+ MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverseName);
+ if (activeDefaultDataverse != null && activeDefaultDataverse.getDataverseName() == dataverseName) {
+ activeDefaultDataverse = null;
+ }
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e) {
+ if (bActiveTxn) {
+ abort(e, e, mdTxnCtx);
+ }
+
+ if (progress == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
+ if (activeDefaultDataverse != null && activeDefaultDataverse.getDataverseName() == dataverseName) {
+ activeDefaultDataverse = null;
+ }
+
+ // #. execute compensation operations
+ // remove the all indexes in NC
+ try {
+ for (JobSpecification jobSpec : jobsToExecute) {
+ JobUtils.runJob(hcc, jobSpec, true);
+ }
+ } catch (Exception e2) {
+ // do no throw exception since still the metadata needs to be compensated.
+ e.addSuppressed(e2);
+ }
+
+ // remove the record from the metadata.
+ mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ try {
+ MetadataManager.INSTANCE.dropDataverse(mdTxnCtx, dataverseName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx);
+ throw new IllegalStateException("System is inconsistent state: pending dataverse(" + dataverseName
+ + ") couldn't be removed from the metadata", e);
+ }
+ }
+
+ throw e;
+ } finally {
+ MetadataLockManager.INSTANCE.releaseDataverseWriteLock(dataverseName);
+ }
+ }
+
+ protected void disconnectFeedBeforeDelete(Identifier dvId, EntityId activeEntityId, FeedConnectionId conn,
+ AqlMetadataProvider metadataProvider, IHyracksClientConnection hcc) {
+ DisconnectFeedStatement disStmt = new DisconnectFeedStatement(dvId,
+ new Identifier(activeEntityId.getEntityName()), new Identifier(conn.getDatasetName()));
+ try {
+ handleDisconnectFeedStatement(metadataProvider, disStmt, hcc);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disconnected feed " + activeEntityId.getEntityName() + " from dataset "
+ + conn.getDatasetName());
+ }
+ } catch (Exception exception) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Unable to disconnect feed " + activeEntityId.getEntityName() + " from dataset "
+ + conn.getDatasetName() + ". Encountered exception " + exception);
+ }
+ }
+ }
+
+ protected Dataset getDataset(MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName)
+ throws MetadataException {
+ return MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
+ }
+
+ protected void handleDatasetDropStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc) throws Exception {
+ DropDatasetStatement stmtDelete = (DropDatasetStatement) stmt;
+ String dataverseName = getActiveDataverse(stmtDelete.getDataverseName());
+ String datasetName = stmtDelete.getDatasetName().getValue();
+ MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS);
+ MutableObject<MetadataTransactionContext> mdTxnCtx = new MutableObject<>(
+ MetadataManager.INSTANCE.beginTransaction());
+ MutableBoolean bActiveTxn = new MutableBoolean(true);
+ metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
+ MetadataLockManager.INSTANCE.dropDatasetBegin(dataverseName, dataverseName + "." + datasetName);
+ List<JobSpecification> jobsToExecute = new ArrayList<>();
+ try {
+ Dataset ds = MetadataManager.INSTANCE.getDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ if (ds == null) {
+ if (stmtDelete.getIfExists()) {
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
+ return;
+ } else {
+ throw new AlgebricksException("There is no dataset with this name " + datasetName
+ + " in dataverse " + dataverseName + ".");
+ }
+ }
+
+ doDropDataset(ds, datasetName, metadataProvider, mdTxnCtx, jobsToExecute, dataverseName, bActiveTxn,
+ progress, hcc);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
+ } catch (Exception e) {
+ if (bActiveTxn.booleanValue()) {
+ abort(e, e, mdTxnCtx.getValue());
+ }
+
+ if (progress.getValue() == ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA) {
+ // #. execute compensation operations
+ // remove the all indexes in NC
+ try {
+ for (JobSpecification jobSpec : jobsToExecute) {
+ JobUtils.runJob(hcc, jobSpec, true);
+ }
+ } catch (Exception e2) {
+ // do no throw exception since still the metadata needs to be compensated.
+ e.addSuppressed(e2);
+ }
+
+ // remove the record from the metadata.
+ mdTxnCtx.setValue(MetadataManager.INSTANCE.beginTransaction());
+ metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
+ try {
+ MetadataManager.INSTANCE.dropDataset(metadataProvider.getMetadataTxnContext(), dataverseName,
+ datasetName);
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
+ } catch (Exception e2) {
+ e.addSuppressed(e2);
+ abort(e, e2, mdTxnCtx.getValue());
+ throw new IllegalStateException("System is inconsistent state: pending dataset(" + dataverseName
+ + "." + datasetName + ") couldn't be removed from the metadata", e);
+ }
+ }
+
+ throw e;
+ } finally {
+ MetadataLockManager.INSTANCE.dropDatasetEnd(dataverseName, dataverseName + "." + datasetName);
+ }
+ }
+
+ protected void doDropDataset(Dataset ds, String datasetName, AqlMetadataProvider metadataProvider,
+ MutableObject<MetadataTransactionContext> mdTxnCtx, List<JobSpecification> jobsToExecute,
+ String dataverseName, MutableBoolean bActiveTxn, MutableObject<ProgressState> progress,
+ IHyracksClientConnection hcc) throws Exception {
+ Map<FeedConnectionId, Pair<JobSpecification, Boolean>> disconnectJobList = new HashMap<>();
+ if (ds.getDatasetType() == DatasetType.INTERNAL) {
+ // prepare job spec(s) that would disconnect any active feeds involving the dataset.
+ IActiveEntityEventsListener[] feedConnections = ActiveJobNotificationHandler.INSTANCE.getEventListeners();
+ for (IActiveEntityEventsListener conn : feedConnections) {
+ if (conn.getEntityId().getExtensionName().equals(Feed.EXTENSION_NAME)
+ && ((FeedEventsListener) conn).isConnectedToDataset(datasetName)) {
+ FeedConnectionId connectionId = new FeedConnectionId(conn.getEntityId(), datasetName);
+ Pair<JobSpecification, Boolean> p = FeedOperations.buildDisconnectFeedJobSpec(metadataProvider,
+ connectionId);
+ disconnectJobList.put(connectionId, p);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disconnecting feed " + connectionId.getFeedId().getEntityName() + " from dataset "
+ + datasetName + " as dataset is being dropped");
+ }
+ // prepare job to remove feed log storage
+ jobsToExecute.add(FeedOperations.buildRemoveFeedStorageJob(MetadataManager.INSTANCE.getFeed(
+ mdTxnCtx.getValue(), connectionId.getFeedId().getDataverse(),
+ connectionId.getFeedId().getEntityName())));
+ }
+ }
+
+ // #. prepare jobs to drop the datatset and the indexes in NC
+ List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
+ datasetName);
+ for (int j = 0; j < indexes.size(); j++) {
+ if (indexes.get(j).isSecondaryIndex()) {
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+ indexes.get(j).getIndexName());
+ jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
+ }
+ }
+ CompiledDatasetDropStatement cds = new CompiledDatasetDropStatement(dataverseName, datasetName);
+ jobsToExecute.add(DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider));
+
+ // #. mark the existing dataset as PendingDropOp
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
+ new Dataset(dataverseName, datasetName, ds.getItemTypeDataverseName(), ds.getItemTypeName(),
+ ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName(), ds.getNodeGroupName(),
+ ds.getCompactionPolicy(), ds.getCompactionPolicyProperties(), ds.getDatasetDetails(),
+ ds.getHints(), ds.getDatasetType(), ds.getDatasetId(), IMetadataEntity.PENDING_DROP_OP));
+
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
+ bActiveTxn.setValue(false);
+ progress.setValue(ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA);
+
+ // # disconnect the feeds
+ for (Pair<JobSpecification, Boolean> p : disconnectJobList.values()) {
+ JobUtils.runJob(hcc, p.first, true);
+ }
+
+ // #. run the jobs
+ for (JobSpecification jobSpec : jobsToExecute) {
+ JobUtils.runJob(hcc, jobSpec, true);
+ }
+
+ mdTxnCtx.setValue(MetadataManager.INSTANCE.beginTransaction());
+ bActiveTxn.setValue(true);
+ metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
+ } else {
+ // External dataset
+ ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
+ // #. prepare jobs to drop the datatset and the indexes in NC
+ List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx.getValue(), dataverseName,
+ datasetName);
+ for (int j = 0; j < indexes.size(); j++) {
+ if (ExternalIndexingOperations.isFileIndex(indexes.get(j))) {
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+ indexes.get(j).getIndexName());
+ jobsToExecute.add(IndexOperations.buildDropSecondaryIndexJobSpec(cds, metadataProvider, ds));
+ } else {
+ CompiledIndexDropStatement cds = new CompiledIndexDropStatement(dataverseName, datasetName,
+ indexes.get(j).getIndexName());
+ jobsToExecute.add(
+ ExternalIndexingOperations.buildDropFilesIndexJobSpec(cds, metadataProvider, ds));
+ }
+ }
+
+ // #. mark the existing dataset as PendingDropOp
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ MetadataManager.INSTANCE.addDataset(mdTxnCtx.getValue(),
+ new Dataset(dataverseName, datasetName, ds.getItemTypeDataverseName(), ds.getItemTypeName(),
+ ds.getNodeGroupName(), ds.getCompactionPolicy(), ds.getCompactionPolicyProperties(),
+ ds.getDatasetDetails(), ds.getHints(), ds.getDatasetType(), ds.getDatasetId(),
+ IMetadataEntity.PENDING_DROP_OP));
+
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
+ bActiveTxn.setValue(false);
+ progress.setValue(ProgressState.ADDED_PENDINGOP_RECORD_TO_METADATA);
+
+ // #. run the jobs
+ for (JobSpecification jobSpec : jobsToExecute) {
+ JobUtils.runJob(hcc, jobSpec, true);
+ }
+ if (!indexes.isEmpty()) {
+ ExternalDatasetsRegistry.INSTANCE.removeDatasetInfo(ds);
+ }
+ mdTxnCtx.setValue(MetadataManager.INSTANCE.beginTransaction());
+ bActiveTxn.setValue(true);
+ metadataProvider.setMetadataTxnContext(mdTxnCtx.getValue());
+ }
+
+ // #. finally, delete the dataset.
+ MetadataManager.INSTANCE.dropDataset(mdTxnCtx.getValue(), dataverseName, datasetName);
+ // Drop the associated nodegroup
+ String nodegroup = ds.getNodeGroupName();
+ if (!nodegroup.equalsIgnoreCase(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)) {
+ MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx.getValue(), dataverseName + ":" + datasetName);
+ }
+ }
+
+ protected void handleIndexDropStatement(AqlMetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc) throws Exception {
+
+ IndexDropStatement stmtIndexDrop = (IndexDropStatement) stmt;
+ String datasetName = stmtIndexDrop.getDatasetName().getValue();
+ String dataverseName = getActiveDataverse(stmtIndexDrop.getDataverseName());
+ ProgressState progress = ProgressState.NO_PROGRESS;
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ boolean bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+
+ MetadataLockManager.INSTANCE.dropIndexBegin(dataverseName, dataverseName + "." + datasetName);
+
+ String indexName = null;
+ // For external index
+ boolean dropFilesIndex = false;
+ List<JobSpecification> jobsToExecute = new ArrayList<>();
+ try {
+
+ Dataset ds = MetadataManager.INSTANCE.getDataset(mdTxnCtx, dataverseName, datasetName);
+ if (ds == null) {
+ throw new AlgebricksException(
+ "There is no dataset with this name " + datasetName + " in dataverse " + dataverseName);
+ }
+ IActiveEntityEventsListener[] listeners = ActiveJobNotificationHandler.INSTANCE.getEventListeners();
+ StringBuilder builder = null;
+ for (IActiveEntityEventsListener listener : listeners) {
+ if (listener.getEntityId().getExtensionName().equals(Feed.EXTENSION_NAME)
+ && ((FeedEventsListener) listener).isConnectedToDataset(datasetName)) {
+ if (builder == null) {
+ builder = new StringBuilder();
+ }
+ builder.append(new FeedConnectionId(listener.getEntityId(), datasetName) + "\n");
+ }
+ }
+ if (builder != null) {
+ throw new AsterixException(
+ "Dataset" + datasetName + " is currently being fed into by the following feeds " + "."
+ + builder.toString() + "\nOperation not supported.");
+ }
+
+ if (ds.getDatasetType() == DatasetType.INTERNAL) {
+ indexName = stmtIndexDrop.getIndexName().getValue();
+ Index index = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, indexName);
+ if (index == null) {
+ if (stmtIndexDrop.getIfExists()) {
+ Metadat
<TRUNCATED>
[05/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index 242bbe3..0acc027 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -39,6 +39,7 @@ import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.metadata.entities.Node;
import org.apache.asterix.metadata.entities.NodeGroup;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
/**
* A metadata manager provides user access to Asterix metadata (e.g., types,
@@ -56,41 +57,46 @@ public interface IMetadataManager {
/**
* Initializes the metadata manager, e.g., finds the remote metadata node.
+ *
* @throws RemoteException
* If an error occurred while contacting the proxy for finding
* the metadata node.
*/
- public void init() throws RemoteException, MetadataException;
+ void init() throws RemoteException, MetadataException;
/**
* Begins a transaction on the metadata node.
+ *
* @return A globally unique transaction id.
* @throws ACIDException
* @throws RemoteException
*/
- public MetadataTransactionContext beginTransaction() throws ACIDException, RemoteException;
+ MetadataTransactionContext beginTransaction() throws ACIDException, RemoteException;
/**
* Commits a remote transaction on the metadata node.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @throws ACIDException
* @throws RemoteException
*/
- public void commitTransaction(MetadataTransactionContext ctx) throws ACIDException, RemoteException;
+ void commitTransaction(MetadataTransactionContext ctx) throws ACIDException, RemoteException;
/**
* Aborts a remote transaction running on the metadata node.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @throws ACIDException
* @throws RemoteException
*/
- public void abortTransaction(MetadataTransactionContext ctx) throws ACIDException, RemoteException;
+ void abortTransaction(MetadataTransactionContext ctx) throws ACIDException, RemoteException;
/**
* Locks the metadata in given mode. The lock acquisition is delegated to
* the metadata node. This method blocks until the lock can be acquired.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param lockMode
@@ -98,19 +104,21 @@ public interface IMetadataManager {
* @throws ACIDException
* @throws RemoteException
*/
- public void lock(MetadataTransactionContext ctx, byte lockMode) throws ACIDException, RemoteException;
+ void lock(MetadataTransactionContext ctx, byte lockMode) throws ACIDException, RemoteException;
/**
* Releases all locks on the metadata held by the given transaction id.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @throws ACIDException
* @throws RemoteException
*/
- public void unlock(MetadataTransactionContext ctx, byte lockMode) throws ACIDException, RemoteException;
+ void unlock(MetadataTransactionContext ctx, byte lockMode) throws ACIDException, RemoteException;
/**
* Inserts a new dataverse into the metadata.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverse
@@ -118,10 +126,11 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataverse already exists.
*/
- public void addDataverse(MetadataTransactionContext ctx, Dataverse dataverse) throws MetadataException;
+ void addDataverse(MetadataTransactionContext ctx, Dataverse dataverse) throws MetadataException;
/**
* Retrieves all dataverses
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @return A list of dataverse instances.
@@ -131,6 +140,7 @@ public interface IMetadataManager {
/**
* Retrieves a dataverse with given name.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -139,10 +149,11 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataverse does not exist.
*/
- public Dataverse getDataverse(MetadataTransactionContext ctx, String dataverseName) throws MetadataException;
+ Dataverse getDataverse(MetadataTransactionContext ctx, String dataverseName) throws MetadataException;
/**
* Retrieves all datasets belonging to the given dataverse.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -151,22 +162,23 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataverse does not exist.
*/
- public List<Dataset> getDataverseDatasets(MetadataTransactionContext ctx, String dataverseName)
- throws MetadataException;
+ List<Dataset> getDataverseDatasets(MetadataTransactionContext ctx, String dataverseName) throws MetadataException;
/**
* Deletes the dataverse with given name, and all it's associated datasets,
* indexes, and types.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @return A list of dataset instances.
* @throws MetadataException
* For example, if the dataverse does not exist.
*/
- public void dropDataverse(MetadataTransactionContext ctx, String dataverseName) throws MetadataException;
+ void dropDataverse(MetadataTransactionContext ctx, String dataverseName) throws MetadataException;
/**
* Inserts a new dataset into the metadata.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataset
@@ -174,10 +186,11 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataset already exists.
*/
- public void addDataset(MetadataTransactionContext ctx, Dataset dataset) throws MetadataException;
+ void addDataset(MetadataTransactionContext ctx, Dataset dataset) throws MetadataException;
/**
* Retrieves a dataset within a given dataverse.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -188,11 +201,12 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataset does not exist.
*/
- public Dataset getDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName)
+ Dataset getDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName)
throws MetadataException;
/**
* Retrieves all indexes of a dataset.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -203,11 +217,12 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataset and/or dataverse does not exist.
*/
- public List<Index> getDatasetIndexes(MetadataTransactionContext ctx, String dataverseName, String datasetName)
+ List<Index> getDatasetIndexes(MetadataTransactionContext ctx, String dataverseName, String datasetName)
throws MetadataException;
/**
* Deletes the dataset with given name, and all it's associated indexes.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -217,12 +232,12 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataset and/or dataverse does not exist.
*/
- public void dropDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName)
- throws MetadataException;
+ void dropDataset(MetadataTransactionContext ctx, String dataverseName, String datasetName) throws MetadataException;
/**
* Inserts an index into the metadata. The index itself knows its name, and
* which dataset it belongs to.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param index
@@ -230,10 +245,11 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the index already exists.
*/
- public void addIndex(MetadataTransactionContext ctx, Index index) throws MetadataException;
+ void addIndex(MetadataTransactionContext ctx, Index index) throws MetadataException;
/**
* Retrieves the index with given name, in given dataverse and dataset.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -245,11 +261,12 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the index does not exist.
*/
- public Index getIndex(MetadataTransactionContext ctx, String dataverseName, String datasetName, String indexName)
+ Index getIndex(MetadataTransactionContext ctx, String dataverseName, String datasetName, String indexName)
throws MetadataException;
/**
* Deletes the index with given name, in given dataverse and dataset.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -260,11 +277,12 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the index does not exist.
*/
- public void dropIndex(MetadataTransactionContext ctx, String dataverseName, String datasetName, String indexName)
+ void dropIndex(MetadataTransactionContext ctx, String dataverseName, String datasetName, String indexName)
throws MetadataException;
/**
* Inserts a datatype.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param datatype
@@ -272,10 +290,11 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the datatype already exists.
*/
- public void addDatatype(MetadataTransactionContext ctx, Datatype datatype) throws MetadataException;
+ void addDatatype(MetadataTransactionContext ctx, Datatype datatype) throws MetadataException;
/**
* Retrieves the datatype with given name in given dataverse.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -286,11 +305,12 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the datatype does not exist.
*/
- public Datatype getDatatype(MetadataTransactionContext ctx, String dataverseName, String datatypeName)
+ Datatype getDatatype(MetadataTransactionContext ctx, String dataverseName, String datatypeName)
throws MetadataException;
/**
* Deletes the given datatype in given dataverse.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -301,11 +321,12 @@ public interface IMetadataManager {
* For example, if there are still datasets using the type to be
* deleted.
*/
- public void dropDatatype(MetadataTransactionContext ctx, String dataverseName, String datatypeName)
+ void dropDatatype(MetadataTransactionContext ctx, String dataverseName, String datatypeName)
throws MetadataException;
/**
* Inserts a node group.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param nodeGroup
@@ -313,10 +334,11 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the node group already exists.
*/
- public void addNodegroup(MetadataTransactionContext ctx, NodeGroup nodeGroup) throws MetadataException;
+ void addNodegroup(MetadataTransactionContext ctx, NodeGroup nodeGroup) throws MetadataException;
/**
* Retrieves a node group.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param nodeGroupName
@@ -324,10 +346,11 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the node group does not exist.
*/
- public NodeGroup getNodegroup(MetadataTransactionContext ctx, String nodeGroupName) throws MetadataException;
+ NodeGroup getNodegroup(MetadataTransactionContext ctx, String nodeGroupName) throws MetadataException;
/**
* Deletes a node group.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param nodeGroupName
@@ -336,10 +359,11 @@ public interface IMetadataManager {
* For example, there are still datasets partitioned on the node
* group to be deleted.
*/
- public void dropNodegroup(MetadataTransactionContext ctx, String nodeGroupName) throws MetadataException;
+ void dropNodegroup(MetadataTransactionContext ctx, String nodeGroupName) throws MetadataException;
/**
* Inserts a node (machine).
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param node
@@ -347,7 +371,7 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the node already exists.
*/
- public void addNode(MetadataTransactionContext ctx, Node node) throws MetadataException;
+ void addNode(MetadataTransactionContext ctx, Node node) throws MetadataException;
/**
* @param mdTxnCtx
@@ -357,7 +381,7 @@ public interface IMetadataManager {
* being added
* @throws MetadataException
*/
- public void addFunction(MetadataTransactionContext mdTxnCtx, Function function) throws MetadataException;
+ void addFunction(MetadataTransactionContext mdTxnCtx, Function function) throws MetadataException;
/**
* @param ctx
@@ -368,8 +392,7 @@ public interface IMetadataManager {
* @throws MetadataException
*/
- public Function getFunction(MetadataTransactionContext ctx, FunctionSignature functionSignature)
- throws MetadataException;
+ Function getFunction(MetadataTransactionContext ctx, FunctionSignature functionSignature) throws MetadataException;
/**
* @param ctx
@@ -378,8 +401,7 @@ public interface IMetadataManager {
* the functions signature (unique to the function)
* @throws MetadataException
*/
- public void dropFunction(MetadataTransactionContext ctx, FunctionSignature functionSignature)
- throws MetadataException;
+ void dropFunction(MetadataTransactionContext ctx, FunctionSignature functionSignature) throws MetadataException;
/**
* @param mdTxnCtx
@@ -389,7 +411,7 @@ public interface IMetadataManager {
* added
* @throws MetadataException
*/
- public void addAdapter(MetadataTransactionContext mdTxnCtx, DatasourceAdapter adapter) throws MetadataException;
+ void addAdapter(MetadataTransactionContext mdTxnCtx, DatasourceAdapter adapter) throws MetadataException;
/**
* @param ctx
@@ -401,7 +423,7 @@ public interface IMetadataManager {
* @return
* @throws MetadataException
*/
- public DatasourceAdapter getAdapter(MetadataTransactionContext ctx, String dataverseName, String name)
+ DatasourceAdapter getAdapter(MetadataTransactionContext ctx, String dataverseName, String name)
throws MetadataException;
/**
@@ -413,14 +435,14 @@ public interface IMetadataManager {
* name of the adapter
* @throws MetadataException
*/
- public void dropAdapter(MetadataTransactionContext ctx, String dataverseName, String name) throws MetadataException;
+ void dropAdapter(MetadataTransactionContext ctx, String dataverseName, String name) throws MetadataException;
/**
* @param ctx
* @param policy
* @throws MetadataException
*/
- public void addCompactionPolicy(MetadataTransactionContext ctx, CompactionPolicy policy) throws MetadataException;
+ void addCompactionPolicy(MetadataTransactionContext ctx, CompactionPolicy policy) throws MetadataException;
/**
* @param ctx
@@ -429,7 +451,7 @@ public interface IMetadataManager {
* @return
* @throws MetadataException
*/
- public CompactionPolicy getCompactionPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
+ CompactionPolicy getCompactionPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
throws MetadataException;
/**
@@ -438,15 +460,14 @@ public interface IMetadataManager {
* @return
* @throws MetadataException
*/
- public List<Function> getDataverseFunctions(MetadataTransactionContext ctx, String dataverseName)
- throws MetadataException;
+ List<Function> getDataverseFunctions(MetadataTransactionContext ctx, String dataverseName) throws MetadataException;
/**
* @param ctx
* @param feed
* @throws MetadataException
*/
- public void addFeed(MetadataTransactionContext ctx, Feed feed) throws MetadataException;
+ void addFeed(MetadataTransactionContext ctx, Feed feed) throws MetadataException;
/**
* @param ctx
@@ -455,7 +476,7 @@ public interface IMetadataManager {
* @return
* @throws MetadataException
*/
- public Feed getFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws MetadataException;
+ Feed getFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws MetadataException;
/**
* @param ctx
@@ -463,14 +484,14 @@ public interface IMetadataManager {
* @param feedName
* @throws MetadataException
*/
- public void dropFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws MetadataException;
+ void dropFeed(MetadataTransactionContext ctx, String dataverse, String feedName) throws MetadataException;
/**
* @param ctx
* @param policy
* @throws MetadataException
*/
- public void addFeedPolicy(MetadataTransactionContext ctx, FeedPolicyEntity policy) throws MetadataException;
+ void addFeedPolicy(MetadataTransactionContext ctx, FeedPolicyEntity policy) throws MetadataException;
/**
* @param ctx
@@ -479,24 +500,25 @@ public interface IMetadataManager {
* @return
* @throws MetadataException
*/
- public FeedPolicyEntity getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
+ FeedPolicyEntity getFeedPolicy(MetadataTransactionContext ctx, String dataverse, String policyName)
throws MetadataException;
- public void initializeDatasetIdFactory(MetadataTransactionContext ctx) throws MetadataException;
+ void initializeDatasetIdFactory(MetadataTransactionContext ctx) throws MetadataException;
- public int getMostRecentDatasetId() throws MetadataException;
+ int getMostRecentDatasetId() throws MetadataException;
- public void acquireWriteLatch();
+ void acquireWriteLatch();
- public void releaseWriteLatch();
+ void releaseWriteLatch();
- public void acquireReadLatch();
+ void acquireReadLatch();
- public void releaseReadLatch();
+ void releaseReadLatch();
/**
* Removes a library , acquiring local locks on behalf of the given
* transaction id.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -506,12 +528,12 @@ public interface IMetadataManager {
* if the library does not exists.
* @throws RemoteException
*/
- public void dropLibrary(MetadataTransactionContext ctx, String dataverseName, String libraryName)
- throws MetadataException;
+ void dropLibrary(MetadataTransactionContext ctx, String dataverseName, String libraryName) throws MetadataException;
/**
* Adds a library, acquiring local locks on behalf of the given
* transaction id.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param library
@@ -520,7 +542,7 @@ public interface IMetadataManager {
* for example, if the library is already added.
* @throws RemoteException
*/
- public void addLibrary(MetadataTransactionContext ctx, Library library) throws MetadataException;
+ void addLibrary(MetadataTransactionContext ctx, Library library) throws MetadataException;
/**
* @param ctx
@@ -533,11 +555,12 @@ public interface IMetadataManager {
* @throws MetadataException
* @throws RemoteException
*/
- public Library getLibrary(MetadataTransactionContext ctx, String dataverseName, String libraryName)
+ Library getLibrary(MetadataTransactionContext ctx, String dataverseName, String libraryName)
throws MetadataException, RemoteException;
/**
* Retireve libraries installed in a given dataverse.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataverseName
@@ -546,8 +569,7 @@ public interface IMetadataManager {
* @throws MetadataException
* @throws RemoteException
*/
- public List<Library> getDataverseLibraries(MetadataTransactionContext ctx, String dataverseName)
- throws MetadataException;
+ List<Library> getDataverseLibraries(MetadataTransactionContext ctx, String dataverseName) throws MetadataException;
/**
* @param mdTxnCtx
@@ -557,8 +579,7 @@ public interface IMetadataManager {
* added
* @throws MetadataException
*/
- public void addExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile)
- throws MetadataException;
+ void addExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile) throws MetadataException;
/**
* @param mdTxnCtx
@@ -568,7 +589,7 @@ public interface IMetadataManager {
* @return A list of external files belonging to the dataset
* @throws MetadataException
*/
- public List<ExternalFile> getDatasetExternalFiles(MetadataTransactionContext mdTxnCtx, Dataset dataset)
+ List<ExternalFile> getDatasetExternalFiles(MetadataTransactionContext mdTxnCtx, Dataset dataset)
throws MetadataException;
/**
@@ -579,8 +600,7 @@ public interface IMetadataManager {
* dropped
* @throws MetadataException
*/
- public void dropExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile)
- throws MetadataException;
+ void dropExternalFile(MetadataTransactionContext mdTxnCtx, ExternalFile externalFile) throws MetadataException;
/**
* @param mdTxnCtx
@@ -590,10 +610,11 @@ public interface IMetadataManager {
* dropped
* @throws MetadataException
*/
- public void dropDatasetExternalFiles(MetadataTransactionContext mdTxnCtx, Dataset dataset) throws MetadataException;
+ void dropDatasetExternalFiles(MetadataTransactionContext mdTxnCtx, Dataset dataset) throws MetadataException;
/**
* Get en external file
+ *
* @param mdTxnCtx
* @param dataverseName
* @param datasetName
@@ -601,11 +622,12 @@ public interface IMetadataManager {
* @return
* @throws MetadataException
*/
- public ExternalFile getExternalFile(MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName,
+ ExternalFile getExternalFile(MetadataTransactionContext mdTxnCtx, String dataverseName, String datasetName,
Integer fileNumber) throws MetadataException;
/**
* update an existing dataset in metadata.
+ *
* @param ctx
* MetadataTransactionContext of an active metadata transaction.
* @param dataset
@@ -613,12 +635,45 @@ public interface IMetadataManager {
* @throws MetadataException
* For example, if the dataset already exists.
*/
- public void updateDataset(MetadataTransactionContext ctx, Dataset dataset) throws MetadataException;
+ void updateDataset(MetadataTransactionContext ctx, Dataset dataset) throws MetadataException;
/**
* Clean up temporary datasets that have not been active for a long time.
+ *
+ * @throws MetadataException
+ */
+ void cleanupTempDatasets() throws MetadataException;
+
+ /**
+ * Add an extension entity to its extension dataset under the ongoing metadata transaction
+ *
+ * @param mdTxnCtx
+ * @param entity
+ * @throws MetadataException
+ */
+ <T extends IExtensionMetadataEntity> void addEntity(MetadataTransactionContext mdTxnCtx, T entity)
+ throws MetadataException;
+
+ /**
+ * Deletes an extension entity from its extension dataset under the ongoing metadata transaction
+ *
+ * @param mdTxnCtx
+ * @param entity
+ * @throws MetadataException
+ */
+ <T extends IExtensionMetadataEntity> void deleteEntity(MetadataTransactionContext mdTxnCtx, T entity)
+ throws MetadataException;
+
+ /**
+ * Gets a list of extension entities matching a search key under the ongoing metadata transaction
+ *
+ * @param mdTxnCtx
+ * @param searchKey
+ * @return
* @throws MetadataException
+ * @throws HyracksDataException
*/
- public void cleanupTempDatasets() throws MetadataException;
+ <T extends IExtensionMetadataEntity> List<T> getEntities(MetadataTransactionContext mdTxnCtx,
+ IExtensionMetadataSearchKey searchKey) throws MetadataException, HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index ec2e692..41d0b6a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -59,7 +59,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws ACIDException
* @throws RemoteException
*/
- public void beginTransaction(JobId jobId) throws ACIDException, RemoteException;
+ void beginTransaction(JobId jobId) throws ACIDException, RemoteException;
/**
* Commits a local transaction against the metadata.
@@ -67,7 +67,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws ACIDException
* @throws RemoteException
*/
- public void commitTransaction(JobId jobId) throws ACIDException, RemoteException;
+ void commitTransaction(JobId jobId) throws ACIDException, RemoteException;
/**
* Aborts a local transaction against the metadata.
@@ -75,7 +75,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws ACIDException
* @throws RemoteException
*/
- public void abortTransaction(JobId jobId) throws ACIDException, RemoteException;
+ void abortTransaction(JobId jobId) throws ACIDException, RemoteException;
/**
* Locally locks the entire metadata in given mode on behalf of given
@@ -84,7 +84,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws ACIDException
* @throws RemoteException
*/
- public void lock(JobId jobId, byte lockMode) throws ACIDException, RemoteException;
+ void lock(JobId jobId, byte lockMode) throws ACIDException, RemoteException;
/**
* Releases all local locks of given transaction id.
@@ -92,7 +92,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws ACIDException
* @throws RemoteException
*/
- public void unlock(JobId jobId, byte lockMode) throws ACIDException, RemoteException;
+ void unlock(JobId jobId, byte lockMode) throws ACIDException, RemoteException;
/**
* Inserts a new dataverse into the metadata, acquiring local locks on
@@ -106,7 +106,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataverse already exists.
* @throws RemoteException
*/
- public void addDataverse(JobId jobId, Dataverse dataverse) throws MetadataException, RemoteException;
+ void addDataverse(JobId jobId, Dataverse dataverse) throws MetadataException, RemoteException;
/**
* Retrieves all dataverses, acquiring local locks on behalf of
@@ -119,7 +119,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataverse does not exist.
* @throws RemoteException
*/
- public List<Dataverse> getDataverses(JobId jobId) throws MetadataException, RemoteException;
+ List<Dataverse> getDataverses(JobId jobId) throws MetadataException, RemoteException;
/**
* Retrieves a dataverse with given name, acquiring local locks on behalf of
@@ -134,7 +134,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataverse does not exist.
* @throws RemoteException
*/
- public Dataverse getDataverse(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
+ Dataverse getDataverse(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
/**
* Retrieves all datasets belonging to the given dataverse, acquiring local
@@ -148,8 +148,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* For example, if the dataverse does not exist. RemoteException
*/
- public List<Dataset> getDataverseDatasets(JobId jobId, String dataverseName)
- throws MetadataException, RemoteException;
+ List<Dataset> getDataverseDatasets(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
/**
* Deletes the dataverse with given name, and all it's associated datasets,
@@ -163,7 +162,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataverse does not exist.
* @throws RemoteException
*/
- public void dropDataverse(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
+ void dropDataverse(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
/**
* Inserts a new dataset into the metadata, acquiring local locks on behalf
@@ -177,7 +176,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataset already exists.
* @throws RemoteException
*/
- public void addDataset(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
+ void addDataset(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
/**
* Retrieves a dataset within a given dataverse, acquiring local locks on
@@ -194,8 +193,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataset does not exist.
* @throws RemoteException
*/
- public Dataset getDataset(JobId jobId, String dataverseName, String datasetName)
- throws MetadataException, RemoteException;
+ Dataset getDataset(JobId jobId, String dataverseName, String datasetName) throws MetadataException, RemoteException;
/**
* Retrieves all indexes of a dataset, acquiring local locks on behalf of
@@ -212,7 +210,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataset and/or dataverse does not exist.
* @throws RemoteException
*/
- public List<Index> getDatasetIndexes(JobId jobId, String dataverseName, String datasetName)
+ List<Index> getDatasetIndexes(JobId jobId, String dataverseName, String datasetName)
throws MetadataException, RemoteException;
/**
@@ -229,8 +227,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataset and/or dataverse does not exist.
* @throws RemoteException
*/
- public void dropDataset(JobId jobId, String dataverseName, String datasetName)
- throws MetadataException, RemoteException;
+ void dropDataset(JobId jobId, String dataverseName, String datasetName) throws MetadataException, RemoteException;
/**
* Inserts an index into the metadata, acquiring local locks on behalf of
@@ -245,7 +242,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the index already exists.
* @throws RemoteException
*/
- public void addIndex(JobId jobId, Index index) throws MetadataException, RemoteException;
+ void addIndex(JobId jobId, Index index) throws MetadataException, RemoteException;
/**
* Retrieves the index with given name, in given dataverse and dataset,
@@ -263,7 +260,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the index does not exist.
* @throws RemoteException
*/
- public Index getIndex(JobId jobId, String dataverseName, String datasetName, String indexName)
+ Index getIndex(JobId jobId, String dataverseName, String datasetName, String indexName)
throws MetadataException, RemoteException;
/**
@@ -281,7 +278,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the index does not exist.
* @throws RemoteException
*/
- public void dropIndex(JobId jobId, String dataverseName, String datasetName, String indexName)
+ void dropIndex(JobId jobId, String dataverseName, String datasetName, String indexName)
throws MetadataException, RemoteException;
/**
@@ -296,7 +293,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the datatype already exists.
* @throws RemoteException
*/
- public void addDatatype(JobId jobId, Datatype datatype) throws MetadataException, RemoteException;
+ void addDatatype(JobId jobId, Datatype datatype) throws MetadataException, RemoteException;
/**
* Retrieves the datatype with given name in given dataverse, acquiring
@@ -313,7 +310,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the datatype does not exist.
* @throws RemoteException
*/
- public Datatype getDatatype(JobId jobId, String dataverseName, String datatypeName)
+ Datatype getDatatype(JobId jobId, String dataverseName, String datatypeName)
throws MetadataException, RemoteException;
/**
@@ -331,8 +328,7 @@ public interface IMetadataNode extends Remote, Serializable {
* deleted.
* @throws RemoteException
*/
- public void dropDatatype(JobId jobId, String dataverseName, String datatypeName)
- throws MetadataException, RemoteException;
+ void dropDatatype(JobId jobId, String dataverseName, String datatypeName) throws MetadataException, RemoteException;
/**
* Inserts a node group, acquiring local locks on behalf of the given
@@ -346,7 +342,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the node group already exists.
* @throws RemoteException
*/
- public void addNodeGroup(JobId jobId, NodeGroup nodeGroup) throws MetadataException, RemoteException;
+ void addNodeGroup(JobId jobId, NodeGroup nodeGroup) throws MetadataException, RemoteException;
/**
* Retrieves a node group, acquiring local locks on behalf of the given
@@ -360,7 +356,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the node group does not exist.
* @throws RemoteException
*/
- public NodeGroup getNodeGroup(JobId jobId, String nodeGroupName) throws MetadataException, RemoteException;
+ NodeGroup getNodeGroup(JobId jobId, String nodeGroupName) throws MetadataException, RemoteException;
/**
* Deletes a node group, acquiring local locks on behalf of the given
@@ -375,7 +371,7 @@ public interface IMetadataNode extends Remote, Serializable {
* group to be deleted.
* @throws RemoteException
*/
- public void dropNodegroup(JobId jobId, String nodeGroupName) throws MetadataException, RemoteException;
+ void dropNodegroup(JobId jobId, String nodeGroupName) throws MetadataException, RemoteException;
/**
* Inserts a node (compute node), acquiring local locks on behalf of the
@@ -389,7 +385,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the node already exists.
* @throws RemoteException
*/
- public void addNode(JobId jobId, Node node) throws MetadataException, RemoteException;
+ void addNode(JobId jobId, Node node) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -400,8 +396,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public Function getFunction(JobId jobId, FunctionSignature functionSignature)
- throws MetadataException, RemoteException;
+ Function getFunction(JobId jobId, FunctionSignature functionSignature) throws MetadataException, RemoteException;
/**
* Deletes a function, acquiring local locks on behalf of the given
@@ -416,8 +411,7 @@ public interface IMetadataNode extends Remote, Serializable {
* group to be deleted.
* @throws RemoteException
*/
- public void dropFunction(JobId jobId, FunctionSignature functionSignature)
- throws MetadataException, RemoteException;
+ void dropFunction(JobId jobId, FunctionSignature functionSignature) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -429,7 +423,7 @@ public interface IMetadataNode extends Remote, Serializable {
* unknown function
* @throws RemoteException
*/
- public void addFunction(JobId jobId, Function function) throws MetadataException, RemoteException;
+ void addFunction(JobId jobId, Function function) throws MetadataException, RemoteException;
/**
* @param ctx
@@ -438,8 +432,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public List<Function> getDataverseFunctions(JobId jobId, String dataverseName)
- throws MetadataException, RemoteException;
+ List<Function> getDataverseFunctions(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
/**
* @param ctx
@@ -448,7 +441,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public List<DatasourceAdapter> getDataverseAdapters(JobId jobId, String dataverseName)
+ List<DatasourceAdapter> getDataverseAdapters(JobId jobId, String dataverseName)
throws MetadataException, RemoteException;
/**
@@ -459,7 +452,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public DatasourceAdapter getAdapter(JobId jobId, String dataverseName, String adapterName)
+ DatasourceAdapter getAdapter(JobId jobId, String dataverseName, String adapterName)
throws MetadataException, RemoteException;
/**
@@ -475,8 +468,7 @@ public interface IMetadataNode extends Remote, Serializable {
* if the adapter does not exists.
* @throws RemoteException
*/
- public void dropAdapter(JobId jobId, String dataverseName, String adapterName)
- throws MetadataException, RemoteException;
+ void dropAdapter(JobId jobId, String dataverseName, String adapterName) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -487,7 +479,7 @@ public interface IMetadataNode extends Remote, Serializable {
* for example, if the adapter already exists.
* @throws RemoteException
*/
- public void addAdapter(JobId jobId, DatasourceAdapter adapter) throws MetadataException, RemoteException;
+ void addAdapter(JobId jobId, DatasourceAdapter adapter) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -495,8 +487,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public void addCompactionPolicy(JobId jobId, CompactionPolicy compactionPolicy)
- throws MetadataException, RemoteException;
+ void addCompactionPolicy(JobId jobId, CompactionPolicy compactionPolicy) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -506,7 +497,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public CompactionPolicy getCompactionPolicy(JobId jobId, String dataverse, String policy)
+ CompactionPolicy getCompactionPolicy(JobId jobId, String dataverse, String policy)
throws MetadataException, RemoteException;
/**
@@ -514,14 +505,14 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public void initializeDatasetIdFactory(JobId jobId) throws MetadataException, RemoteException;
+ void initializeDatasetIdFactory(JobId jobId) throws MetadataException, RemoteException;
/**
* @return
* @throws MetadataException
* @throws RemoteException
*/
- public int getMostRecentDatasetId() throws MetadataException, RemoteException;
+ int getMostRecentDatasetId() throws MetadataException, RemoteException;
/**
* @param jobId
@@ -529,7 +520,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public void addFeed(JobId jobId, Feed feed) throws MetadataException, RemoteException;
+ void addFeed(JobId jobId, Feed feed) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -539,7 +530,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public Feed getFeed(JobId jobId, String dataverse, String feedName) throws MetadataException, RemoteException;
+ Feed getFeed(JobId jobId, String dataverse, String feedName) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -548,7 +539,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public void dropFeed(JobId jobId, String dataverse, String feedName) throws MetadataException, RemoteException;
+ void dropFeed(JobId jobId, String dataverse, String feedName) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -556,7 +547,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public void addFeedPolicy(JobId jobId, FeedPolicyEntity feedPolicy) throws MetadataException, RemoteException;
+ void addFeedPolicy(JobId jobId, FeedPolicyEntity feedPolicy) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -566,7 +557,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public FeedPolicyEntity getFeedPolicy(JobId jobId, String dataverse, String policy)
+ FeedPolicyEntity getFeedPolicy(JobId jobId, String dataverse, String policy)
throws MetadataException, RemoteException;
/**
@@ -582,8 +573,7 @@ public interface IMetadataNode extends Remote, Serializable {
* if the library does not exists.
* @throws RemoteException
*/
- public void dropLibrary(JobId jobId, String dataverseName, String libraryName)
- throws MetadataException, RemoteException;
+ void dropLibrary(JobId jobId, String dataverseName, String libraryName) throws MetadataException, RemoteException;
/**
* Adds a library, acquiring local locks on behalf of the given
@@ -597,7 +587,7 @@ public interface IMetadataNode extends Remote, Serializable {
* for example, if the library is already added.
* @throws RemoteException
*/
- public void addLibrary(JobId jobId, Library library) throws MetadataException, RemoteException;
+ void addLibrary(JobId jobId, Library library) throws MetadataException, RemoteException;
/**
* @param txnId
@@ -610,8 +600,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public Library getLibrary(JobId jobId, String dataverseName, String libraryName)
- throws MetadataException, RemoteException;
+ Library getLibrary(JobId jobId, String dataverseName, String libraryName) throws MetadataException, RemoteException;
/**
* Retireve libraries installed in a given dataverse.
@@ -624,8 +613,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public List<Library> getDataverseLibraries(JobId jobId, String dataverseName)
- throws MetadataException, RemoteException;
+ List<Library> getDataverseLibraries(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -634,7 +622,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public List<Feed> getDataverseFeeds(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
+ List<Feed> getDataverseFeeds(JobId jobId, String dataverseName) throws MetadataException, RemoteException;
/**
* delete a give feed (ingestion) policy
@@ -646,8 +634,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws RemoteException
* @throws MetadataException
*/
- public void dropFeedPolicy(JobId jobId, String dataverseName, String policyName)
- throws MetadataException, RemoteException;
+ void dropFeedPolicy(JobId jobId, String dataverseName, String policyName) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -656,7 +643,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public List<FeedPolicyEntity> getDataversePolicies(JobId jobId, String dataverse)
+ List<FeedPolicyEntity> getDataversePolicies(JobId jobId, String dataverse)
throws MetadataException, RemoteException;
/**
@@ -668,7 +655,7 @@ public interface IMetadataNode extends Remote, Serializable {
* for example, if the file already exists.
* @throws RemoteException
*/
- public void addExternalFile(JobId jobId, ExternalFile externalFile) throws MetadataException, RemoteException;
+ void addExternalFile(JobId jobId, ExternalFile externalFile) throws MetadataException, RemoteException;
/**
* @param jobId
@@ -678,7 +665,7 @@ public interface IMetadataNode extends Remote, Serializable {
* @throws MetadataException
* @throws RemoteException
*/
- public List<ExternalFile> getExternalFiles(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
+ List<ExternalFile> getExternalFiles(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
/**
* Deletes an externalFile , acquiring local locks on behalf of the given
@@ -694,7 +681,7 @@ public interface IMetadataNode extends Remote, Serializable {
* the id number for the file to be deleted
* @throws RemoteException
*/
- public void dropExternalFile(JobId jobId, String dataverseName, String datasetName, int fileNumber)
+ void dropExternalFile(JobId jobId, String dataverseName, String datasetName, int fileNumber)
throws MetadataException, RemoteException;
/**
@@ -707,7 +694,7 @@ public interface IMetadataNode extends Remote, Serializable {
* An external dataset the files belong to.
* @throws RemoteException
*/
- public void dropExternalFiles(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
+ void dropExternalFiles(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
/**
* Retrieves the file with given number, in given dataverse and dataset,
@@ -726,7 +713,7 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the index does not exist.
* @throws RemoteException
*/
- public ExternalFile getExternalFile(JobId jobId, String dataverseName, String datasetName, Integer fileNumber)
+ ExternalFile getExternalFile(JobId jobId, String dataverseName, String datasetName, Integer fileNumber)
throws MetadataException, RemoteException;
/**
@@ -741,6 +728,40 @@ public interface IMetadataNode extends Remote, Serializable {
* For example, if the dataset already exists.
* @throws RemoteException
*/
- public void updateDataset(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
+ void updateDataset(JobId jobId, Dataset dataset) throws MetadataException, RemoteException;
+
+ /**
+ * Adds an extension entity under the ongoing transaction job id
+ *
+ * @param jobId
+ * @param entity
+ * @throws MetadataException
+ * @throws RemoteException
+ */
+ <T extends IExtensionMetadataEntity> void addEntity(JobId jobId, T entity)
+ throws MetadataException, RemoteException;
+
+ /**
+ * Deletes an extension entity under the ongoing transaction job id
+ *
+ * @param jobId
+ * @param entity
+ * @throws MetadataException
+ * @throws RemoteException
+ */
+ <T extends IExtensionMetadataEntity> void deleteEntity(JobId jobId, T entity)
+ throws MetadataException, RemoteException;
+
+ /**
+ * Gets a list of extension entities matching a search key under the ongoing transaction
+ *
+ * @param jobId
+ * @param searchKey
+ * @return
+ * @throws MetadataException
+ * @throws RemoteException
+ */
+ <T extends IExtensionMetadataEntity> List<T> getEntities(JobId jobId, IExtensionMetadataSearchKey searchKey)
+ throws MetadataException, RemoteException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 757694d..f5ccb9e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -35,7 +35,6 @@ import org.apache.asterix.common.config.AsterixMetadataProperties;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.config.IAsterixPropertiesProvider;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.context.BaseOperationTracker;
import org.apache.asterix.common.context.CorrelatedPrefixMergePolicyFactory;
import org.apache.asterix.common.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
@@ -65,6 +64,7 @@ import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningS
import org.apache.asterix.metadata.entities.Node;
import org.apache.asterix.metadata.entities.NodeGroup;
import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.util.AsterixClusterProperties;
@@ -75,6 +75,7 @@ import org.apache.asterix.transaction.management.service.transaction.Transaction
import org.apache.hyracks.api.application.INCApplicationContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrame;
@@ -103,47 +104,39 @@ import org.apache.hyracks.storage.common.file.LocalResource;
* stopUniverse() should be called upon application undeployment.
*/
public class MetadataBootstrap {
- private static final Logger LOGGER = Logger.getLogger(MetadataBootstrap.class.getName());
public static final boolean IS_DEBUG_MODE = false;// true
-
+ private static final Logger LOGGER = Logger.getLogger(MetadataBootstrap.class.getName());
private static IAsterixAppRuntimeContext runtimeContext;
-
private static IBufferCache bufferCache;
private static IFileMapProvider fileMapProvider;
private static IDatasetLifecycleManager dataLifecycleManager;
private static ILocalResourceRepository localResourceRepository;
private static IIOManager ioManager;
-
private static String metadataNodeName;
private static List<String> nodeNames;
private static String outputDir;
+ private static boolean isNewUniverse;
- private static IMetadataIndex[] primaryIndexes;
+ private static final IMetadataIndex[] PRIMARY_INDEXES =
+ new IMetadataIndex[] { MetadataPrimaryIndexes.DATAVERSE_DATASET, MetadataPrimaryIndexes.DATASET_DATASET,
+ MetadataPrimaryIndexes.DATATYPE_DATASET, MetadataPrimaryIndexes.INDEX_DATASET,
+ MetadataPrimaryIndexes.NODE_DATASET, MetadataPrimaryIndexes.NODEGROUP_DATASET,
+ MetadataPrimaryIndexes.FUNCTION_DATASET, MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET,
+ MetadataPrimaryIndexes.FEED_DATASET, MetadataPrimaryIndexes.FEED_POLICY_DATASET,
+ MetadataPrimaryIndexes.LIBRARY_DATASET, MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET,
+ MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET };
private static IAsterixPropertiesProvider propertiesProvider;
- private static void initLocalIndexArrays() {
- primaryIndexes = new IMetadataIndex[] { MetadataPrimaryIndexes.DATAVERSE_DATASET,
- MetadataPrimaryIndexes.DATASET_DATASET, MetadataPrimaryIndexes.DATATYPE_DATASET,
- MetadataPrimaryIndexes.INDEX_DATASET, MetadataPrimaryIndexes.NODE_DATASET,
- MetadataPrimaryIndexes.NODEGROUP_DATASET, MetadataPrimaryIndexes.FUNCTION_DATASET,
- MetadataPrimaryIndexes.DATASOURCE_ADAPTER_DATASET, MetadataPrimaryIndexes.FEED_DATASET,
- MetadataPrimaryIndexes.FEED_POLICY_DATASET, MetadataPrimaryIndexes.LIBRARY_DATASET,
- MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET, MetadataPrimaryIndexes.EXTERNAL_FILE_DATASET };
- }
-
public static void startUniverse(IAsterixPropertiesProvider asterixPropertiesProvider,
INCApplicationContext ncApplicationContext, boolean isNewUniverse) throws Exception {
+ MetadataBootstrap.setNewUniverse(isNewUniverse);
runtimeContext = (IAsterixAppRuntimeContext) ncApplicationContext.getApplicationObject();
propertiesProvider = asterixPropertiesProvider;
- MetadataPrimaryIndexes.init();
- initLocalIndexArrays();
-
AsterixMetadataProperties metadataProperties = propertiesProvider.getMetadataProperties();
metadataNodeName = metadataProperties.getMetadataNodeName();
nodeNames = metadataProperties.getNodeNames();
-
dataLifecycleManager = runtimeContext.getDatasetLifecycleManager();
localResourceRepository = runtimeContext.getLocalResourceRepository();
bufferCache = runtimeContext.getBufferCache();
@@ -156,37 +149,26 @@ public class MetadataBootstrap {
// Lock the metadata in X mode.
MetadataManager.INSTANCE.lock(mdTxnCtx, LockMode.X);
+ for (int i = 0; i < PRIMARY_INDEXES.length; i++) {
+ enlistMetadataDataset(PRIMARY_INDEXES[i]);
+ }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(
+ "Finished enlistment of metadata B-trees in " + (isNewUniverse ? "new" : "old") + " universe");
+ }
if (isNewUniverse) {
- for (int i = 0; i < primaryIndexes.length; i++) {
- enlistMetadataDataset(primaryIndexes[i], true, mdTxnCtx);
- }
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Finished enlistment of metadata B-trees in new universe");
- }
-
insertInitialDataverses(mdTxnCtx);
- insertInitialDatasets(mdTxnCtx);
- insertInitialDatatypes(mdTxnCtx);
+ insertMetadataDatasets(mdTxnCtx, PRIMARY_INDEXES);
+ insertMetadataDatatypes(mdTxnCtx);
insertNodes(mdTxnCtx);
insertInitialGroups(mdTxnCtx);
insertInitialAdapters(mdTxnCtx);
insertInitialFeedPolicies(mdTxnCtx);
insertInitialCompactionPolicies(mdTxnCtx);
-
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Finished creating metadata B-trees.");
}
- } else {
- for (int i = 0; i < primaryIndexes.length; i++) {
- enlistMetadataDataset(primaryIndexes[i], false, mdTxnCtx);
- }
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Finished enlistment of metadata B-trees in old universe.");
- }
}
-
// #. initialize datasetIdFactory
MetadataManager.INSTANCE.initializeDatasetIdFactory(mdTxnCtx);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -211,24 +193,26 @@ public class MetadataBootstrap {
// metadata datasets will be closed when the dataset life cycle manger is closed
}
- public static void insertInitialDataverses(MetadataTransactionContext mdTxnCtx) throws Exception {
+ private static void insertInitialDataverses(MetadataTransactionContext mdTxnCtx) throws Exception {
String dataverseName = MetadataPrimaryIndexes.DATAVERSE_DATASET.getDataverseName();
String dataFormat = NonTaggedDataFormat.NON_TAGGED_DATA_FORMAT;
MetadataManager.INSTANCE.addDataverse(mdTxnCtx,
new Dataverse(dataverseName, dataFormat, IMetadataEntity.PENDING_NO_OP));
}
- public static void insertInitialDatasets(MetadataTransactionContext mdTxnCtx) throws Exception {
- for (int i = 0; i < primaryIndexes.length; i++) {
+ public static void insertMetadataDatasets(MetadataTransactionContext mdTxnCtx, IMetadataIndex[] indexes)
+ throws Exception {
+ for (int i = 0; i < indexes.length; i++) {
IDatasetDetails id = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH,
- primaryIndexes[i].getPartitioningExpr(), primaryIndexes[i].getPartitioningExpr(), null,
- primaryIndexes[i].getPartitioningExprType(), false, null, false);
- MetadataManager.INSTANCE.addDataset(mdTxnCtx, new Dataset(primaryIndexes[i].getDataverseName(),
- primaryIndexes[i].getIndexedDatasetName(), primaryIndexes[i].getDataverseName(),
- primaryIndexes[i].getPayloadRecordType().getTypeName(), primaryIndexes[i].getNodeGroupName(),
- GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME, GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, id,
- new HashMap<String, String>(), DatasetType.INTERNAL, primaryIndexes[i].getDatasetId().getId(),
- IMetadataEntity.PENDING_NO_OP));
+ indexes[i].getPartitioningExpr(), indexes[i].getPartitioningExpr(), null,
+ indexes[i].getPartitioningExprType(), false, null, false);
+ MetadataManager.INSTANCE.addDataset(mdTxnCtx,
+ new Dataset(indexes[i].getDataverseName(), indexes[i].getIndexedDatasetName(),
+ indexes[i].getDataverseName(), indexes[i].getPayloadRecordType().getTypeName(),
+ indexes[i].getNodeGroupName(), GlobalConfig.DEFAULT_COMPACTION_POLICY_NAME,
+ GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, id, new HashMap<String, String>(),
+ DatasetType.INTERNAL, indexes[i].getDatasetId().getId(),
+ IMetadataEntity.PENDING_NO_OP));
}
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Finished inserting initial datasets.");
@@ -244,12 +228,12 @@ public class MetadataBootstrap {
}
public static void getMetadataTypes(ArrayList<IAType> types) throws Exception {
- for (int i = 0; i < primaryIndexes.length; i++) {
- types.add(primaryIndexes[i].getPayloadRecordType());
+ for (int i = 0; i < PRIMARY_INDEXES.length; i++) {
+ types.add(PRIMARY_INDEXES[i].getPayloadRecordType());
}
}
- public static void insertInitialDatatypes(MetadataTransactionContext mdTxnCtx) throws Exception {
+ public static void insertMetadataDatatypes(MetadataTransactionContext mdTxnCtx) throws Exception {
String dataverseName = MetadataPrimaryIndexes.DATAVERSE_DATASET.getDataverseName();
ArrayList<IAType> types = new ArrayList<IAType>();
getBuiltinTypes(types);
@@ -313,9 +297,9 @@ public class MetadataBootstrap {
}
private static void insertInitialCompactionPolicies(MetadataTransactionContext mdTxnCtx) throws Exception {
- String[] builtInCompactionPolicyClassNames = new String[] { ConstantMergePolicyFactory.class.getName(),
- PrefixMergePolicyFactory.class.getName(), NoMergePolicyFactory.class.getName(),
- CorrelatedPrefixMergePolicyFactory.class.getName() };
+ String[] builtInCompactionPolicyClassNames =
+ new String[] { ConstantMergePolicyFactory.class.getName(), PrefixMergePolicyFactory.class.getName(),
+ NoMergePolicyFactory.class.getName(), CorrelatedPrefixMergePolicyFactory.class.getName() };
CompactionPolicy compactionPolicy;
for (String policyClassName : builtInCompactionPolicyClassNames) {
compactionPolicy = getCompactionPolicyEntity(policyClassName);
@@ -330,13 +314,12 @@ public class MetadataBootstrap {
}
private static CompactionPolicy getCompactionPolicyEntity(String compactionPolicyClassName) throws Exception {
- String policyName = ((ILSMMergePolicyFactory) (Class.forName(compactionPolicyClassName).newInstance()))
- .getName();
+ String policyName =
+ ((ILSMMergePolicyFactory) (Class.forName(compactionPolicyClassName).newInstance())).getName();
return new CompactionPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, policyName, compactionPolicyClassName);
}
- private static void enlistMetadataDataset(IMetadataIndex index, boolean create, MetadataTransactionContext mdTxnCtx)
- throws Exception {
+ public static void enlistMetadataDataset(IMetadataIndex index) throws HyracksDataException {
ClusterPartition metadataPartition = propertiesProvider.getMetadataProperties().getMetadataPartition();
int metadataDeviceId = metadataPartition.getIODeviceNum();
String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath(
@@ -349,14 +332,14 @@ public class MetadataBootstrap {
ITypeTraits[] typeTraits = index.getTypeTraits();
IBinaryComparatorFactory[] comparatorFactories = index.getKeyBinaryComparatorFactory();
int[] bloomFilterKeyFields = index.getBloomFilterKeyFields();
- LSMBTree lsmBtree = null;
- long resourceID = -1;
- ILSMOperationTracker opTracker = index.isPrimaryIndex()
- ? runtimeContext.getLSMBTreeOperationTracker(index.getDatasetId().getId())
- : new BaseOperationTracker(index.getDatasetId().getId(),
- dataLifecycleManager.getDatasetInfo(index.getDatasetId().getId()));
+ LSMBTree lsmBtree;
+ long resourceID;
+ ILSMOperationTracker opTracker =
+ index.isPrimaryIndex() ? runtimeContext.getLSMBTreeOperationTracker(index.getDatasetId().getId())
+ : new BaseOperationTracker(index.getDatasetId().getId(),
+ dataLifecycleManager.getDatasetInfo(index.getDatasetId().getId()));
final String absolutePath = file.getFile().getPath();
- if (create) {
+ if (isNewUniverse()) {
lsmBtree = LSMBTreeUtils.createLSMTree(virtualBufferCaches, file, bufferCache, fileMapProvider, typeTraits,
comparatorFactories, bloomFilterKeyFields, runtimeContext.getBloomFilterFalsePositiveRate(),
runtimeContext.getMetadataMergePolicyFactory()
@@ -370,8 +353,8 @@ public class MetadataBootstrap {
comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(), index.getDatasetId().getId(),
runtimeContext.getMetadataMergePolicyFactory(), GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES,
null, null, null, null);
- ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
- localResourceMetadata, LocalResource.LSMBTreeResource);
+ ILocalResourceFactoryProvider localResourceFactoryProvider =
+ new PersistentLocalResourceFactoryProvider(localResourceMetadata, LocalResource.LSMBTreeResource);
ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory();
localResourceRepository.insert(localResourceFactory.createLocalResource(resourceID, resourceName,
metadataPartition.getPartitionId(), LIFOMetaDataFrame.VERSION, absolutePath));
@@ -379,7 +362,7 @@ public class MetadataBootstrap {
} else {
final LocalResource resource = localResourceRepository.getResourceByPath(absolutePath);
if (resource == null) {
- throw new Exception("Could not find required metadata indexes. Please delete "
+ throw new HyracksDataException("Could not find required metadata indexes. Please delete "
+ propertiesProvider.getMetadataProperties().getTransactionLogDirs()
.get(runtimeContext.getTransactionSubsystem().getId())
+ " to intialize as a new instance. (WARNING: all data will be lost.)");
@@ -398,7 +381,6 @@ public class MetadataBootstrap {
dataLifecycleManager.register(absolutePath, lsmBtree);
}
}
-
index.setResourceID(resourceID);
index.setFile(file);
}
@@ -418,9 +400,7 @@ public class MetadataBootstrap {
String datasetName = null;
String indexName = null;
MetadataTransactionContext mdTxnCtx = null;
-
MetadataManager.INSTANCE.acquireWriteLatch();
-
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Starting DDL recovery ...");
}
@@ -448,8 +428,8 @@ public class MetadataBootstrap {
LOGGER.info("Dropped a pending dataset: " + dataverseName + "." + datasetName);
}
} else {
- List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
- datasetName);
+ List<Index> indexes =
+ MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
for (Index index : indexes) {
indexName = index.getIndexName();
if (index.getPendingOp() != IMetadataEntity.PENDING_NO_OP) {
@@ -464,11 +444,11 @@ public class MetadataBootstrap {
}
if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
// if the dataset has no indexes, delete all its files
- List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
- datasetName);
+ List<Index> indexes =
+ MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
if (indexes.size() == 0) {
- List<ExternalFile> files = MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx,
- dataset);
+ List<ExternalFile> files =
+ MetadataManager.INSTANCE.getDatasetExternalFiles(mdTxnCtx, dataset);
for (ExternalFile file : files) {
MetadataManager.INSTANCE.dropExternalFile(mdTxnCtx, file);
if (LOGGER.isLoggable(Level.INFO)) {
@@ -500,4 +480,12 @@ public class MetadataBootstrap {
MetadataManager.INSTANCE.releaseWriteLatch();
}
}
+
+ public static boolean isNewUniverse() {
+ return isNewUniverse;
+ }
+
+ public static void setNewUniverse(boolean isNewUniverse) {
+ MetadataBootstrap.isNewUniverse = isNewUniverse;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
index 0a3d622..6c8bebd 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndex.java
@@ -24,14 +24,13 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.transactions.DatasetId;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlBinaryHashFunctionFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
-import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.api.IMetadataIndex;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -44,7 +43,9 @@ import org.apache.hyracks.api.io.FileReference;
/**
* Descriptor for a primary or secondary index on metadata datasets.
*/
-public final class MetadataIndex implements IMetadataIndex {
+public class MetadataIndex implements IMetadataIndex {
+
+ private static final long serialVersionUID = 1L;
// Name of dataset that is indexed.
protected final String datasetName;
// Name of index. null for primary indexes. non-null for secondary indexes.
@@ -52,6 +53,7 @@ public final class MetadataIndex implements IMetadataIndex {
// Types of key fields.
protected final IAType[] keyTypes;
// Names of key fields. Used to compute partitionExprs.
+ // Note: used list implementation must implement java.io.Serializable
protected final List<List<String>> keyNames;
// Field permutation for BTree insert. Auto-created based on numFields.
protected final int[] fieldPermutation;
@@ -82,13 +84,13 @@ public final class MetadataIndex implements IMetadataIndex {
public MetadataIndex(MetadataIndexImmutableProperties indexImmutableProperties, int numFields, IAType[] keyTypes,
List<List<String>> keyNames, int numSecondaryIndexKeys, ARecordType payloadType, boolean isPrimaryIndex,
- int[] primaryKeyIndexes) throws MetadataException {
+ int[] primaryKeyIndexes) {
// Sanity checks.
if (keyTypes.length != keyNames.size()) {
- throw new MetadataException("Unequal number of key types and names given.");
+ throw new AssertionError("Unequal number of key names and key types");
}
if (keyTypes.length > numFields) {
- throw new MetadataException("Number of keys given is greater than total number of fields.");
+ throw new AssertionError("Key size is larger than total number of fields");
}
// Set simple fields.
this.datasetName = indexImmutableProperties.getDatasetName();
@@ -101,7 +103,7 @@ public final class MetadataIndex implements IMetadataIndex {
for (int i = 0; i < numFields; i++) {
fieldPermutation[i] = i;
}
- // Create serdes for RecordDescriptor;
+ // Create serdes for RecordDescriptor
@SuppressWarnings("rawtypes")
ISerializerDeserializer[] serdes = new ISerializerDeserializer[numFields];
for (int i = 0; i < keyTypes.length; i++) {
@@ -189,7 +191,7 @@ public final class MetadataIndex implements IMetadataIndex {
@Override
public List<List<String>> getPartitioningExpr() {
- ArrayList<List<String>> partitioningExpr = new ArrayList<List<String>>();
+ ArrayList<List<String>> partitioningExpr = new ArrayList<>();
for (int i = 0; i < keyNames.size(); i++) {
partitioningExpr.add(keyNames.get(i));
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java
index 2c2468c..53a3cbd 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataIndexImmutableProperties.java
@@ -18,46 +18,19 @@
*/
package org.apache.asterix.metadata.bootstrap;
-import org.apache.asterix.common.config.MetadataConstants;
-
-public enum MetadataIndexImmutableProperties {
- METADATA(MetadataConstants.METADATA_DATAVERSE_NAME, 0, 0),
- DATAVERSE("Dataverse", 1, 1),
- DATASET("Dataset", 2, 2),
- DATATYPE("Datatype", 3, 3),
- INDEX("Index", 4, 4),
- NODE("Node", 5, 5),
- NODEGROUP("Nodegroup", 6, 6),
- FUNCTION("Function", 7, 7),
- DATASOURCE_ADAPTER("DatasourceAdapter", 8, 8),
- LIBRARY("Library", 9, 9),
- FEED("Feed", 10, 10),
- FEED_ACTIVITY_DATASET_ID("FeedActivity", 11, 11),
- FEED_POLICY("FeedPolicy", 12, 12),
- COMPACTION_POLICY("CompactionPolicy", 13, 13),
- EXTERNAL_FILE("ExternalFile", 14, 14);
-
+public class MetadataIndexImmutableProperties {
private final String indexName;
private final int datasetId;
private final long resourceId;
- private final MetadataIndexImmutableProperties dataset;
+ // TODO(till? should we reconsider these numbers?!)
+ public static final int FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID = 52;
public static final int FIRST_AVAILABLE_USER_DATASET_ID = 100;
- private MetadataIndexImmutableProperties(String indexName, int datasetId, long resourceId) {
+ public MetadataIndexImmutableProperties(String indexName, int datasetId, long resourceId) {
this.indexName = indexName;
this.datasetId = datasetId;
this.resourceId = resourceId;
- //a primary index's dataset is itself
- this.dataset = this;
- }
-
- private MetadataIndexImmutableProperties(String indexName, MetadataIndexImmutableProperties dataset,
- long resourceId) {
- this.indexName = indexName;
- this.datasetId = dataset.datasetId;
- this.resourceId = resourceId;
- this.dataset = dataset;
}
public long getResourceId() {
@@ -68,11 +41,12 @@ public enum MetadataIndexImmutableProperties {
return indexName;
}
- public String getDatasetName() {
- return dataset.indexName;
+ public int getDatasetId() {
+ return datasetId;
}
- public int getDatasetId() {
- return dataset.datasetId;
+ // Right now, we only have primary indexes. Hence, dataset name is always index name
+ public String getDatasetName() {
+ return indexName;
}
}
[03/16] asterixdb git commit: Add Asterix Extension Manager
Posted by bu...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 679379b..da54e32 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -30,7 +30,6 @@ import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.GlobalConfig;
-import org.apache.asterix.common.config.MetadataConstants;
import org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider;
import org.apache.asterix.common.context.ITransactionSubsystemProvider;
import org.apache.asterix.common.context.TransactionSubsystemProvider;
@@ -50,7 +49,6 @@ import org.apache.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveVal
import org.apache.asterix.external.adapter.factory.LookupAdapterFactory;
import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.external.api.IDataSourceAdapter;
-import org.apache.asterix.external.feed.management.FeedConnectionId;
import org.apache.asterix.external.feed.policy.FeedPolicyAccessor;
import org.apache.asterix.external.indexing.ExternalFile;
import org.apache.asterix.external.indexing.IndexingConstants;
@@ -58,7 +56,6 @@ import org.apache.asterix.external.operators.ExternalBTreeSearchOperatorDescript
import org.apache.asterix.external.operators.ExternalDataScanOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalLookupOperatorDescriptor;
import org.apache.asterix.external.operators.ExternalRTreeSearchOperatorDescriptor;
-import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
import org.apache.asterix.external.provider.AdapterFactoryProvider;
import org.apache.asterix.external.util.ExternalDataConstants;
@@ -66,7 +63,6 @@ import org.apache.asterix.external.util.FeedConstants;
import org.apache.asterix.formats.base.IDataFormat;
import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
import org.apache.asterix.formats.nontagged.AqlLinearizeComparatorFactoryProvider;
-import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.AqlTypeTraitProvider;
import org.apache.asterix.metadata.MetadataException;
import org.apache.asterix.metadata.MetadataManager;
@@ -83,10 +79,10 @@ import org.apache.asterix.metadata.entities.FeedPolicyEntity;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.entities.NodeGroup;
-import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
import org.apache.asterix.metadata.utils.DatasetUtils;
import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
+import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil;
import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
import org.apache.asterix.om.types.ARecordType;
@@ -97,7 +93,6 @@ import org.apache.asterix.om.util.AsterixClusterProperties;
import org.apache.asterix.om.util.NonTaggedFormatUtil;
import org.apache.asterix.runtime.base.AsterixTupleFilterFactory;
import org.apache.asterix.runtime.formats.FormatUtils;
-import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.runtime.operators.AsterixLSMInvertedIndexUpsertOperatorDescriptor;
import org.apache.asterix.runtime.operators.AsterixLSMTreeUpsertOperatorDescriptor;
@@ -334,110 +329,31 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv,
JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
try {
- switch (((AqlDataSource) dataSource).getDatasourceType()) {
- case FEED:
- return buildFeedCollectRuntime(jobSpec, (FeedDataSource) dataSource);
- case INTERNAL_DATASET: {
- // querying an internal dataset
- return buildInternalDatasetScan(jobSpec, scanVariables, minFilterVars, maxFilterVars, opSchema,
- typeEnv, dataSource, context, implConfig);
- }
- case EXTERNAL_DATASET: {
- // querying an external dataset
- Dataset dataset = ((DatasetDataSource) dataSource).getDataset();
- String itemTypeName = dataset.getItemTypeName();
- IAType itemType = MetadataManager.INSTANCE
- .getDatatype(mdTxnCtx, dataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
-
- ExternalDatasetDetails edd = (ExternalDatasetDetails) dataset.getDatasetDetails();
- IAdapterFactory adapterFactory = getConfiguredAdapterFactory(dataset, edd.getAdapter(),
- edd.getProperties(), (ARecordType) itemType, false, null, null);
- return buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
- NonTaggedDataFormat.INSTANCE);
- }
- case LOADABLE: {
- // This is a load into dataset operation
- LoadableDataSource alds = (LoadableDataSource) dataSource;
- List<List<String>> partitioningKeys = alds.getPartitioningKeys();
- boolean isPKAutoGenerated =
- ((InternalDatasetDetails) alds.getTargetDataset().getDatasetDetails()).isAutogenerated();
- ARecordType itemType = (ARecordType) alds.getLoadedType();
- int pkIndex = 0;
- IAdapterFactory adapterFactory =
- getConfiguredAdapterFactory(alds.getTargetDataset(), alds.getAdapter(),
- alds.getAdapterProperties(), itemType, isPKAutoGenerated, partitioningKeys, null);
- RecordDescriptor rDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context);
- return buildLoadableDatasetScan(jobSpec, alds, adapterFactory, rDesc, isPKAutoGenerated,
- partitioningKeys, itemType, pkIndex);
- }
- default: {
- throw new IllegalArgumentException();
- }
-
- }
+ return ((AqlDataSource) dataSource).buildDatasourceScanRuntime(this, dataSource, scanVariables,
+ projectVariables, projectPushed, minFilterVars, maxFilterVars, opSchema, typeEnv, context, jobSpec,
+ implConfig);
} catch (AsterixException e) {
throw new AlgebricksException(e);
}
}
- public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildFeedCollectRuntime(JobSpecification jobSpec,
- FeedDataSource feedDataSource) throws AlgebricksException {
-
- try {
- ARecordType feedOutputType = (ARecordType) feedDataSource.getItemType();
- ISerializerDeserializer payloadSerde =
- NonTaggedDataFormat.INSTANCE.getSerdeProvider().getSerializerDeserializer(feedOutputType);
- IAType metaType = feedDataSource.getMetaItemType();
- List<IAType> pkTypes = feedDataSource.getPkTypes();
- ArrayList<ISerializerDeserializer> serdes = new ArrayList<>();
- serdes.add(payloadSerde);
- if (metaType != null) {
- serdes.add(AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(metaType));
- }
- if (pkTypes != null) {
- for (IAType type : pkTypes) {
- serdes.add(AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(type));
- }
- }
- RecordDescriptor feedDesc =
- new RecordDescriptor(serdes.toArray(new ISerializerDeserializer[serdes.size()]));
- FeedPolicyEntity feedPolicy =
- (FeedPolicyEntity) feedDataSource.getProperties().get(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY);
- if (feedPolicy == null) {
- throw new AlgebricksException("Feed not configured with a policy");
- }
- feedPolicy.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy.getPolicyName());
- FeedConnectionId feedConnectionId = new FeedConnectionId(feedDataSource.getId().getDataverseName(),
- feedDataSource.getId().getDatasourceName(), feedDataSource.getTargetDataset());
- FeedCollectOperatorDescriptor feedCollector =
- new FeedCollectOperatorDescriptor(jobSpec, feedConnectionId, feedDataSource.getSourceFeedId(),
- feedOutputType, feedDesc, feedPolicy.getProperties(), feedDataSource.getLocation());
-
- return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(feedCollector,
- determineLocationConstraint(feedDataSource));
-
- } catch (Exception e) {
- throw new AlgebricksException(e);
- }
- }
-
- private AlgebricksAbsolutePartitionConstraint determineLocationConstraint(FeedDataSource feedDataSource)
+ public static AlgebricksAbsolutePartitionConstraint determineLocationConstraint(FeedDataSource feedDataSource)
throws AsterixException {
return new AlgebricksAbsolutePartitionConstraint(feedDataSource.getLocations());
}
- private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildLoadableDatasetScan(JobSpecification jobSpec,
- LoadableDataSource alds, IAdapterFactory adapterFactory, RecordDescriptor rDesc, boolean isPKAutoGenerated,
- List<List<String>> primaryKeys, ARecordType recType, int pkIndex) throws AlgebricksException {
- ExternalDataScanOperatorDescriptor dataScanner =
- new ExternalDataScanOperatorDescriptor(jobSpec, rDesc, adapterFactory);
+ protected Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildLoadableDatasetScan(
+ JobSpecification jobSpec, IAdapterFactory adapterFactory, RecordDescriptor rDesc)
+ throws AlgebricksException {
+ ExternalDataScanOperatorDescriptor dataScanner = new ExternalDataScanOperatorDescriptor(jobSpec, rDesc,
+ adapterFactory);
AlgebricksPartitionConstraint constraint;
try {
constraint = adapterFactory.getPartitionConstraint();
} catch (Exception e) {
throw new AlgebricksException(e);
}
- return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(dataScanner, constraint);
+ return new Pair<>(dataScanner, constraint);
}
public IDataFormat getDataFormat(String dataverseName) throws AsterixException {
@@ -451,40 +367,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
return format;
}
- private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildInternalDatasetScan(JobSpecification jobSpec,
- List<LogicalVariable> outputVars, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars,
- IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, IDataSource<AqlSourceId> dataSource,
- JobGenContext context, Object implConfig) throws AlgebricksException, MetadataException {
- AqlSourceId asid = dataSource.getId();
- String dataverseName = asid.getDataverseName();
- String datasetName = asid.getDatasourceName();
- Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, datasetName);
-
- int[] minFilterFieldIndexes = null;
- if (minFilterVars != null && !minFilterVars.isEmpty()) {
- minFilterFieldIndexes = new int[minFilterVars.size()];
- int i = 0;
- for (LogicalVariable v : minFilterVars) {
- minFilterFieldIndexes[i] = opSchema.findVariable(v);
- i++;
- }
- }
- int[] maxFilterFieldIndexes = null;
- if (maxFilterVars != null && !maxFilterVars.isEmpty()) {
- maxFilterFieldIndexes = new int[maxFilterVars.size()];
- int i = 0;
- for (LogicalVariable v : maxFilterVars) {
- maxFilterFieldIndexes[i] = opSchema.findVariable(v);
- i++;
- }
- }
-
- return buildBtreeRuntime(jobSpec, outputVars, opSchema, typeEnv, context, true, false,
- ((DatasetDataSource) dataSource).getDataset(), primaryIndex.getIndexName(), null, null, true, true,
- implConfig, minFilterFieldIndexes, maxFilterFieldIndexes);
- }
-
- private IAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName,
+ protected IAdapterFactory getConfiguredAdapterFactory(Dataset dataset, String adapterName,
Map<String, String> configuration, ARecordType itemType, boolean isPKAutoGenerated,
List<List<String>> primaryKeys, ARecordType metaType) throws AlgebricksException {
try {
@@ -493,9 +376,9 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
configuration, itemType, metaType);
// check to see if dataset is indexed
- Index filesIndex =
- MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(), dataset.getDatasetName(),
- dataset.getDatasetName().concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX));
+ Index filesIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
+ dataset.getDatasetName(),
+ dataset.getDatasetName().concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX));
if (filesIndex != null && filesIndex.getPendingOp() == 0) {
// get files
@@ -526,8 +409,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
ISerializerDeserializer payloadSerde = format.getSerdeProvider().getSerializerDeserializer(itemType);
RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde });
- ExternalDataScanOperatorDescriptor dataScanner =
- new ExternalDataScanOperatorDescriptor(jobSpec, scannerDesc, adapterFactory);
+ ExternalDataScanOperatorDescriptor dataScanner = new ExternalDataScanOperatorDescriptor(jobSpec, scannerDesc,
+ adapterFactory);
AlgebricksPartitionConstraint constraint;
try {
@@ -542,8 +425,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
public Triple<IOperatorDescriptor, AlgebricksPartitionConstraint, IAdapterFactory> buildFeedIntakeRuntime(
JobSpecification jobSpec, Feed primaryFeed, FeedPolicyAccessor policyAccessor) throws Exception {
Triple<IAdapterFactory, RecordDescriptor, IDataSourceAdapter.AdapterType> factoryOutput = null;
- factoryOutput =
- FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx, libraryManager);
+ factoryOutput = FeedMetadataUtil.getPrimaryFeedFactoryAndOutput(primaryFeed, policyAccessor, mdTxnCtx,
+ libraryManager);
ARecordType recordType = FeedMetadataUtil.getOutputType(primaryFeed, primaryFeed.getAdapterConfiguration(),
ExternalDataConstants.KEY_TYPE_NAME);
IAdapterFactory adapterFactory = factoryOutput.first;
@@ -587,18 +470,19 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
ITypeTraits[] typeTraits;
IBinaryComparatorFactory[] comparatorFactories;
- ARecordType itemType =
- (ARecordType) this.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
+ ARecordType itemType = (ARecordType) this.findType(dataset.getItemTypeDataverseName(),
+ dataset.getItemTypeName());
ARecordType metaType = null;
List<Integer> primaryKeyIndicators = null;
if (dataset.hasMetaPart()) {
- metaType =
- (ARecordType) findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ metaType = (ARecordType) findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
primaryKeyIndicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
}
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, itemType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
itemType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = null;
int[] btreeFields = null;
@@ -612,11 +496,11 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
bloomFilterKeyFields[i] = i;
}
Pair<IBinaryComparatorFactory[], ITypeTraits[]> comparatorFactoriesAndTypeTraits =
- getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex(secondaryIndex.getIndexType(),
- secondaryIndex.getKeyFieldNames(), secondaryIndex.getKeyFieldTypes(),
- DatasetUtils.getPartitioningKeys(dataset), itemType, dataset.getDatasetType(),
- dataset.hasMetaPart(), primaryKeyIndicators,
- secondaryIndex.getKeyFieldSourceIndicators(), metaType);
+ getComparatorFactoriesAndTypeTraitsOfSecondaryBTreeIndex(
+ secondaryIndex.getIndexType(), secondaryIndex.getKeyFieldNames(),
+ secondaryIndex.getKeyFieldTypes(), DatasetUtils.getPartitioningKeys(dataset), itemType,
+ dataset.getDatasetType(), dataset.hasMetaPart(), primaryKeyIndicators,
+ secondaryIndex.getKeyFieldSourceIndicators(), metaType);
comparatorFactories = comparatorFactoriesAndTypeTraits.first;
typeTraits = comparatorFactoriesAndTypeTraits.second;
if (filterTypeTraits != null) {
@@ -636,8 +520,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
// get meta item type
ARecordType metaItemType = DatasetUtils.getMetaType(this, dataset);
typeTraits = DatasetUtils.computeTupleTypeTraits(dataset, itemType, metaItemType);
- comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset, itemType, metaItemType,
- context.getBinaryComparatorFactoryProvider());
+ comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset, itemType,
+ metaItemType, context.getBinaryComparatorFactoryProvider());
filterFields = DatasetUtils.createFilterFields(dataset);
btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
}
@@ -673,8 +557,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
: new PrimaryIndexInstantSearchOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
txnSubsystemProvider, ResourceType.LSM_BTREE);
}
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
AsterixRuntimeComponentsProvider rtcProvider = AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER;
BTreeSearchOperatorDescriptor btreeSearchOp;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
@@ -696,12 +580,13 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
// Be Careful of Key Start Index ?
int[] buddyBreeFields = new int[] { numSecondaryKeys };
ExternalBTreeWithBuddyDataflowHelperFactory indexDataflowHelperFactory =
- new ExternalBTreeWithBuddyDataflowHelperFactory(compactionInfo.first, compactionInfo.second,
- new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
- AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
- getStorageProperties().getBloomFilterFalsePositiveRate(), buddyBreeFields,
- ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this), !temp);
+ new ExternalBTreeWithBuddyDataflowHelperFactory(
+ compactionInfo.first, compactionInfo.second,
+ new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
+ AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
+ LSMBTreeWithBuddyIOOperationCallbackFactory.INSTANCE,
+ getStorageProperties().getBloomFilterFalsePositiveRate(), buddyBreeFields,
+ ExternalDatasetsRegistry.INSTANCE.getAndLockDatasetVersion(dataset, this), !temp);
btreeSearchOp = new ExternalBTreeSearchOperatorDescriptor(jobSpec, outputRecDesc, rtcProvider,
rtcProvider, spPc.first, typeTraits, comparatorFactories, bloomFilterKeyFields, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive, indexDataflowHelperFactory, retainInput,
@@ -730,12 +615,12 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
int i = 0;
for (; i < sidxKeyFieldCount; ++i) {
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(sidxKeyFieldTypes.get(i), sidxKeyFieldNames.get(i),
- (hasMeta && secondaryIndexIndicators.get(i).intValue() == 1) ? metaType : recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(sidxKeyFieldTypes.get(i),
+ sidxKeyFieldNames.get(i),
+ (hasMeta && secondaryIndexIndicators.get(i).intValue() == 1) ? metaType : recType);
IAType keyType = keyPairType.first;
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType,
+ true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
}
@@ -757,8 +642,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
} catch (AsterixException e) {
throw new AlgebricksException(e);
}
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType,
+ true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
}
@@ -771,7 +656,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
int[] keyFields, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) throws AlgebricksException {
try {
- ARecordType recType = (ARecordType) findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
+ ARecordType recType = (ARecordType) findType(dataset.getItemTypeDataverseName(),
+ dataset.getItemTypeName());
int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
boolean temp = dataset.getDatasetDetails().isTemp();
@@ -786,10 +672,11 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
int numSecondaryKeys = secondaryKeyFields.size();
if (numSecondaryKeys != 1) {
throw new AlgebricksException("Cannot use " + numSecondaryKeys
- + " fields as a key for the R-tree index. There can be only one field as a key for the R-tree index.");
+ + " fields as a key for the R-tree index. "
+ + "There can be only one field as a key for the R-tree index.");
}
- Pair<IAType, Boolean> keyTypePair =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyFields.get(0), recType);
+ Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
+ secondaryKeyFields.get(0), recType);
IAType keyType = keyTypePair.first;
if (keyType == null) {
throw new AlgebricksException("Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
@@ -815,12 +702,12 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
numNestedSecondaryKeyFields + numPrimaryKeys, typeEnv, context);
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> spPc =
- splitProviderAndPartitionConstraintsForDataset(dataset.getDataverseName(), dataset.getDatasetName(),
- indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataset.getDataverseName(), dataset.getDatasetName(), indexName, temp);
ARecordType metaType = null;
if (dataset.hasMetaPart()) {
- metaType =
- (ARecordType) findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ metaType = (ARecordType) findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
}
IBinaryComparatorFactory[] primaryComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
@@ -831,7 +718,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
recType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = null;
int[] rtreeFields = null;
@@ -845,15 +733,15 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
- ISearchOperationCallbackFactory searchCallbackFactory =
- temp ? NoOpOperationCallbackFactory.INSTANCE : new SecondaryIndexSearchOperationCallbackFactory();
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
+ ISearchOperationCallbackFactory searchCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new SecondaryIndexSearchOperationCallbackFactory();
RTreeSearchOperatorDescriptor rtreeSearchOp;
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
- IBinaryComparatorFactory[] deletedKeyBTreeCompFactories =
- getMergedComparatorFactories(comparatorFactories, primaryComparatorFactories);
+ IBinaryComparatorFactory[] deletedKeyBTreeCompFactories = getMergedComparatorFactories(
+ comparatorFactories, primaryComparatorFactories);
IIndexDataflowHelperFactory idff = new LSMRTreeWithAntiMatterTuplesDataflowHelperFactory(
valueProviderFactories, RTreePolicyType.RTREE, deletedKeyBTreeCompFactories,
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -915,8 +803,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
File outFile = fs.getLocalFile().getFile();
String nodeId = fs.getNodeName();
- SinkWriterRuntimeFactory runtime =
- new SinkWriterRuntimeFactory(printColumns, printerFactories, outFile, getWriterFactory(), inputDesc);
+ SinkWriterRuntimeFactory runtime = new SinkWriterRuntimeFactory(printColumns, printerFactories, outFile,
+ getWriterFactory(), inputDesc);
AlgebricksPartitionConstraint apc = new AlgebricksAbsolutePartitionConstraint(new String[] { nodeId });
return new Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint>(runtime, apc);
}
@@ -1045,8 +933,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
ARecordType metaType = null;
if (dataset.hasMetaPart()) {
- metaType =
- (ARecordType) findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ metaType = (ARecordType) findType(dataset.getMetaItemTypeDataverseName(),
+ dataset.getMetaItemTypeName());
}
String itemTypeName = dataset.getItemTypeName();
@@ -1057,14 +945,15 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
itemType, metaType, context.getBinaryComparatorFactoryProvider());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataSource.getId().getDataverseName(), datasetName,
- indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataSource.getId().getDataverseName(), datasetName, indexName, temp);
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
long numElementsHint = getCardinalityPerPartitionHint(dataset);
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, itemType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
itemType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = DatasetUtils.createFilterFields(dataset);
int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
@@ -1074,8 +963,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
// right callback
// (ex. what's the expected behavior when there is an error during
// bulkload?)
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec, null,
appContext.getStorageManagerInterface(), appContext.getIndexLifecycleManagerProvider(),
splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields, fieldPermutation,
@@ -1147,8 +1036,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
itemType, metaItemType, context.getBinaryComparatorFactoryProvider());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataSource.getId().getDataverseName(), datasetName,
- indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataSource.getId().getDataverseName(), datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -1171,8 +1060,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
: new PrimaryIndexModificationOperationCallbackFactory(jobId, datasetId, primaryKeyFields,
txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE, dataset.hasMetaPart());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
IIndexDataflowHelperFactory idfh = new LSMBTreeDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(datasetId), compactionInfo.first, compactionInfo.second,
new PrimaryIndexOperationTrackerProvider(dataset.getDatasetId()),
@@ -1243,12 +1132,12 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
AsterixTupleFilterFactory filterFactory = createTupleFilterFactory(inputSchemas, typeEnv, filterExpr, context);
switch (secondaryIndex.getIndexType()) {
case BTREE: {
- return getBTreeDmlRuntime(dataverseName, datasetName, indexName, propagatedSchema, typeEnv, primaryKeys,
+ return getBTreeDmlRuntime(dataverseName, datasetName, indexName, propagatedSchema, primaryKeys,
secondaryKeys, additionalNonKeyFields, filterFactory, recordDesc, context, spec, indexOp,
bulkload);
}
case RTREE: {
- return getRTreeDmlRuntime(dataverseName, datasetName, indexName, propagatedSchema, typeEnv, primaryKeys,
+ return getRTreeDmlRuntime(dataverseName, datasetName, indexName, propagatedSchema, primaryKeys,
secondaryKeys, additionalNonKeyFields, filterFactory, recordDesc, context, spec, indexOp,
bulkload);
}
@@ -1469,8 +1358,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
secondaryKeyType.getTypeTag(), indexType, secondaryIndex.getGramLength());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName,
- dataset.getDatasetDetails().isTemp());
+ splitProviderAndPartitionConstraintsForDataset(
+ dataverseName, datasetName, indexName, dataset.getDatasetDetails().isTemp());
// Generate Output Record format
ISerializerDeserializer<?>[] tokenKeyPairFields = new ISerializerDeserializer[numTokenKeyPairFields];
@@ -1539,13 +1428,13 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
return null;
}
IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider();
- IScalarEvaluatorFactory filterEvalFactory =
- expressionRuntimeProvider.createEvaluatorFactory(filterExpr, typeEnv, inputSchemas, context);
+ IScalarEvaluatorFactory filterEvalFactory = expressionRuntimeProvider.createEvaluatorFactory(filterExpr,
+ typeEnv, inputSchemas, context);
return new AsterixTupleFilterFactory(filterEvalFactory, context.getBinaryBooleanInspectorFactory());
}
private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getBTreeDmlRuntime(String dataverseName,
- String datasetName, String indexName, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
+ String datasetName, String indexName, IOperatorSchema propagatedSchema,
List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalNonKeyFields, AsterixTupleFilterFactory filterFactory,
RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, IndexOperation indexOp,
@@ -1602,7 +1491,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
dataset.getDatasetName(), indexName);
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
recType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = null;
int[] btreeFields = null;
@@ -1620,25 +1510,26 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
ITypeTraits[] typeTraits = new ITypeTraits[numKeys];
IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[numKeys];
for (i = 0; i < secondaryKeys.size(); ++i) {
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(i), secondaryKeyNames.get(i), recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(i),
+ secondaryKeyNames.get(i), recType);
IAType keyType = keyPairType.first;
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType,
+ true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
}
List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
for (List<String> partitioningKey : partitioningKeys) {
IAType keyType = recType.getSubFieldType(partitioningKey);
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType,
+ true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
++i;
}
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataverseName, datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -1651,8 +1542,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_BTREE,
dataset.hasMetaPart());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
IIndexDataflowHelperFactory idfh = new LSMBTreeDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(datasetId), compactionInfo.first, compactionInfo.second,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
@@ -1792,8 +1683,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IAType secondaryKeyType = null;
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyExprs.get(0), recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
+ secondaryKeyExprs.get(0), recType);
secondaryKeyType = keyPairType.first;
List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
@@ -1842,7 +1733,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataverseName, datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -1856,8 +1748,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp,
ResourceType.LSM_INVERTED_INDEX, dataset.hasMetaPart());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
IIndexDataflowHelperFactory indexDataFlowFactory;
if (!isPartitioned) {
indexDataFlowFactory = new LSMInvertedIndexDataflowHelperFactory(
@@ -1899,7 +1791,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getRTreeDmlRuntime(String dataverseName,
- String datasetName, String indexName, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
+ String datasetName, String indexName, IOperatorSchema propagatedSchema,
List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalNonKeyFields, AsterixTupleFilterFactory filterFactory,
RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, IndexOperation indexOp,
@@ -1921,11 +1813,11 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
dataset.getDatasetName(), indexName);
List<List<String>> secondaryKeyExprs = secondaryIndex.getKeyFieldNames();
List<IAType> secondaryKeyTypes = secondaryIndex.getKeyFieldTypes();
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyExprs.get(0), recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
+ secondaryKeyExprs.get(0), recType);
IAType spatialType = keyPairType.first;
- boolean isPointMBR =
- spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
+ boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
+ || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numSecondaryKeys = dimension * 2;
int numPrimaryKeys = primaryKeys.size();
@@ -1960,8 +1852,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IPrimitiveValueProviderFactory[] valueProviderFactories =
new IPrimitiveValueProviderFactory[numSecondaryKeys];
for (i = 0; i < numSecondaryKeys; i++) {
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(nestedKeyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE
+ .getBinaryComparatorFactory(nestedKeyType, true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
}
@@ -1976,14 +1868,16 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
dataset, recType, metaItemType, context.getBinaryComparatorFactoryProvider());
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataverseName, datasetName, indexName, temp);
int[] btreeFields = new int[primaryComparatorFactories.length];
for (int k = 0; k < btreeFields.length; k++) {
btreeFields[k] = k + numSecondaryKeys;
}
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
recType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = null;
int[] rtreeFields = null;
@@ -2002,16 +1896,18 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
TransactionSubsystemProvider txnSubsystemProvider = new TransactionSubsystemProvider();
IModificationOperationCallbackFactory modificationCallbackFactory = temp
? new TempDatasetSecondaryIndexModificationOperationCallbackFactory(jobId, datasetId,
- modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_RTREE)
+ modificationCallbackPrimaryKeyFields, txnSubsystemProvider,
+ indexOp, ResourceType.LSM_RTREE)
: new SecondaryIndexModificationOperationCallbackFactory(jobId, datasetId,
- modificationCallbackPrimaryKeyFields, txnSubsystemProvider, indexOp, ResourceType.LSM_RTREE,
+ modificationCallbackPrimaryKeyFields, txnSubsystemProvider,
+ indexOp, ResourceType.LSM_RTREE,
dataset.hasMetaPart());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
- IBinaryComparatorFactory[] deletedKeyBTreeCompFactories =
- getMergedComparatorFactories(comparatorFactories, primaryComparatorFactories);
+ IBinaryComparatorFactory[] deletedKeyBTreeCompFactories = getMergedComparatorFactories(comparatorFactories,
+ primaryComparatorFactories);
IIndexDataflowHelperFactory idff = new LSMRTreeWithAntiMatterTuplesDataflowHelperFactory(
valueProviderFactories, RTreePolicyType.RTREE, deletedKeyBTreeCompFactories,
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -2069,8 +1965,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
numElementsHint = Long.parseLong(numElementsHintString);
}
int numPartitions = 0;
- List<String> nodeGroup =
- MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName()).getNodeNames();
+ List<String> nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, dataset.getNodeGroupName())
+ .getNodeNames();
for (String nd : nodeGroup) {
numPartitions += AsterixClusterProperties.INSTANCE.getNodePartitionsCount(nd);
}
@@ -2112,8 +2008,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
public Dataset findDataset(String dataverse, String dataset) throws AlgebricksException {
- String dv =
- dataverse == null ? (defaultDataverse == null ? null : defaultDataverse.getDataverseName()) : dataverse;
+ String dv = dataverse == null ? (defaultDataverse == null ? null : defaultDataverse.getDataverseName())
+ : dataverse;
if (dv == null) {
return null;
}
@@ -2244,8 +2140,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
spPc = metadataProvider.splitProviderAndPartitionConstraintsForFilesIndex(dataset.getDataverseName(),
dataset.getDatasetName(),
dataset.getDatasetName().concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX), false);
- ISearchOperationCallbackFactory searchOpCallbackFactory =
- temp ? NoOpOperationCallbackFactory.INSTANCE : new SecondaryIndexSearchOperationCallbackFactory();
+ ISearchOperationCallbackFactory searchOpCallbackFactory = temp ? NoOpOperationCallbackFactory.INSTANCE
+ : new SecondaryIndexSearchOperationCallbackFactory();
// Create the operator
ExternalLookupOperatorDescriptor op = new ExternalLookupOperatorDescriptor(jobSpec, adapterFactory,
outRecDesc, indexDataflowHelperFactory, retainInput, appContext.getIndexLifecycleManagerProvider(),
@@ -2319,8 +2215,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IBinaryComparatorFactory[] comparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(dataset,
itemType, metaItemType, context.getBinaryComparatorFactoryProvider());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataSource.getId().getDataverseName(), datasetName,
- indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataSource.getId().getDataverseName(), datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -2331,7 +2227,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
}
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, itemType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
itemType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = DatasetUtils.createFilterFields(dataset);
int[] btreeFields = DatasetUtils.createBTreeFieldsWhenThereisAFilter(dataset);
@@ -2346,8 +2243,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
LockThenSearchOperationCallbackFactory searchCallbackFactory = new LockThenSearchOperationCallbackFactory(
jobId, datasetId, primaryKeyFields, txnSubsystemProvider, ResourceType.LSM_BTREE);
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
IIndexDataflowHelperFactory idfh = new LSMBTreeDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(datasetId), compactionInfo.first, compactionInfo.second,
new PrimaryIndexOperationTrackerProvider(dataset.getDatasetId()),
@@ -2356,24 +2253,24 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
btreeFields, filterFields, !temp);
AsterixLSMTreeUpsertOperatorDescriptor op;
- ITypeTraits[] outputTypeTraits =
- new ITypeTraits[recordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
+ ITypeTraits[] outputTypeTraits = new ITypeTraits[recordDesc.getFieldCount()
+ + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
ISerializerDeserializer[] outputSerDes = new ISerializerDeserializer[recordDesc.getFieldCount()
+ (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
for (int j = 0; j < recordDesc.getFieldCount(); j++) {
outputTypeTraits[j] = recordDesc.getTypeTraits()[j];
outputSerDes[j] = recordDesc.getFields()[j];
}
- outputSerDes[outputSerDes.length - (dataset.hasMetaPart() ? 2 : 1) - numFilterFields] =
- FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType);
- outputTypeTraits[outputTypeTraits.length - (dataset.hasMetaPart() ? 2 : 1) - numFilterFields] =
- FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(itemType);
+ outputSerDes[outputSerDes.length - (dataset.hasMetaPart() ? 2 : 1) - numFilterFields] = FormatUtils
+ .getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType);
+ outputTypeTraits[outputTypeTraits.length - (dataset.hasMetaPart() ? 2 : 1) - numFilterFields] = FormatUtils
+ .getDefaultFormat().getTypeTraitProvider().getTypeTrait(itemType);
if (dataset.hasMetaPart()) {
- outputSerDes[outputSerDes.length - 1 - numFilterFields] =
- FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(metaItemType);
- outputTypeTraits[outputTypeTraits.length - 1 - numFilterFields] =
- FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(metaItemType);
+ outputSerDes[outputSerDes.length - 1 - numFilterFields] = FormatUtils.getDefaultFormat()
+ .getSerdeProvider().getSerializerDeserializer(metaItemType);
+ outputTypeTraits[outputTypeTraits.length - 1 - numFilterFields] = FormatUtils.getDefaultFormat()
+ .getTypeTraitProvider().getTypeTrait(metaItemType);
}
int fieldIdx = -1;
@@ -2399,7 +2296,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
searchCallbackFactory, null);
op.setType(itemType);
op.setFilterIndex(fieldIdx);
- return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(op, splitsAndConstraint.second);
+ return new Pair<>(op, splitsAndConstraint.second);
} catch (MetadataException me) {
throw new AlgebricksException(me);
@@ -2433,7 +2330,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
AsterixTupleFilterFactory filterFactory = createTupleFilterFactory(inputSchemas, typeEnv, filterExpr, context);
ArrayList<LogicalVariable> prevAdditionalFilteringKeys = null;
if (prevAdditionalFilteringKey != null) {
- prevAdditionalFilteringKeys = new ArrayList<LogicalVariable>();
+ prevAdditionalFilteringKeys = new ArrayList<>();
prevAdditionalFilteringKeys.add(prevAdditionalFilteringKey);
}
switch (secondaryIndex.getIndexType()) {
@@ -2451,7 +2348,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
case SINGLE_PARTITION_NGRAM_INVIX:
case LENGTH_PARTITIONED_WORD_INVIX:
case LENGTH_PARTITIONED_NGRAM_INVIX: {
- return getInvertedIndexUpsertRuntime(dataverseName, datasetName, indexName, propagatedSchema, typeEnv,
+ return getInvertedIndexUpsertRuntime(dataverseName, datasetName, indexName, propagatedSchema,
primaryKeys, secondaryKeys, additionalFilteringKeys, filterFactory, recordDesc, context, spec,
secondaryIndex.getIndexType(), prevSecondaryKeys, prevAdditionalFilteringKeys);
}
@@ -2464,7 +2361,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
//TODO: refactor this method
private Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInvertedIndexUpsertRuntime(String dataverseName,
- String datasetName, String indexName, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
+ String datasetName, String indexName, IOperatorSchema propagatedSchema,
List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
List<LogicalVariable> additionalFilteringKeys, AsterixTupleFilterFactory filterFactory,
RecordDescriptor recordDesc, JobGenContext context, JobSpecification spec, IndexType indexType,
@@ -2590,8 +2487,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IAType secondaryKeyType = null;
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyExprs.get(0), recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
+ secondaryKeyExprs.get(0), recType);
secondaryKeyType = keyPairType.first;
List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
@@ -2615,7 +2512,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
secondaryKeyType.getTypeTag(), indexType, secondaryIndex.getGramLength());
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
recType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = null;
@@ -2640,7 +2538,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataverseName, datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -2654,8 +2553,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
modificationCallbackPrimaryKeyFields, txnSubsystemProvider, IndexOperation.UPSERT,
ResourceType.LSM_INVERTED_INDEX, dataset.hasMetaPart());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
IIndexDataflowHelperFactory indexDataFlowFactory;
if (!isPartitioned) {
indexDataFlowFactory = new LSMInvertedIndexDataflowHelperFactory(
@@ -2713,12 +2612,12 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
List<List<String>> secondaryKeyExprs = secondaryIndex.getKeyFieldNames();
List<IAType> secondaryKeyTypes = secondaryIndex.getKeyFieldTypes();
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyExprs.get(0), recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
+ secondaryKeyExprs.get(0), recType);
IAType spatialType = keyPairType.first;
- boolean isPointMBR =
- spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
+ boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
+ || spatialType.getTypeTag() == ATypeTag.POINT3D;
int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
int numSecondaryKeys = dimension * 2;
int numPrimaryKeys = primaryKeys.size();
@@ -2775,8 +2674,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
IPrimitiveValueProviderFactory[] valueProviderFactories =
new IPrimitiveValueProviderFactory[numSecondaryKeys];
for (i = 0; i < numSecondaryKeys; i++) {
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(nestedKeyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE
+ .getBinaryComparatorFactory(nestedKeyType, true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
}
@@ -2792,14 +2691,16 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
dataset, recType, metaItemType, context.getBinaryComparatorFactoryProvider());
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataverseName, datasetName, indexName, temp);
int[] btreeFields = new int[primaryComparatorFactories.length];
for (int k = 0; k < btreeFields.length; k++) {
btreeFields[k] = k + numSecondaryKeys;
}
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
recType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = null;
int[] rtreeFields = null;
@@ -2824,10 +2725,10 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
modificationCallbackPrimaryKeyFields, txnSubsystemProvider, IndexOperation.UPSERT,
ResourceType.LSM_RTREE, dataset.hasMetaPart());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
- IBinaryComparatorFactory[] deletedKeyBTreeCompFactories =
- getMergedComparatorFactories(comparatorFactories, primaryComparatorFactories);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
+ IBinaryComparatorFactory[] deletedKeyBTreeCompFactories = getMergedComparatorFactories(comparatorFactories,
+ primaryComparatorFactories);
IIndexDataflowHelperFactory idff = new LSMRTreeWithAntiMatterTuplesDataflowHelperFactory(
valueProviderFactories, RTreePolicyType.RTREE, deletedKeyBTreeCompFactories,
new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()), compactionInfo.first,
@@ -2926,7 +2827,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
dataset.getDatasetName(), indexName);
ITypeTraits[] filterTypeTraits = DatasetUtils.computeFilterTypeTraits(dataset, recType);
- IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.computeFilterBinaryComparatorFactories(dataset,
+ IBinaryComparatorFactory[] filterCmpFactories = DatasetUtils.
+ computeFilterBinaryComparatorFactories(dataset,
recType, context.getBinaryComparatorFactoryProvider());
int[] filterFields = null;
int[] btreeFields = null;
@@ -2944,25 +2846,26 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
ITypeTraits[] typeTraits = new ITypeTraits[numKeys];
IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[numKeys];
for (i = 0; i < secondaryKeys.size(); ++i) {
- Pair<IAType, Boolean> keyPairType =
- Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(i), secondaryKeyNames.get(i), recType);
+ Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(i),
+ secondaryKeyNames.get(i), recType);
IAType keyType = keyPairType.first;
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.
+ getBinaryComparatorFactory(keyType, true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
}
List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
for (List<String> partitioningKey : partitioningKeys) {
IAType keyType = recType.getSubFieldType(partitioningKey);
- comparatorFactories[i] =
- AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
+ comparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.
+ getBinaryComparatorFactory(keyType, true);
typeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(keyType);
++i;
}
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
- splitProviderAndPartitionConstraintsForDataset(dataverseName, datasetName, indexName, temp);
+ splitProviderAndPartitionConstraintsForDataset(
+ dataverseName, datasetName, indexName, temp);
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -2976,8 +2879,8 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
modificationCallbackPrimaryKeyFields, txnSubsystemProvider, IndexOperation.UPSERT,
ResourceType.LSM_BTREE, dataset.hasMetaPart());
- Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo =
- DatasetUtils.getMergePolicyFactory(dataset, mdTxnCtx);
+ Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils
+ .getMergePolicyFactory(dataset, mdTxnCtx);
IIndexDataflowHelperFactory idfh = new LSMBTreeDataflowHelperFactory(
new AsterixVirtualBufferCacheProvider(datasetId), compactionInfo.first, compactionInfo.second,
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
@@ -2989,7 +2892,7 @@ public class AqlMetadataProvider implements IMetadataProvider<AqlSourceId, Strin
splitsAndConstraint.first, typeTraits, comparatorFactories, bloomFilterKeyFields, fieldPermutation,
idfh, filterFactory, false, indexName, null, modificationCallbackFactory,
NoOpOperationCallbackFactory.INSTANCE, prevFieldPermutation);
- return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(op, splitsAndConstraint.second);
+ return new Pair<>(op, splitsAndConstraint.second);
} catch (Exception e) {
throw new AlgebricksException(e);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 4148d65..5a601bc 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -20,14 +20,28 @@ package org.apache.asterix.metadata.declared;
import java.util.List;
+import org.apache.asterix.external.api.IAdapterFactory;
import org.apache.asterix.metadata.IDatasetDetails;
+import org.apache.asterix.metadata.MetadataManager;
import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
+import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.job.JobSpecification;
public class DatasetDataSource extends AqlDataSource {
@@ -35,7 +49,7 @@ public class DatasetDataSource extends AqlDataSource {
public DatasetDataSource(AqlSourceId id, Dataset dataset, IAType itemType, IAType metaItemType,
AqlDataSourceType datasourceType, IDatasetDetails datasetDetails, INodeDomain datasetDomain)
- throws AlgebricksException {
+ throws AlgebricksException {
super(id, itemType, metaItemType, datasourceType, datasetDomain);
this.dataset = dataset;
switch (dataset.getDatasetType()) {
@@ -57,8 +71,8 @@ public class DatasetDataSource extends AqlDataSource {
InternalDatasetDetails internalDatasetDetails = (InternalDatasetDetails) datasetDetails;
ARecordType recordType = (ARecordType) itemType;
ARecordType metaRecordType = (ARecordType) metaItemType;
- List<IAType> partitioningKeyTypes = KeyFieldTypeUtils.getPartitioningKeyTypes(internalDatasetDetails,
- recordType, metaRecordType);
+ List<IAType> partitioningKeyTypes =
+ KeyFieldTypeUtils.getPartitioningKeyTypes(internalDatasetDetails, recordType, metaRecordType);
int n = partitioningKeyTypes.size();
schemaTypes = metaItemType == null ? new IAType[n + 1] : new IAType[n + 2];
for (int keyIndex = 0; keyIndex < n; ++keyIndex) {
@@ -75,4 +89,56 @@ public class DatasetDataSource extends AqlDataSource {
schemaTypes[0] = itemType;
}
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime(
+ AqlMetadataProvider aqlMetadataProvider, IDataSource<AqlSourceId> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+ throws AlgebricksException {
+ switch (dataset.getDatasetType()) {
+ case EXTERNAL:
+ Dataset externalDataset = ((DatasetDataSource) dataSource).getDataset();
+ String itemTypeName = externalDataset.getItemTypeName();
+ IAType itemType = MetadataManager.INSTANCE.getDatatype(aqlMetadataProvider.getMetadataTxnContext(),
+ externalDataset.getItemTypeDataverseName(), itemTypeName).getDatatype();
+
+ ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails();
+ IAdapterFactory adapterFactory = aqlMetadataProvider.getConfiguredAdapterFactory(externalDataset,
+ edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, false, null, null);
+ return aqlMetadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory,
+ NonTaggedDataFormat.INSTANCE);
+ case INTERNAL:
+ AqlSourceId asid = getId();
+ String dataverseName = asid.getDataverseName();
+ String datasetName = asid.getDatasourceName();
+ Index primaryIndex = MetadataManager.INSTANCE.getIndex(aqlMetadataProvider.getMetadataTxnContext(),
+ dataverseName, datasetName, datasetName);
+
+ int[] minFilterFieldIndexes = null;
+ if (minFilterVars != null && !minFilterVars.isEmpty()) {
+ minFilterFieldIndexes = new int[minFilterVars.size()];
+ int i = 0;
+ for (LogicalVariable v : minFilterVars) {
+ minFilterFieldIndexes[i] = opSchema.findVariable(v);
+ i++;
+ }
+ }
+ int[] maxFilterFieldIndexes = null;
+ if (maxFilterVars != null && !maxFilterVars.isEmpty()) {
+ maxFilterFieldIndexes = new int[maxFilterVars.size()];
+ int i = 0;
+ for (LogicalVariable v : maxFilterVars) {
+ maxFilterFieldIndexes[i] = opSchema.findVariable(v);
+ i++;
+ }
+ }
+ return aqlMetadataProvider.buildBtreeRuntime(jobSpec, scanVariables, opSchema, typeEnv, context, true,
+ false, ((DatasetDataSource) dataSource).getDataset(), primaryIndex.getIndexName(), null, null,
+ true, true, implConfig, minFilterFieldIndexes, maxFilterFieldIndexes);
+ default:
+ throw new AlgebricksException("Unknown datasource type");
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
index 48f33ed..0a81f03 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java
@@ -18,20 +18,39 @@
*/
package org.apache.asterix.metadata.declared;
+import java.util.ArrayList;
import java.util.List;
import org.apache.asterix.active.EntityId;
import org.apache.asterix.external.feed.api.IFeed;
+import org.apache.asterix.external.feed.management.FeedConnectionId;
+import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
import org.apache.asterix.external.util.FeedUtils.FeedRuntimeType;
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
import org.apache.asterix.metadata.entities.Feed;
+import org.apache.asterix.metadata.entities.FeedPolicyEntity;
+import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.util.AsterixClusterProperties;
+import org.apache.asterix.runtime.formats.NonTaggedDataFormat;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
+import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.job.JobSpecification;
-public class FeedDataSource extends AqlDataSource {
+public class FeedDataSource extends AqlDataSource implements IMutationDataSource {
private final Feed feed;
private final EntityId sourceFeedId;
@@ -113,6 +132,7 @@ public class FeedDataSource extends AqlDataSource {
return pkTypes;
}
+ @Override
public List<ScalarFunctionCallExpression> getKeyAccessExpression() {
return keyAccessExpression;
}
@@ -127,10 +147,12 @@ public class FeedDataSource extends AqlDataSource {
return dataScanVariables.get(0);
}
+ @Override
public boolean isChange() {
return pkTypes != null;
}
+ @Override
public List<LogicalVariable> getPkVars(List<LogicalVariable> allVars) {
if (pkTypes == null) {
return null;
@@ -141,4 +163,45 @@ public class FeedDataSource extends AqlDataSource {
return allVars.subList(1, allVars.size());
}
}
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime(
+ AqlMetadataProvider aqlMetadataProvider, IDataSource<AqlSourceId> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema,
+ IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig)
+ throws AlgebricksException {
+ try {
+ ARecordType feedOutputType = (ARecordType) itemType;
+ ISerializerDeserializer payloadSerde = NonTaggedDataFormat.INSTANCE.getSerdeProvider()
+ .getSerializerDeserializer(feedOutputType);
+ ArrayList<ISerializerDeserializer> serdes = new ArrayList<>();
+ serdes.add(payloadSerde);
+ if (metaItemType != null) {
+ serdes.add(AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(metaItemType));
+ }
+ if (pkTypes != null) {
+ for (IAType type : pkTypes) {
+ serdes.add(AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(type));
+ }
+ }
+ RecordDescriptor feedDesc = new RecordDescriptor(
+ serdes.toArray(new ISerializerDeserializer[serdes.size()]));
+ FeedPolicyEntity feedPolicy = (FeedPolicyEntity) getProperties()
+ .get(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY);
+ if (feedPolicy == null) {
+ throw new AlgebricksException("Feed not configured with a policy");
+ }
+ feedPolicy.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy.getPolicyName());
+ FeedConnectionId feedConnectionId = new FeedConnectionId(getId().getDataverseName(),
+ getId().getDatasourceName(), getTargetDataset());
+ FeedCollectOperatorDescriptor feedCollector = new FeedCollectOperatorDescriptor(jobSpec, feedConnectionId,
+ getSourceFeedId(), feedOutputType, feedDesc, feedPolicy.getProperties(), getLocation());
+
+ return new Pair<>(feedCollector, new AlgebricksAbsolutePartitionConstraint(getLocations()));
+
+ } catch (Exception e) {
+ throw new AlgebricksException(e);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IMutationDataSource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IMutationDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IMutationDataSource.java
new file mode 100644
index 0000000..2feb652
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/IMutationDataSource.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.declared;
+
+import java.util.List;
+
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+
+public interface IMutationDataSource {
+
+ boolean isChange();
+
+ List<LogicalVariable> getPkVars(List<LogicalVariable> allVars);
+
+ List<ScalarFunctionCallExpression> getKeyAccessExpression();
+
+}