You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by im...@apache.org on 2015/08/25 18:43:52 UTC
[04/51] [partial] incubator-asterixdb git commit: Change folder
structure for Java repackage
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
new file mode 100644
index 0000000..956b447
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -0,0 +1,279 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.hyracks.bootstrap;
+
+import java.io.File;
+import java.rmi.server.UnicastRemoteObject;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+
+import edu.uci.ics.asterix.api.common.AsterixAppRuntimeContext;
+import edu.uci.ics.asterix.common.api.AsterixThreadFactory;
+import edu.uci.ics.asterix.common.api.IAsterixAppRuntimeContext;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
+import edu.uci.ics.asterix.common.transactions.IRecoveryManager;
+import edu.uci.ics.asterix.common.transactions.IRecoveryManager.SystemState;
+import edu.uci.ics.asterix.event.schema.cluster.Cluster;
+import edu.uci.ics.asterix.event.schema.cluster.Node;
+import edu.uci.ics.asterix.metadata.MetadataManager;
+import edu.uci.ics.asterix.metadata.MetadataNode;
+import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
+import edu.uci.ics.asterix.metadata.api.IMetadataNode;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataBootstrap;
+import edu.uci.ics.asterix.om.util.AsterixClusterProperties;
+import edu.uci.ics.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import edu.uci.ics.hyracks.api.application.INCApplicationContext;
+import edu.uci.ics.hyracks.api.application.INCApplicationEntryPoint;
+import edu.uci.ics.hyracks.api.lifecycle.ILifeCycleComponentManager;
+import edu.uci.ics.hyracks.api.lifecycle.LifeCycleComponentManager;
+
+public class NCApplicationEntryPoint implements INCApplicationEntryPoint {
+ private static final Logger LOGGER = Logger.getLogger(NCApplicationEntryPoint.class.getName());
+
+ @Option(name = "-metadata-port", usage = "IP port to bind metadata listener (default: random port)", required = false)
+ public int metadataRmiPort = 0;
+
+ @Option(name = "-initial-run", usage = "A flag indicating if it's the first time the NC is started (default: false)", required = false)
+ public boolean initialRun = false;
+
+ private INCApplicationContext ncApplicationContext = null;
+ private IAsterixAppRuntimeContext runtimeContext;
+ private String nodeId;
+ private boolean isMetadataNode = false;
+ private boolean stopInitiated = false;
+ private SystemState systemState = SystemState.NEW_UNIVERSE;
+ private final long NON_SHARP_CHECKPOINT_TARGET_LSN = -1;
+
+ @Override
+ public void start(INCApplicationContext ncAppCtx, String[] args) throws Exception {
+ CmdLineParser parser = new CmdLineParser(this);
+
+ try {
+ parser.parseArgument(args);
+ } catch (CmdLineException e) {
+ System.err.println(e.getMessage());
+ System.err.println("Usage:");
+ parser.printUsage(System.err);
+ throw e;
+ }
+
+ ncAppCtx.setThreadFactory(new AsterixThreadFactory(ncAppCtx.getLifeCycleComponentManager()));
+ ncApplicationContext = ncAppCtx;
+ nodeId = ncApplicationContext.getNodeId();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Starting Asterix node controller: " + nodeId);
+ }
+
+ runtimeContext = new AsterixAppRuntimeContext(ncApplicationContext);
+ AsterixMetadataProperties metadataProperties = ((IAsterixPropertiesProvider) runtimeContext)
+ .getMetadataProperties();
+ if (!metadataProperties.getNodeNames().contains(ncApplicationContext.getNodeId())) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Substitute node joining : " + ncApplicationContext.getNodeId());
+ }
+ updateOnNodeJoin();
+ }
+ runtimeContext.initialize();
+ ncApplicationContext.setApplicationObject(runtimeContext);
+
+ if (initialRun) {
+ LOGGER.info("System is being initialized. (first run)");
+ systemState = SystemState.NEW_UNIVERSE;
+ } else {
+ // #. recover if the system is corrupted by checking system state.
+ IRecoveryManager recoveryMgr = runtimeContext.getTransactionSubsystem().getRecoveryManager();
+ systemState = recoveryMgr.getSystemState();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("System is in a state: " + systemState);
+ }
+
+ if (systemState != SystemState.NEW_UNIVERSE) {
+ PersistentLocalResourceRepository localResourceRepository = (PersistentLocalResourceRepository) runtimeContext
+ .getLocalResourceRepository();
+ localResourceRepository.initialize(nodeId, null, false, runtimeContext.getResourceIdFactory());
+ }
+
+ if (systemState == SystemState.CORRUPTED) {
+ recoveryMgr.startRecovery(true);
+ }
+ }
+
+ }
+
+ @Override
+ public void stop() throws Exception {
+ if (!stopInitiated) {
+ runtimeContext.setShuttingdown(true);
+ stopInitiated = true;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Stopping Asterix node controller: " + nodeId);
+ }
+
+ IRecoveryManager recoveryMgr = runtimeContext.getTransactionSubsystem().getRecoveryManager();
+ recoveryMgr.checkpoint(true, NON_SHARP_CHECKPOINT_TARGET_LSN);
+
+ if (isMetadataNode) {
+ MetadataBootstrap.stopUniverse();
+ }
+
+ ncApplicationContext.getLifeCycleComponentManager().stopAll(false);
+ runtimeContext.deinitialize();
+ } else {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Duplicate attempt to stop ignored: " + nodeId);
+ }
+ }
+ }
+
+ @Override
+ public void notifyStartupComplete() throws Exception {
+ AsterixMetadataProperties metadataProperties = ((IAsterixPropertiesProvider) runtimeContext)
+ .getMetadataProperties();
+
+ if (systemState == SystemState.NEW_UNIVERSE) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("System state: " + SystemState.NEW_UNIVERSE);
+ LOGGER.info("Node ID: " + nodeId);
+ LOGGER.info("Stores: " + metadataProperties.getStores());
+ LOGGER.info("Root Metadata Store: " + metadataProperties.getStores().get(nodeId)[0]);
+ }
+
+ PersistentLocalResourceRepository localResourceRepository = (PersistentLocalResourceRepository) runtimeContext
+ .getLocalResourceRepository();
+ localResourceRepository.initialize(nodeId, metadataProperties.getStores().get(nodeId)[0], true, null);
+ }
+
+ IAsterixStateProxy proxy = null;
+ isMetadataNode = nodeId.equals(metadataProperties.getMetadataNodeName());
+ if (isMetadataNode) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bootstrapping metadata");
+ }
+ MetadataNode.INSTANCE.initialize(runtimeContext);
+
+ proxy = (IAsterixStateProxy) ncApplicationContext.getDistributedState();
+ if (proxy == null) {
+ throw new IllegalStateException("Metadata node cannot access distributed state");
+ }
+
+ // This is a special case, we just give the metadataNode directly.
+ // This way we can delay the registration of the metadataNode until
+ // it is completely initialized.
+ MetadataManager.INSTANCE = new MetadataManager(proxy, MetadataNode.INSTANCE);
+ MetadataBootstrap.startUniverse(((IAsterixPropertiesProvider) runtimeContext), ncApplicationContext,
+ systemState == SystemState.NEW_UNIVERSE);
+ MetadataBootstrap.startDDLRecovery();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Metadata node bound");
+ }
+ }
+
+ ExternalLibraryBootstrap.setUpExternaLibraries(isMetadataNode);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Starting lifecycle components");
+ }
+
+ Map<String, String> lifecycleMgmtConfiguration = new HashMap<String, String>();
+ String dumpPathKey = LifeCycleComponentManager.Config.DUMP_PATH_KEY;
+ String dumpPath = metadataProperties.getCoredumpPath(nodeId);
+ lifecycleMgmtConfiguration.put(dumpPathKey, dumpPath);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Coredump directory for NC is: " + dumpPath);
+ }
+ ILifeCycleComponentManager lccm = ncApplicationContext.getLifeCycleComponentManager();
+ lccm.configure(lifecycleMgmtConfiguration);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Configured:" + lccm);
+ }
+ ncApplicationContext.setStateDumpHandler(new AsterixStateDumpHandler(ncApplicationContext.getNodeId(), lccm
+ .getDumpPath(), lccm));
+
+ lccm.startAll();
+
+ IRecoveryManager recoveryMgr = runtimeContext.getTransactionSubsystem().getRecoveryManager();
+ recoveryMgr.checkpoint(true, NON_SHARP_CHECKPOINT_TARGET_LSN);
+
+ if (isMetadataNode) {
+ IMetadataNode stub = null;
+ stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, metadataRmiPort);
+ proxy.setMetadataNode(stub);
+ }
+
+ // Reclaim storage for temporary datasets.
+ String[] ioDevices = AsterixClusterProperties.INSTANCE.getIODevices(nodeId);
+ String[] nodeStores = metadataProperties.getStores().get(nodeId);
+ int numIoDevices = AsterixClusterProperties.INSTANCE.getNumberOfIODevices(nodeId);
+ for (int j = 0; j < nodeStores.length; j++) {
+ for (int k = 0; k < numIoDevices; k++) {
+ File f = new File(ioDevices[k] + File.separator + nodeStores[j] + File.separator + "temp");
+ f.delete();
+ }
+ }
+
+ // TODO
+ // reclaim storage for orphaned index artifacts in NCs.
+
+ }
+
+ private void updateOnNodeJoin() {
+ AsterixMetadataProperties metadataProperties = ((IAsterixPropertiesProvider) runtimeContext)
+ .getMetadataProperties();
+ if (!metadataProperties.getNodeNames().contains(nodeId)) {
+ metadataProperties.getNodeNames().add(nodeId);
+ Cluster cluster = AsterixClusterProperties.INSTANCE.getCluster();
+ String asterixInstanceName = cluster.getInstanceName();
+ AsterixTransactionProperties txnProperties = ((IAsterixPropertiesProvider) runtimeContext)
+ .getTransactionProperties();
+ Node self = null;
+ for (Node node : cluster.getSubstituteNodes().getNode()) {
+ String ncId = asterixInstanceName + "_" + node.getId();
+ if (ncId.equalsIgnoreCase(nodeId)) {
+ String storeDir = node.getStore() == null ? cluster.getStore() : node.getStore();
+ metadataProperties.getStores().put(nodeId, storeDir.split(","));
+
+ String coredumpPath = node.getLogDir() == null ? cluster.getLogDir() : node.getLogDir();
+ metadataProperties.getCoredumpPaths().put(nodeId, coredumpPath);
+
+ String txnLogDir = node.getTxnLogDir() == null ? cluster.getTxnLogDir() : node.getTxnLogDir();
+ txnProperties.getLogDirectories().put(nodeId, txnLogDir);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Store set to : " + storeDir);
+ LOGGER.info("Coredump dir set to : " + coredumpPath);
+ LOGGER.info("Transaction log dir set to :" + txnLogDir);
+ }
+ self = node;
+ break;
+ }
+ }
+ if (self != null) {
+ cluster.getSubstituteNodes().getNode().remove(self);
+ cluster.getNode().add(self);
+ } else {
+ throw new IllegalStateException("Unknown node joining the cluster");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java b/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java
new file mode 100644
index 0000000..c27f859
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/asterix/result/ResultReader.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.result;
+
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
+import edu.uci.ics.hyracks.api.comm.IFrame;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.dataset.DatasetJobRecord.Status;
+import edu.uci.ics.hyracks.api.dataset.IHyracksDataset;
+import edu.uci.ics.hyracks.api.dataset.IHyracksDatasetReader;
+import edu.uci.ics.hyracks.api.dataset.ResultSetId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
+
+public class ResultReader {
+ private final IHyracksDataset hyracksDataset;
+
+ private IHyracksDatasetReader reader;
+
+ private IFrameTupleAccessor frameTupleAccessor;
+
+ // Number of parallel result reader buffers
+ public static final int NUM_READERS = 1;
+
+ public static final int FRAME_SIZE = AsterixAppContextInfo.getInstance().getCompilerProperties().getFrameSize();
+
+ public ResultReader(IHyracksClientConnection hcc, IHyracksDataset hdc) throws Exception {
+ hyracksDataset = hdc;
+ }
+
+ public void open(JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
+ reader = hyracksDataset.createReader(jobId, resultSetId);
+ frameTupleAccessor = new ResultFrameTupleAccessor();
+ }
+
+ public Status getStatus() {
+ return reader.getResultStatus();
+ }
+
+ public int read(IFrame frame) throws HyracksDataException {
+ return reader.read(frame);
+ }
+
+ public IFrameTupleAccessor getFrameTupleAccessor() {
+ return frameTupleAccessor;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java b/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
new file mode 100644
index 0000000..3a4fd5f
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/asterix/result/ResultUtils.java
@@ -0,0 +1,343 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.result;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.http.ParseException;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.asterix.api.common.SessionConfig;
+import edu.uci.ics.asterix.api.common.SessionConfig.OutputFormat;
+import edu.uci.ics.asterix.api.http.servlet.APIServlet;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrame;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.comm.VSizeFrame;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.control.nc.resources.memory.FrameManager;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+
+public class ResultUtils {
+ private static final Charset UTF_8 = Charset.forName("UTF-8");
+
+ static Map<Character, String> HTML_ENTITIES = new HashMap<Character, String>();
+
+ static {
+ HTML_ENTITIES.put('"', """);
+ HTML_ENTITIES.put('&', "&");
+ HTML_ENTITIES.put('<', "<");
+ HTML_ENTITIES.put('>', ">");
+ }
+
+ public static String escapeHTML(String s) {
+ for (Character c : HTML_ENTITIES.keySet()) {
+ if (s.indexOf(c) >= 0) {
+ s = s.replace(c.toString(), HTML_ENTITIES.get(c));
+ }
+ }
+ return s;
+ }
+
+ public static void displayCSVHeader(ARecordType recordType, SessionConfig conf) {
+ // If HTML-ifying, we have to output this here before the header -
+ // pretty ugly
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("<h4>Results:</h4>");
+ conf.out().println("<pre>");
+ }
+
+ String[] fieldNames = recordType.getFieldNames();
+ boolean notfirst = false;
+ for (String name : fieldNames) {
+ if (notfirst) {
+ conf.out().print(',');
+ }
+ notfirst = true;
+ conf.out().print('"');
+ conf.out().print(name.replace("\"", "\"\""));
+ conf.out().print('"');
+ }
+ conf.out().print("\r\n");
+ }
+
+ public static FrameManager resultDisplayFrameMgr = new FrameManager(ResultReader.FRAME_SIZE);
+
+ public static void displayResults(ResultReader resultReader, SessionConfig conf)
+ throws HyracksDataException {
+ IFrameTupleAccessor fta = resultReader.getFrameTupleAccessor();
+
+ IFrame frame = new VSizeFrame(resultDisplayFrameMgr);
+ int bytesRead = resultReader.read(frame);
+ ByteBufferInputStream bbis = new ByteBufferInputStream();
+
+ // Whether we need to separate top-level ADM instances with commas
+ boolean need_commas = true;
+ // Whether this is the first instance being output
+ boolean notfirst = false;
+
+ // If we're outputting CSV with a header, the HTML header was already
+ // output by displayCSVHeader(), so skip it here
+ if (conf.is(SessionConfig.FORMAT_HTML) &&
+ ! (conf.fmt() == OutputFormat.CSV && conf.is(SessionConfig.FORMAT_CSV_HEADER))) {
+ conf.out().println("<h4>Results:</h4>");
+ conf.out().println("<pre>");
+ }
+
+ switch (conf.fmt()) {
+ case CSV:
+ need_commas = false;
+ break;
+ case JSON:
+ case ADM:
+ // Conveniently, JSON and ADM have the same syntax for an
+ // "ordered list", and our representation of the result of a
+ // statement is an ordered list of instances.
+ conf.out().print("[ ");
+ break;
+ }
+
+ if (bytesRead > 0) {
+ do {
+ try {
+ fta.reset(frame.getBuffer());
+ int last = fta.getTupleCount();
+ String result;
+ for (int tIndex = 0; tIndex < last; tIndex++) {
+ int start = fta.getTupleStartOffset(tIndex);
+ int length = fta.getTupleEndOffset(tIndex) - start;
+ bbis.setByteBuffer(frame.getBuffer(), start);
+ byte[] recordBytes = new byte[length];
+ int numread = bbis.read(recordBytes, 0, length);
+ if (conf.fmt() == OutputFormat.CSV) {
+ if ( (numread > 0) && (recordBytes[numread-1] == '\n') ) {
+ numread--;
+ }
+ }
+ result = new String(recordBytes, 0, numread, UTF_8);
+ if (need_commas && notfirst) {
+ conf.out().print(", ");
+ }
+ notfirst = true;
+ conf.out().print(result);
+ if (conf.fmt() == OutputFormat.CSV) {
+ conf.out().print("\r\n");
+ }
+ }
+ frame.getBuffer().clear();
+ } finally {
+ try {
+ bbis.close();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ } while (resultReader.read(frame) > 0);
+ }
+
+ conf.out().flush();
+
+ switch (conf.fmt()) {
+ case JSON:
+ case ADM:
+ conf.out().println(" ]");
+ break;
+ case CSV:
+ // Nothing to do
+ break;
+ }
+
+ if (conf.is(SessionConfig.FORMAT_HTML)) {
+ conf.out().println("</pre>");
+ }
+ }
+
+ public static JSONObject getErrorResponse(int errorCode, String errorMessage, String errorSummary,
+ String errorStackTrace) {
+ JSONObject errorResp = new JSONObject();
+ JSONArray errorArray = new JSONArray();
+ errorArray.put(errorCode);
+ errorArray.put(errorMessage);
+ try {
+ errorResp.put("error-code", errorArray);
+ if (!errorSummary.equals(""))
+ errorResp.put("summary", errorSummary);
+ if (!errorStackTrace.equals(""))
+ errorResp.put("stacktrace", errorStackTrace);
+ } catch (JSONException e) {
+ // TODO(madhusudancs): Figure out what to do when JSONException occurs while building the results.
+ }
+ return errorResp;
+ }
+
+ public static void webUIErrorHandler(PrintWriter out, Exception e) {
+ String errorTemplate = readTemplateFile("/webui/errortemplate.html", "%s\n%s\n%s");
+
+ String errorOutput = String.format(errorTemplate, escapeHTML(extractErrorMessage(e)),
+ escapeHTML(extractErrorSummary(e)), escapeHTML(extractFullStackTrace(e)));
+ out.println(errorOutput);
+ }
+
+ public static void webUIParseExceptionHandler(PrintWriter out, Throwable e, String query) {
+ String errorTemplate = readTemplateFile("/webui/errortemplate_message.html", "<pre class=\"error\">%s\n</pre>");
+
+ String errorOutput = String.format(errorTemplate, buildParseExceptionMessage(e, query));
+ out.println(errorOutput);
+ }
+
+ public static void apiErrorHandler(PrintWriter out, Exception e) {
+ int errorCode = 99;
+ if (e instanceof ParseException) {
+ errorCode = 2;
+ } else if (e instanceof AlgebricksException) {
+ errorCode = 3;
+ } else if (e instanceof HyracksDataException) {
+ errorCode = 4;
+ }
+
+ JSONObject errorResp = ResultUtils.getErrorResponse(errorCode, extractErrorMessage(e), extractErrorSummary(e),
+ extractFullStackTrace(e));
+ out.write(errorResp.toString());
+ }
+
+ public static String buildParseExceptionMessage(Throwable e, String query) {
+ StringBuilder errorMessage = new StringBuilder();
+ String message = e.getMessage();
+ message = message.replace("<", "<");
+ message = message.replace(">", ">");
+ errorMessage.append("SyntaxError: " + message + "\n");
+ int pos = message.indexOf("line");
+ if (pos > 0) {
+ Pattern p = Pattern.compile("\\d+");
+ Matcher m = p.matcher(message);
+ if (m.find(pos)) {
+ int lineNo = Integer.parseInt(message.substring(m.start(), m.end()));
+ String[] lines = query.split("\n");
+ if (lineNo > lines.length) {
+ errorMessage.append("===> <BLANK LINE> \n");
+ } else {
+ String line = lines[lineNo - 1];
+ errorMessage.append("==> " + line);
+ }
+ }
+ }
+ return errorMessage.toString();
+ }
+
+ private static Throwable getRootCause(Throwable cause) {
+ Throwable nextCause = cause.getCause();
+ while (nextCause != null) {
+ cause = nextCause;
+ nextCause = cause.getCause();
+ }
+ return cause;
+ }
+
+ /**
+ * Extract the message in the root cause of the stack trace:
+ *
+ * @param e
+ * @return error message string.
+ */
+ private static String extractErrorMessage(Throwable e) {
+ Throwable cause = getRootCause(e);
+ String fullyQualifiedExceptionClassName = cause.getClass().getName();
+ String[] hierarchySplits = fullyQualifiedExceptionClassName.split("\\.");
+ //try returning the class without package qualification
+ String exceptionClassName = hierarchySplits[hierarchySplits.length - 1];
+ String localizedMessage = cause.getLocalizedMessage();
+ if(localizedMessage == null){
+ localizedMessage = "Internal error. Please check instance logs for further details.";
+ }
+ return localizedMessage + " [" + exceptionClassName + "]";
+ }
+
+ /**
+ * Extract the meaningful part of a stack trace:
+ * a. the causes in the stack trace hierarchy
+ * b. the top exception for each cause
+ *
+ * @param e
+ * @return the contacted message containing a and b.
+ */
+ private static String extractErrorSummary(Throwable e) {
+ StringBuilder errorMessageBuilder = new StringBuilder();
+ Throwable cause = e;
+ errorMessageBuilder.append(cause.getLocalizedMessage());
+ while (cause != null) {
+ StackTraceElement[] stackTraceElements = cause.getStackTrace();
+ errorMessageBuilder.append(stackTraceElements.length > 0 ? "\n caused by: " + stackTraceElements[0] : "");
+ cause = cause.getCause();
+ }
+ return errorMessageBuilder.toString();
+ }
+
+ /**
+ * Extract the full stack trace:
+ *
+ * @param e
+ * @return the string containing the full stack trace of the error.
+ */
+ private static String extractFullStackTrace(Throwable e) {
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter printWriter = new PrintWriter(stringWriter);
+ e.printStackTrace(printWriter);
+ return stringWriter.toString();
+ }
+
+ /**
+ * Read the template file which is stored as a resource and return its content. If the file does not exist or is
+ * not readable return the default template string.
+ *
+ * @param path
+ * The path to the resource template file
+ * @param defaultTemplate
+ * The default template string if the template file does not exist or is not readable
+ * @return The template string to be used to render the output.
+ */
+ private static String readTemplateFile(String path, String defaultTemplate) {
+ String errorTemplate = defaultTemplate;
+ try {
+ String resourcePath = "/webui/errortemplate_message.html";
+ InputStream is = APIServlet.class.getResourceAsStream(resourcePath);
+ InputStreamReader isr = new InputStreamReader(is);
+ StringBuilder sb = new StringBuilder();
+ BufferedReader br = new BufferedReader(isr);
+ String line = br.readLine();
+
+ while (line != null) {
+ sb.append(line);
+ line = br.readLine();
+ }
+ errorTemplate = sb.toString();
+ } catch (IOException ioe) {
+ // If there is an IOException reading the error template html file, default value of error template is used.
+ }
+ return errorTemplate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java b/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
new file mode 100644
index 0000000..a788c37
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.misc;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+
+public class ConstantTupleSourceOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ private int[] fieldSlots;
+ private byte[] tupleData;
+ private int tupleSize;
+
+ public ConstantTupleSourceOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc, int[] fieldSlots,
+ byte[] tupleData, int tupleSize) {
+ super(spec, 0, 1);
+ this.tupleData = tupleData;
+ this.fieldSlots = fieldSlots;
+ this.tupleSize = tupleSize;
+ recordDescriptors[0] = recDesc;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ return new ConstantTupleSourceOperatorNodePushable(ctx, fieldSlots, tupleData, tupleSize);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java b/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
new file mode 100644
index 0000000..ba75fb0
--- /dev/null
+++ b/asterix-app/src/main/java/org/apache/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorNodePushable.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.dataflow.std.misc;
+
+import edu.uci.ics.hyracks.api.comm.VSizeFrame;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+
+public class ConstantTupleSourceOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
+ private IHyracksTaskContext ctx;
+
+ private int[] fieldSlots;
+ private byte[] tupleData;
+ private int tupleSize;
+
+ public ConstantTupleSourceOperatorNodePushable(IHyracksTaskContext ctx, int[] fieldSlots, byte[] tupleData,
+ int tupleSize) {
+ super();
+ this.fieldSlots = fieldSlots;
+ this.tupleData = tupleData;
+ this.tupleSize = tupleSize;
+ this.ctx = ctx;
+ }
+
+ @Override
+ public void initialize() throws HyracksDataException {
+ FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
+ if (fieldSlots != null && tupleData != null && tupleSize > 0)
+ appender.append(fieldSlots, tupleData, 0, tupleSize);
+ writer.open();
+ try {
+ appender.flush(writer, true);
+ }
+ finally {
+ writer.close();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/api/http/servlet/ConnectorAPIServletTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/api/http/servlet/ConnectorAPIServletTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/api/http/servlet/ConnectorAPIServletTest.java
deleted file mode 100644
index 7f5f480..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/api/http/servlet/ConnectorAPIServletTest.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.api.http.servlet;
-
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.servlet.ServletConfig;
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import junit.extensions.PA;
-import junit.framework.Assert;
-
-import org.json.JSONArray;
-import org.json.JSONObject;
-import org.json.JSONTokener;
-import org.junit.Test;
-
-import edu.uci.ics.asterix.feeds.CentralFeedManager;
-import edu.uci.ics.asterix.metadata.MetadataManager;
-import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
-import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
-import edu.uci.ics.asterix.metadata.entities.Dataset;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.BuiltinType;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.util.JSONDeserializerForTypes;
-import edu.uci.ics.asterix.test.runtime.ExecutionTest;
-import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
-import edu.uci.ics.hyracks.api.client.NodeControllerInfo;
-import edu.uci.ics.hyracks.api.comm.NetworkAddress;
-import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
-
-@SuppressWarnings("deprecation")
-public class ConnectorAPIServletTest {
-
- @Test
- public void testGet() throws Exception {
- // Starts test asterixdb cluster.
- ExecutionTest.setUp();
-
- // Configures a test connector api servlet.
- ConnectorAPIServlet servlet = spy(new ConnectorAPIServlet());
- ServletConfig mockServletConfig = mock(ServletConfig.class);
- servlet.init(mockServletConfig);
- Map<String, NodeControllerInfo> nodeMap = new HashMap<String, NodeControllerInfo>();
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- PrintWriter outputWriter = new PrintWriter(outputStream);
-
- // Creates mocks.
- ServletContext mockContext = mock(ServletContext.class);
- IHyracksClientConnection mockHcc = mock(IHyracksClientConnection.class);
- NodeControllerInfo mockInfo1 = mock(NodeControllerInfo.class);
- NodeControllerInfo mockInfo2 = mock(NodeControllerInfo.class);
- HttpServletRequest mockRequest = mock(HttpServletRequest.class);
- HttpServletResponse mockResponse = mock(HttpServletResponse.class);
-
- // Sets up mock returns.
- when(servlet.getServletContext()).thenReturn(mockContext);
- when(mockRequest.getParameter("dataverseName")).thenReturn("Metadata");
- when(mockRequest.getParameter("datasetName")).thenReturn("Dataset");
- when(mockResponse.getWriter()).thenReturn(outputWriter);
- when(mockContext.getAttribute(anyString())).thenReturn(mockHcc);
- when(mockHcc.getNodeControllerInfos()).thenReturn(nodeMap);
- when(mockInfo1.getNetworkAddress()).thenReturn(new NetworkAddress("127.0.0.1", 3099));
- when(mockInfo2.getNetworkAddress()).thenReturn(new NetworkAddress("127.0.0.2", 3099));
-
- // Calls ConnectorAPIServlet.formResponseObject.
- nodeMap.put("nc1", mockInfo1);
- nodeMap.put("nc2", mockInfo2);
- servlet.doGet(mockRequest, mockResponse);
-
- // Constructs the actual response.
- JSONTokener tokener = new JSONTokener(new InputStreamReader(
- new ByteArrayInputStream(outputStream.toByteArray())));
- JSONObject actualResponse = new JSONObject(tokener);
-
- // Checks the data type of the dataset.
- String primaryKey = actualResponse.getString("keys");
- Assert.assertEquals("DataverseName,DatasetName", primaryKey);
- ARecordType recordType = (ARecordType) JSONDeserializerForTypes.convertFromJSON((JSONObject) actualResponse
- .get("type"));
- Assert.assertEquals(getMetadataRecordType("Metadata", "Dataset"), recordType);
-
- // Checks the correctness of results.
- JSONArray splits = actualResponse.getJSONArray("splits");
- String path = ((JSONObject) splits.get(0)).getString("path");
- Assert.assertTrue(path.endsWith("Metadata/Dataset_idx_Dataset"));
-
- // Tears down the asterixdb cluster.
- ExecutionTest.tearDown();
- }
-
- @Test
- public void testFormResponseObject() throws Exception {
- ConnectorAPIServlet servlet = new ConnectorAPIServlet();
- JSONObject actualResponse = new JSONObject();
- FileSplit[] splits = new FileSplit[2];
- splits[0] = new FileSplit("nc1", "foo1");
- splits[1] = new FileSplit("nc2", "foo2");
- Map<String, NodeControllerInfo> nodeMap = new HashMap<String, NodeControllerInfo>();
- NodeControllerInfo mockInfo1 = mock(NodeControllerInfo.class);
- NodeControllerInfo mockInfo2 = mock(NodeControllerInfo.class);
-
- // Sets up mock returns.
- when(mockInfo1.getNetworkAddress()).thenReturn(new NetworkAddress("127.0.0.1", 3099));
- when(mockInfo2.getNetworkAddress()).thenReturn(new NetworkAddress("127.0.0.2", 3099));
-
- String[] fieldNames = new String[] { "a1", "a2" };
- IAType[] fieldTypes = new IAType[] { BuiltinType.ABOOLEAN, BuiltinType.ADAYTIMEDURATION };
- ARecordType recordType = new ARecordType("record", fieldNames, fieldTypes, true);
- String primaryKey = "a1";
-
- // Calls ConnectorAPIServlet.formResponseObject.
- nodeMap.put("nc1", mockInfo1);
- nodeMap.put("nc2", mockInfo2);
- PA.invokeMethod(servlet,
- "formResponseObject(org.json.JSONObject, edu.uci.ics.hyracks.dataflow.std.file.FileSplit[], "
- + "edu.uci.ics.asterix.om.types.ARecordType, java.lang.String, java.util.Map)", actualResponse,
- splits, recordType, primaryKey, nodeMap);
-
- // Constructs expected response.
- JSONObject expectedResponse = new JSONObject();
- expectedResponse.put("keys", primaryKey);
- expectedResponse.put("type", recordType.toJSON());
- JSONArray splitsArray = new JSONArray();
- JSONObject element1 = new JSONObject();
- element1.put("ip", "127.0.0.1");
- element1.put("path", splits[0].getLocalFile().getFile().getAbsolutePath());
- JSONObject element2 = new JSONObject();
- element2.put("ip", "127.0.0.2");
- element2.put("path", splits[1].getLocalFile().getFile().getAbsolutePath());
- splitsArray.put(element1);
- splitsArray.put(element2);
- expectedResponse.put("splits", splitsArray);
-
- // Checks results.
- Assert.assertEquals(actualResponse.toString(), expectedResponse.toString());
- }
-
- private ARecordType getMetadataRecordType(String dataverseName, String datasetName) throws Exception {
- MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
- // Retrieves file splits of the dataset.
- AqlMetadataProvider metadataProvider = new AqlMetadataProvider(null, CentralFeedManager.getInstance());
- metadataProvider.setMetadataTxnContext(mdTxnCtx);
- Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
- ARecordType recordType = (ARecordType) metadataProvider.findType(dataverseName, dataset.getItemTypeName());
- // Metadata transaction commits.
- MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
- return recordType;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestCase.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestCase.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestCase.java
deleted file mode 100644
index da5068a..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestCase.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.aql;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.Reader;
-import java.io.StringWriter;
-import java.io.UnsupportedEncodingException;
-import java.util.List;
-
-import junit.framework.TestCase;
-
-import org.junit.Test;
-
-import edu.uci.ics.asterix.aql.base.Statement;
-import edu.uci.ics.asterix.aql.parser.AQLParser;
-import edu.uci.ics.asterix.aql.parser.ParseException;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public class AQLTestCase extends TestCase {
-
- private final File queryFile;
-
- AQLTestCase(File queryFile) {
- super("testAQL");
- this.queryFile = queryFile;
- }
-
- @Test
- public void testAQL() throws UnsupportedEncodingException, FileNotFoundException, ParseException, AsterixException,
- AlgebricksException {
- Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(queryFile), "UTF-8"));
- AQLParser parser = new AQLParser(fis);
- List<Statement> statements;
- GlobalConfig.ASTERIX_LOGGER.info(queryFile.toString());
- try {
- statements = parser.parse();
- } catch (ParseException e) {
- GlobalConfig.ASTERIX_LOGGER.warning("Failed while testing file " + fis);
- StringWriter sw = new StringWriter();
- PrintWriter writer = new PrintWriter(sw);
- e.printStackTrace(writer);
- GlobalConfig.ASTERIX_LOGGER.warning(sw.toString());
- throw new ParseException("Parsing " + queryFile.toString());
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestSuite.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestSuite.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestSuite.java
deleted file mode 100644
index c536113..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/aql/AQLTestSuite.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.aql;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.UnsupportedEncodingException;
-
-import junit.framework.Test;
-import junit.framework.TestSuite;
-
-import org.apache.commons.lang3.StringUtils;
-
-import edu.uci.ics.asterix.aql.parser.ParseException;
-
-public class AQLTestSuite extends TestSuite {
- private static String AQLTS_PATH = StringUtils.join(new String[] { "src", "test", "resources", "AQLTS",
- "queries" + File.separator }, File.separator);
- private static String AQLTS_SQL_LIKE_PATH = StringUtils.join(new String[] { "src", "test", "resources", "AQLTS",
- "queries-sql-like" + File.separator }, File.separator);
-
- public static Test suite() throws ParseException, UnsupportedEncodingException, FileNotFoundException {
- File testData = new File(AQLTS_PATH);
- File[] queries = testData.listFiles();
- TestSuite testSuite = new TestSuite();
- for (File file : queries) {
- if (file.isFile()) {
- testSuite.addTest(new AQLTestCase(file));
- }
- }
- testData = new File(AQLTS_SQL_LIKE_PATH);
- queries = testData.listFiles();
- for (File file : queries) {
- if (file.isFile()) {
- testSuite.addTest(new AQLTestCase(file));
- }
- }
-
- return testSuite;
-
- }
-
- public static void main(String args[]) throws Throwable {
- junit.textui.TestRunner.run(AQLTestSuite.suite());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/common/TestHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/common/TestHelper.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/common/TestHelper.java
deleted file mode 100644
index 827b40b..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/common/TestHelper.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.common;
-
-import java.util.List;
-
-public final class TestHelper {
-
- public static boolean isInPrefixList(List<String> prefixList, String s) {
- for (String s2 : prefixList) {
- if (s.startsWith(s2)) {
- return true;
- }
- }
- return false;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/dml/DmlTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/dml/DmlTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/dml/DmlTest.java
deleted file mode 100644
index 9222081..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/dml/DmlTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.dml;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.Reader;
-
-import org.junit.Test;
-
-import edu.uci.ics.asterix.api.common.AsterixHyracksIntegrationUtil;
-import edu.uci.ics.asterix.api.java.AsterixJavaClient;
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.test.aql.TestsUtils;
-import edu.uci.ics.asterix.test.base.AsterixTestHelper;
-
-public class DmlTest {
-
- private static final String[] ASTERIX_DATA_DIRS = new String[] { "nc1data", "nc2data" };
- private static final String PATH_ACTUAL = "dmltest" + File.separator;
- private static final String SEPARATOR = File.separator;
- private static final String PATH_BASE = "src" + SEPARATOR + "test" + SEPARATOR + "resources" + SEPARATOR + "dmlts"
- + SEPARATOR;
- private static final String PATH_EXPECTED = PATH_BASE + "results" + SEPARATOR;
- private static final String PATH_SCRIPTS = PATH_BASE + "scripts" + SEPARATOR;
- private static final String LOAD_FOR_ENLIST_FILE = PATH_SCRIPTS + "load-cust.aql";
- private static final String ENLIST_FILE = PATH_SCRIPTS + "enlist-scan-cust.aql";
-
- private static final PrintWriter ERR = new PrintWriter(System.err);
-
- @Test
- public void enlistTest() throws Exception {
- File outdir = new File(PATH_ACTUAL);
- if (outdir.exists()) {
- AsterixTestHelper.deleteRec(outdir);
- }
- outdir.mkdirs();
-
- AsterixHyracksIntegrationUtil.init();
- Reader loadReader = new BufferedReader(
- new InputStreamReader(new FileInputStream(LOAD_FOR_ENLIST_FILE), "UTF-8"));
- AsterixJavaClient asterixLoad = new AsterixJavaClient(
- AsterixHyracksIntegrationUtil.getHyracksClientConnection(), loadReader, ERR);
- try {
- asterixLoad.compile(true, false, false, false, false, true, false);
- } catch (AsterixException e) {
- throw new Exception("Compile ERROR for " + LOAD_FOR_ENLIST_FILE + ": " + e.getMessage(), e);
- } finally {
- loadReader.close();
- }
- asterixLoad.execute();
- File enlistFile = new File(ENLIST_FILE);
- int dot = enlistFile.getName().lastIndexOf('.');
- String resultFileName = enlistFile.getName().substring(0, dot + 1) + ".adm";
- File expectedFile = new File(PATH_EXPECTED + SEPARATOR + resultFileName);
- File actualFile = new File(PATH_ACTUAL + SEPARATOR + resultFileName);
- // Khurram
- //TestsUtils.runScriptAndCompareWithResult(AsterixHyracksIntegrationUtil.getHyracksClientConnection(),
- //enlistFile, ERR, expectedFile, actualFile);
-
- AsterixHyracksIntegrationUtil.deinit();
- for (String d : ASTERIX_DATA_DIRS) {
- TestsUtils.deleteRec(new File(d));
- }
- outdir.delete();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java
deleted file mode 100644
index 069c5aa..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/metadata/MetadataTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.metadata;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import edu.uci.ics.asterix.api.common.AsterixHyracksIntegrationUtil;
-import edu.uci.ics.asterix.common.config.AsterixPropertiesAccessor;
-import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
-import edu.uci.ics.asterix.test.aql.TestsUtils;
-import edu.uci.ics.asterix.testframework.context.TestCaseContext;
-
-/**
- * Executes the Metadata tests.
- */
-@RunWith(Parameterized.class)
-public class MetadataTest {
-
- private TestCaseContext tcCtx;
-
- private static final String PATH_ACTUAL = "mdtest" + File.separator;
- private static final String PATH_BASE = StringUtils.join(new String[] { "src", "test", "resources",
- "metadata" + File.separator }, File.separator);
- private static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
-
- private static AsterixTransactionProperties txnProperties;
-
- @BeforeClass
- public static void setUp() throws Exception {
- System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
- File outdir = new File(PATH_ACTUAL);
- outdir.mkdirs();
-
- AsterixPropertiesAccessor apa = new AsterixPropertiesAccessor();
- txnProperties = new AsterixTransactionProperties(apa);
-
- deleteTransactionLogs();
-
- AsterixHyracksIntegrationUtil.init();
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- AsterixHyracksIntegrationUtil.deinit();
- File outdir = new File(PATH_ACTUAL);
- File[] files = outdir.listFiles();
- if (files == null || files.length == 0) {
- outdir.delete();
- }
-
- // clean up the files written by the ASTERIX storage manager
- for (String d : AsterixHyracksIntegrationUtil.getDataDirs()) {
- TestsUtils.deleteRec(new File(d));
- }
- }
-
- private static void deleteTransactionLogs() throws Exception {
- for (String ncId : AsterixHyracksIntegrationUtil.getNcNames()) {
- File log = new File(txnProperties.getLogDirectory(ncId));
- if (log.exists()) {
- FileUtils.deleteDirectory(log);
- }
- }
- }
-
- @Parameters
- public static Collection<Object[]> tests() throws Exception {
- Collection<Object[]> testArgs = new ArrayList<Object[]>();
- TestCaseContext.Builder b = new TestCaseContext.Builder();
- for (TestCaseContext ctx : b.build(new File(PATH_BASE))) {
- testArgs.add(new Object[] { ctx });
- }
- return testArgs;
- }
-
- public MetadataTest(TestCaseContext tcCtx) {
- this.tcCtx = tcCtx;
- }
-
- @Test
- public void test() throws Exception {
- TestsUtils.executeTest(PATH_ACTUAL, tcCtx, null, false);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/optimizer/OptimizerTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/optimizer/OptimizerTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/optimizer/OptimizerTest.java
deleted file mode 100644
index 58531aa..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/optimizer/OptimizerTest.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.optimizer;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.Reader;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.logging.Logger;
-
-import org.apache.commons.io.FileUtils;
-import org.junit.AfterClass;
-import org.junit.Assume;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.internal.AssumptionViolatedException;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import edu.uci.ics.asterix.api.common.AsterixHyracksIntegrationUtil;
-import edu.uci.ics.asterix.api.java.AsterixJavaClient;
-import edu.uci.ics.asterix.common.config.AsterixPropertiesAccessor;
-import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.external.dataset.adapter.FileSystemBasedAdapter;
-import edu.uci.ics.asterix.external.util.IdentitiyResolverFactory;
-import edu.uci.ics.asterix.test.base.AsterixTestHelper;
-import edu.uci.ics.asterix.test.common.TestHelper;
-
-@RunWith(Parameterized.class)
-public class OptimizerTest {
-
- private static final Logger LOGGER = Logger.getLogger(OptimizerTest.class.getName());
-
- private static final String SEPARATOR = File.separator;
- private static final String EXTENSION_QUERY = "aql";
- private static final String EXTENSION_RESULT = "plan";
- private static final String FILENAME_IGNORE = "ignore.txt";
- private static final String FILENAME_ONLY = "only.txt";
- private static final String PATH_BASE = "src" + SEPARATOR + "test" + SEPARATOR + "resources" + SEPARATOR
- + "optimizerts" + SEPARATOR;
- private static final String PATH_QUERIES = PATH_BASE + "queries" + SEPARATOR;
- private static final String PATH_EXPECTED = PATH_BASE + "results" + SEPARATOR;
- private static final String PATH_ACTUAL = "opttest" + SEPARATOR;
-
- private static final ArrayList<String> ignore = AsterixTestHelper.readFile(FILENAME_IGNORE, PATH_BASE);
- private static final ArrayList<String> only = AsterixTestHelper.readFile(FILENAME_ONLY, PATH_BASE);
- private static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
-
- private static AsterixTransactionProperties txnProperties;
-
- @BeforeClass
- public static void setUp() throws Exception {
- // File outdir = new File(PATH_ACTUAL);
- // outdir.mkdirs();
-
- System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
- File outdir = new File(PATH_ACTUAL);
- outdir.mkdirs();
-
- AsterixPropertiesAccessor apa = new AsterixPropertiesAccessor();
- txnProperties = new AsterixTransactionProperties(apa);
-
- deleteTransactionLogs();
-
- AsterixHyracksIntegrationUtil.init();
- // Set the node resolver to be the identity resolver that expects node names
- // to be node controller ids; a valid assumption in test environment.
- System.setProperty(FileSystemBasedAdapter.NODE_RESOLVER_FACTORY_PROPERTY,
- IdentitiyResolverFactory.class.getName());
- }
-
- private static void deleteTransactionLogs() throws Exception {
- for (String ncId : AsterixHyracksIntegrationUtil.getNcNames()) {
- File log = new File(txnProperties.getLogDirectory(ncId));
- if (log.exists()) {
- FileUtils.deleteDirectory(log);
- }
- }
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- // _bootstrap.stop();
- File outdir = new File(PATH_ACTUAL);
- File[] files = outdir.listFiles();
- if (files == null || files.length == 0) {
- outdir.delete();
- }
- AsterixHyracksIntegrationUtil.deinit();
- }
-
- private static void suiteBuild(File dir, Collection<Object[]> testArgs, String path) {
- for (File file : dir.listFiles()) {
- if (file.isDirectory() && !file.getName().startsWith(".")) {
- suiteBuild(file, testArgs, path + file.getName() + SEPARATOR);
- }
- if (file.isFile() && file.getName().endsWith(EXTENSION_QUERY)
- // && !ignore.contains(path + file.getName())
- ) {
- String resultFileName = AsterixTestHelper.extToResExt(file.getName(), EXTENSION_RESULT);
- File expectedFile = new File(PATH_EXPECTED + path + resultFileName);
- File actualFile = new File(PATH_ACTUAL + SEPARATOR + path.replace(SEPARATOR, "_") + resultFileName);
- testArgs.add(new Object[] { file, expectedFile, actualFile });
- }
- }
- }
-
- @Parameters
- public static Collection<Object[]> tests() {
- Collection<Object[]> testArgs = new ArrayList<Object[]>();
- suiteBuild(new File(PATH_QUERIES), testArgs, "");
- return testArgs;
- }
-
- private File actualFile;
- private File expectedFile;
- private File queryFile;
-
- public OptimizerTest(File queryFile, File expectedFile, File actualFile) {
- this.queryFile = queryFile;
- this.expectedFile = expectedFile;
- this.actualFile = actualFile;
- }
-
- @Test
- public void test() throws Exception {
- try {
- String queryFileShort = queryFile.getPath().substring(PATH_QUERIES.length())
- .replace(SEPARATOR.charAt(0), '/');
- if (!only.isEmpty()) {
- boolean toRun = TestHelper.isInPrefixList(only, queryFileShort);
- if (!toRun) {
- LOGGER.info("SKIP TEST: \"" + queryFile.getPath()
- + "\" \"only.txt\" not empty and not in \"only.txt\".");
- }
- Assume.assumeTrue(toRun);
- }
- boolean skipped = TestHelper.isInPrefixList(ignore, queryFileShort);
- if (skipped) {
- LOGGER.info("SKIP TEST: \"" + queryFile.getPath() + "\" in \"ignore.txt\".");
- }
- Assume.assumeTrue(!skipped);
-
- LOGGER.info("RUN TEST: \"" + queryFile.getPath() + "\"");
- Reader query = new BufferedReader(new InputStreamReader(new FileInputStream(queryFile), "UTF-8"));
- PrintWriter plan = new PrintWriter(actualFile);
- AsterixJavaClient asterix = new AsterixJavaClient(
- AsterixHyracksIntegrationUtil.getHyracksClientConnection(), query, plan);
- try {
- asterix.compile(true, false, false, true, true, false, false);
- } catch (AsterixException e) {
- plan.close();
- query.close();
- throw new Exception("Compile ERROR for " + queryFile + ": " + e.getMessage(), e);
- }
- plan.close();
- query.close();
-
- BufferedReader readerExpected = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFile),
- "UTF-8"));
- BufferedReader readerActual = new BufferedReader(new InputStreamReader(new FileInputStream(actualFile),
- "UTF-8"));
-
- String lineExpected, lineActual;
- int num = 1;
- try {
- while ((lineExpected = readerExpected.readLine()) != null) {
- lineActual = readerActual.readLine();
- // Assert.assertEquals(lineExpected, lineActual);
- if (lineActual == null) {
- throw new Exception("Result for " + queryFile + " changed at line " + num + ":\n< "
- + lineExpected + "\n> ");
- }
- if (!lineExpected.equals(lineActual)) {
- throw new Exception("Result for " + queryFile + " changed at line " + num + ":\n< "
- + lineExpected + "\n> " + lineActual);
- }
- ++num;
- }
- lineActual = readerActual.readLine();
- // Assert.assertEquals(null, lineActual);
- if (lineActual != null) {
- throw new Exception("Result for " + queryFile + " changed at line " + num + ":\n< \n> "
- + lineActual);
- }
- LOGGER.info("Test \"" + queryFile.getPath() + "\" PASSED!");
- actualFile.delete();
- } finally {
- readerExpected.close();
- readerActual.close();
- }
- } catch (Exception e) {
- if (!(e instanceof AssumptionViolatedException)) {
- LOGGER.severe("Test \"" + queryFile.getPath() + "\" FAILED!");
- throw new Exception("Test \"" + queryFile.getPath() + "\" FAILED!", e);
- } else {
- throw e;
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java
deleted file mode 100644
index 8547eb0..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/ExecutionTest.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.runtime;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-import edu.uci.ics.asterix.api.common.AsterixHyracksIntegrationUtil;
-import edu.uci.ics.asterix.common.config.AsterixPropertiesAccessor;
-import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
-import edu.uci.ics.asterix.external.dataset.adapter.FileSystemBasedAdapter;
-import edu.uci.ics.asterix.external.util.IdentitiyResolverFactory;
-import edu.uci.ics.asterix.test.aql.TestsUtils;
-import edu.uci.ics.asterix.testframework.context.TestCaseContext;
-
-/**
- * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'.
- */
-@RunWith(Parameterized.class)
-public class ExecutionTest {
-
- protected static final Logger LOGGER = Logger.getLogger(ExecutionTest.class.getName());
-
- protected static final String PATH_ACTUAL = "rttest" + File.separator;
- protected static final String PATH_BASE = StringUtils.join(
- new String[] { "src", "test", "resources", "runtimets" }, File.separator);
-
- protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
-
- protected static AsterixTransactionProperties txnProperties;
-
- @BeforeClass
- public static void setUp() throws Exception {
- System.out.println("Starting setup");
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Starting setup");
- }
- System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY, TEST_CONFIG_FILE_NAME);
- File outdir = new File(PATH_ACTUAL);
- outdir.mkdirs();
-
- AsterixPropertiesAccessor apa = new AsterixPropertiesAccessor();
- txnProperties = new AsterixTransactionProperties(apa);
-
- deleteTransactionLogs();
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("initializing pseudo cluster");
- }
- AsterixHyracksIntegrationUtil.init();
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("initializing HDFS");
- }
-
- HDFSCluster.getInstance().setup();
-
- // Set the node resolver to be the identity resolver that expects node names
- // to be node controller ids; a valid assumption in test environment.
- System.setProperty(FileSystemBasedAdapter.NODE_RESOLVER_FACTORY_PROPERTY,
- IdentitiyResolverFactory.class.getName());
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- AsterixHyracksIntegrationUtil.deinit();
- File outdir = new File(PATH_ACTUAL);
- File[] files = outdir.listFiles();
- if (files == null || files.length == 0) {
- outdir.delete();
- }
- // clean up the files written by the ASTERIX storage manager
- for (String d : AsterixHyracksIntegrationUtil.getDataDirs()) {
- TestsUtils.deleteRec(new File(d));
- }
- HDFSCluster.getInstance().cleanup();
- }
-
- private static void deleteTransactionLogs() throws Exception {
- for (String ncId : AsterixHyracksIntegrationUtil.getNcNames()) {
- File log = new File(txnProperties.getLogDirectory(ncId));
- if (log.exists()) {
- FileUtils.deleteDirectory(log);
- }
- }
- }
-
- @Parameters
- public static Collection<Object[]> tests() throws Exception {
- Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.ONLY_TESTSUITE_XML_NAME);
- if (testArgs.size() == 0) {
- testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME);
- }
- return testArgs;
- }
-
- protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception {
- Collection<Object[]> testArgs = new ArrayList<Object[]>();
- TestCaseContext.Builder b = new TestCaseContext.Builder();
- for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) {
- testArgs.add(new Object[] { ctx });
- }
- return testArgs;
-
- }
-
- protected TestCaseContext tcCtx;
-
- public ExecutionTest(TestCaseContext tcCtx) {
- this.tcCtx = tcCtx;
- }
-
- @Test
- public void test() throws Exception {
- TestsUtils.executeTest(PATH_ACTUAL, tcCtx, null, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java
deleted file mode 100644
index 9463657..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/HDFSCluster.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.runtime;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-
-import edu.uci.ics.asterix.external.dataset.adapter.HDFSAdapter;
-
-/**
- * Manages a Mini (local VM) HDFS cluster with a configured number of datanodes.
- *
- * @author ramangrover29
- */
-@SuppressWarnings("deprecation")
-public class HDFSCluster {
-
- private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf";
- private static final int nameNodePort = 31888;
- private static final String DATA_PATH = "data/hdfs";
- private static final String HDFS_PATH = "/asterix";
- private static final HDFSCluster INSTANCE = new HDFSCluster();
-
- private MiniDFSCluster dfsCluster;
- private int numDataNodes = 2;
- private JobConf conf = new JobConf();
- private FileSystem dfs;
-
- public static HDFSCluster getInstance() {
- return INSTANCE;
- }
-
- private HDFSCluster() {
-
- }
-
- /**
- * Instantiates the (Mini) DFS Cluster with the configured number of datanodes.
- * Post instantiation, data is laoded to HDFS.
- * Called prior to running the Runtime test suite.
- */
- public void setup() throws Exception {
- conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
- conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
- conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
- cleanupLocal();
- //this constructor is deprecated in hadoop 2x
- //dfsCluster = new MiniDFSCluster(nameNodePort, conf, numDataNodes, true, true, StartupOption.REGULAR, null);
- MiniDFSCluster.Builder build = new MiniDFSCluster.Builder(conf);
- build.nameNodePort(nameNodePort);
- build.numDataNodes(numDataNodes);
- build.startupOption(StartupOption.REGULAR);
- dfsCluster = build.build();
- dfs = FileSystem.get(conf);
- loadData();
- }
-
- private void loadData() throws IOException {
- Path destDir = new Path(HDFS_PATH);
- dfs.mkdirs(destDir);
- File srcDir = new File(DATA_PATH);
- File[] listOfFiles = srcDir.listFiles();
- for (File srcFile : listOfFiles) {
- Path path = new Path(srcFile.getAbsolutePath());
- dfs.copyFromLocalFile(path, destDir);
- }
- }
-
- private void cleanupLocal() throws IOException {
- // cleanup artifacts created on the local file system
- FileSystem lfs = FileSystem.getLocal(new Configuration());
- lfs.delete(new Path("build"), true);
- System.setProperty("hadoop.log.dir", "logs");
- }
-
- public void cleanup() throws Exception {
- if (dfsCluster != null) {
- dfsCluster.shutdown();
- cleanupLocal();
- }
- }
-
- public static void main(String[] args) throws Exception {
- HDFSCluster cluster = new HDFSCluster();
- cluster.setup();
- JobConf conf = configureJobConf();
- FileSystem fs = FileSystem.get(conf);
- InputSplit[] inputSplits = conf.getInputFormat().getSplits(conf, 0);
- for (InputSplit split : inputSplits) {
- System.out.println("split :" + split);
- }
- // cluster.cleanup();
- }
-
- private static JobConf configureJobConf() throws Exception {
- JobConf conf = new JobConf();
- String hdfsUrl = "hdfs://127.0.0.1:31888";
- String hdfsPath = "/asterix/extrasmalltweets.txt";
- conf.set("fs.default.name", hdfsUrl);
- conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
- conf.setClassLoader(HDFSAdapter.class.getClassLoader());
- conf.set("mapred.input.dir", hdfsPath);
- conf.set("mapred.input.format.class", "org.apache.hadoop.mapred.TextInputFormat");
- return conf;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/RepeatedTest.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/RepeatedTest.java b/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/RepeatedTest.java
deleted file mode 100644
index faeb688..0000000
--- a/asterix-app/src/test/java/edu/uci/ics/asterix/test/runtime/RepeatedTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.test.runtime;
-
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-import java.util.Collection;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.MethodRule;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-import org.junit.runners.model.FrameworkMethod;
-import org.junit.runners.model.Statement;
-
-import edu.uci.ics.asterix.test.aql.TestsUtils;
-import edu.uci.ics.asterix.test.runtime.RepeatRule.Repeat;
-import edu.uci.ics.asterix.testframework.context.TestCaseContext;
-
-/**
- * Runs runtime test cases that have been identified in the repeatedtestsuite.xml.
- *
- * Each test is run 10000 times.
- */
-class RepeatRule implements MethodRule {
-
- @Retention(RetentionPolicy.RUNTIME)
- @Target({ java.lang.annotation.ElementType.METHOD })
- public @interface Repeat {
- public abstract int times();
-
- }
-
- private static class RepeatStatement extends Statement {
-
- private final int times;
- private final Statement statement;
-
- private RepeatStatement(int times, Statement statement) {
- this.times = times;
- this.statement = statement;
- }
-
- @Override
- public void evaluate() throws Throwable {
- for (int i = 0; i < times; i++) {
- statement.evaluate();
- }
- }
- }
-
- @Override
- public Statement apply(Statement statement, FrameworkMethod method, Object target) {
- Statement result = statement;
- Repeat repeat = method.getAnnotation(Repeat.class);
- if (repeat != null) {
- int times = repeat.times();
- result = new RepeatStatement(times, statement);
- }
- return result;
-
- }
-}
-
-@RunWith(Parameterized.class)
-public class RepeatedTest extends ExecutionTest {
-
- private int count;
-
- @Parameters
- public static Collection<Object[]> tests() throws Exception {
- Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_REPEADED_TESTSUITE_XML_NAME);
- return testArgs;
- }
-
- public RepeatedTest(TestCaseContext tcCtx) {
- super(tcCtx);
- count = 0;
- }
-
- @Rule
- public RepeatRule repeatRule = new RepeatRule();
-
- @Test
- @Repeat(times = 10000)
- public void test() throws Exception {
- System.err.println("***** Test Count: " + (++count) + " ******");
- TestsUtils.executeTest(PATH_ACTUAL, tcCtx, null, false);
- }
-}