You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ti...@apache.org on 2018/07/28 19:22:04 UTC
[1/5] asterixdb git commit: [ASTERIXDB-2314][HYR] Dataset in class
names in Hyracks
Repository: asterixdb
Updated Branches:
refs/heads/master 6b1b52c1f -> 1d2f00d88
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java
new file mode 100644
index 0000000..82c7d50
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.result;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executor;
+
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.io.IWorkspaceFileFactory;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.IResultPartitionManager;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.control.common.result.AbstractResultManager;
+import org.apache.hyracks.control.common.result.ResultStateSweeper;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.control.nc.io.WorkspaceFileFactory;
+import org.apache.hyracks.control.nc.resources.DefaultDeallocatableRegistry;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ResultPartitionManager extends AbstractResultManager implements IResultPartitionManager {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final NodeControllerService ncs;
+
+ private final Executor executor;
+
+ private final Map<JobId, ResultSetMap> partitionResultStateMap;
+
+ private final DefaultDeallocatableRegistry deallocatableRegistry;
+
+ private final IWorkspaceFileFactory fileFactory;
+
+ private final ResultMemoryManager resultMemoryManager;
+
+ public ResultPartitionManager(NodeControllerService ncs, Executor executor, int availableMemory, long resultTTL,
+ long resultSweepThreshold) {
+ super(resultTTL);
+ this.ncs = ncs;
+ this.executor = executor;
+ deallocatableRegistry = new DefaultDeallocatableRegistry();
+ fileFactory = new WorkspaceFileFactory(deallocatableRegistry, ncs.getIoManager());
+ if (availableMemory >= ResultMemoryManager.getPageSize()) {
+ resultMemoryManager = new ResultMemoryManager(availableMemory);
+ } else {
+ resultMemoryManager = null;
+ }
+ partitionResultStateMap = new HashMap<>();
+ executor.execute(new ResultStateSweeper(this, resultSweepThreshold, LOGGER));
+ }
+
+ @Override
+ public IFrameWriter createResultPartitionWriter(IHyracksTaskContext ctx, ResultSetId rsId, boolean orderedResult,
+ boolean asyncMode, int partition, int nPartitions, long maxReads) {
+ ResultPartitionWriter dpw;
+ JobId jobId = ctx.getJobletContext().getJobId();
+ synchronized (this) {
+ dpw = new ResultPartitionWriter(ctx, this, jobId, rsId, asyncMode, orderedResult, partition, nPartitions,
+ resultMemoryManager, fileFactory, maxReads);
+ ResultSetMap rsIdMap = partitionResultStateMap.computeIfAbsent(jobId, k -> new ResultSetMap());
+ ResultState[] resultStates = rsIdMap.createOrGetResultStates(rsId, nPartitions);
+ resultStates[partition] = dpw.getResultState();
+ }
+ LOGGER.debug("Initialized partition writer: JobId: {}:partition: {}", jobId, partition);
+ return dpw;
+ }
+
+ @Override
+ public void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, int partition, int nPartitions,
+ boolean orderedResult, boolean emptyResult) throws HyracksException {
+ try {
+ // Be sure to send the *public* network address to the CC
+ ncs.getClusterController(jobId.getCcId()).registerResultPartitionLocation(jobId, rsId, orderedResult,
+ emptyResult, partition, nPartitions, ncs.getResultNetworkManager().getPublicNetworkAddress());
+ } catch (Exception e) {
+ throw HyracksException.create(e);
+ }
+ }
+
+ @Override
+ public void reportPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition) throws HyracksException {
+ try {
+ LOGGER.debug("Reporting partition write completion: JobId: {}:ResultSetId: {}:partition: {}", jobId, rsId,
+ partition);
+ ncs.getClusterController(jobId.getCcId()).reportResultPartitionWriteCompletion(jobId, rsId, partition);
+ } catch (Exception e) {
+ throw HyracksException.create(e);
+ }
+ }
+
+ @Override
+ public void initializeResultPartitionReader(JobId jobId, ResultSetId resultSetId, int partition,
+ IFrameWriter writer) throws HyracksException {
+ ResultState resultState = getResultState(jobId, resultSetId, partition);
+ ResultPartitionReader dpr = new ResultPartitionReader(this, resultMemoryManager, executor, resultState);
+ dpr.writeTo(writer);
+ LOGGER.debug("Initialized partition reader: JobId: {}:ResultSetId: {}:partition: {}", jobId, resultSetId,
+ partition);
+ }
+
+ private synchronized ResultState getResultState(JobId jobId, ResultSetId resultSetId, int partition)
+ throws HyracksException {
+ ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
+ if (rsIdMap == null) {
+ throw new HyracksException("Unknown JobId " + jobId);
+ }
+ ResultState[] resultStates = rsIdMap.getResultStates(resultSetId);
+ if (resultStates == null) {
+ throw new HyracksException("Unknown JobId: " + jobId + " ResultSetId: " + resultSetId);
+ }
+ ResultState resultState = resultStates[partition];
+ if (resultState == null) {
+ throw new HyracksException("No ResultPartitionWriter for partition " + partition);
+ }
+ return resultState;
+ }
+
+ @Override
+ public synchronized void removePartition(JobId jobId, ResultSetId resultSetId, int partition) {
+ ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
+ if (rsIdMap != null && rsIdMap.removePartition(jobId, resultSetId, partition)) {
+ partitionResultStateMap.remove(jobId);
+ }
+ }
+
+ @Override
+ public synchronized void abortReader(JobId jobId) {
+ ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
+ if (rsIdMap != null) {
+ rsIdMap.abortAll();
+ }
+ }
+
+ @Override
+ public synchronized void close() {
+ for (JobId jobId : getJobIds()) {
+ deinit(jobId);
+ }
+ deallocatableRegistry.close();
+ }
+
+ @Override
+ public synchronized Set<JobId> getJobIds() {
+ return partitionResultStateMap.keySet();
+ }
+
+ @Override
+ public synchronized ResultSetMap getState(JobId jobId) {
+ return partitionResultStateMap.get(jobId);
+ }
+
+ @Override
+ public synchronized void sweep(JobId jobId) {
+ deinit(jobId);
+ partitionResultStateMap.remove(jobId);
+ }
+
+ private synchronized void deinit(JobId jobId) {
+ ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
+ if (rsIdMap != null) {
+ rsIdMap.closeAndDeleteAll();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionReader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionReader.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionReader.java
new file mode 100644
index 0000000..121d5a1
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionReader.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.result;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.Executor;
+
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.partitions.ResultSetPartitionId;
+import org.apache.hyracks.comm.channels.NetworkOutputChannel;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ResultPartitionReader {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final ResultPartitionManager resultPartitionManager;
+ private final ResultMemoryManager resultMemoryManager;
+ private final Executor executor;
+ private final ResultState resultState;
+
+ public ResultPartitionReader(ResultPartitionManager resultPartitionManager, ResultMemoryManager resultMemoryManager,
+ Executor executor, ResultState resultState) {
+ this.resultPartitionManager = resultPartitionManager;
+ this.resultMemoryManager = resultMemoryManager;
+ this.executor = executor;
+ this.resultState = resultState;
+ }
+
+ public void writeTo(final IFrameWriter writer) {
+ executor.execute(new ResultPartitionSender((NetworkOutputChannel) writer));
+ }
+
+ private class ResultPartitionSender implements Runnable {
+
+ private final NetworkOutputChannel channel;
+
+ ResultPartitionSender(final NetworkOutputChannel channel) {
+ this.channel = channel;
+ }
+
+ @Override
+ public void run() {
+ channel.setFrameSize(resultState.getFrameSize());
+ channel.open();
+ try {
+ resultState.readOpen();
+ long offset = 0;
+ final ByteBuffer buffer = ByteBuffer.allocate(resultState.getFrameSize());
+ while (true) {
+ buffer.clear();
+ final long size = read(offset, buffer);
+ if (size <= 0) {
+ break;
+ } else if (size < buffer.limit()) {
+ throw new IllegalStateException(
+ "Premature end of file - readSize: " + size + " buffer limit: " + buffer.limit());
+ }
+ offset += size;
+ buffer.flip();
+ channel.nextFrame(buffer);
+ }
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("result reading successful(" + resultState.getResultSetPartitionId() + ")");
+ }
+ } catch (Exception e) {
+ LOGGER.error(() -> "failed to send result partition " + resultState.getResultSetPartitionId(), e);
+ channel.abort();
+ } finally {
+ close();
+ }
+ }
+
+ private long read(long offset, ByteBuffer buffer) throws HyracksDataException {
+ return resultMemoryManager != null ? resultState.read(resultMemoryManager, offset, buffer)
+ : resultState.read(offset, buffer);
+ }
+
+ private void close() {
+ try {
+ channel.close();
+ resultState.readClose();
+ if (resultState.isExhausted()) {
+ final ResultSetPartitionId partitionId = resultState.getResultSetPartitionId();
+ resultPartitionManager.removePartition(partitionId.getJobId(), partitionId.getResultSetId(),
+ partitionId.getPartition());
+ }
+ } catch (HyracksDataException e) {
+ LOGGER.error("unexpected failure in partition reader clean up", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java
new file mode 100644
index 0000000..f25bc58
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.result;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.io.IWorkspaceFileFactory;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.partitions.ResultSetPartitionId;
+import org.apache.hyracks.api.result.IResultPartitionManager;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ResultPartitionWriter implements IFrameWriter {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final IResultPartitionManager manager;
+
+ private final JobId jobId;
+
+ private final ResultSetId resultSetId;
+
+ private final boolean orderedResult;
+
+ private final int partition;
+
+ private final int nPartitions;
+
+ private final ResultMemoryManager resultMemoryManager;
+
+ private final ResultSetPartitionId resultSetPartitionId;
+
+ private final ResultState resultState;
+
+ private boolean partitionRegistered;
+
+ private boolean failed = false;
+
+ public ResultPartitionWriter(IHyracksTaskContext ctx, IResultPartitionManager manager, JobId jobId,
+ ResultSetId rsId, boolean asyncMode, boolean orderedResult, int partition, int nPartitions,
+ ResultMemoryManager resultMemoryManager, IWorkspaceFileFactory fileFactory, long maxReads) {
+ this.manager = manager;
+ this.jobId = jobId;
+ this.resultSetId = rsId;
+ this.orderedResult = orderedResult;
+ this.partition = partition;
+ this.nPartitions = nPartitions;
+ this.resultMemoryManager = resultMemoryManager;
+
+ resultSetPartitionId = new ResultSetPartitionId(jobId, rsId, partition);
+ resultState = new ResultState(resultSetPartitionId, asyncMode, ctx.getIoManager(), fileFactory,
+ ctx.getInitialFrameSize(), maxReads);
+ }
+
+ public ResultState getResultState() {
+ return resultState;
+ }
+
+ @Override
+ public void open() {
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("open(" + partition + ")");
+ }
+ partitionRegistered = false;
+ resultState.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ registerResultPartitionLocation(false);
+ if (resultMemoryManager == null) {
+ resultState.write(buffer);
+ } else {
+ resultState.write(resultMemoryManager, buffer);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ failed = true;
+ resultState.closeAndDelete();
+ resultState.abort();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("close(" + partition + ")");
+ }
+ try {
+ if (!failed) {
+ registerResultPartitionLocation(true);
+ }
+ } finally {
+ resultState.close();
+ }
+ try {
+ if (partitionRegistered) {
+ manager.reportPartitionWriteCompletion(jobId, resultSetId, partition);
+ }
+ } catch (HyracksException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ void registerResultPartitionLocation(boolean empty) throws HyracksDataException {
+ try {
+ if (!partitionRegistered) {
+ manager.registerResultPartitionLocation(jobId, resultSetId, partition, nPartitions, orderedResult,
+ empty);
+ partitionRegistered = true;
+ }
+ } catch (HyracksException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultSetMap.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultSetMap.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultSetMap.java
new file mode 100644
index 0000000..41b7f07
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultSetMap.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.result;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.IResultStateRecord;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+class ResultSetMap implements IResultStateRecord, Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final long timestamp;
+ private final HashMap<ResultSetId, ResultState[]> resultStateMap;
+
+ ResultSetMap() {
+ timestamp = System.nanoTime();
+ resultStateMap = new HashMap<>();
+ }
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ ResultState[] getResultStates(ResultSetId rsId) {
+ return resultStateMap.get(rsId);
+ }
+
+ ResultState[] createOrGetResultStates(ResultSetId rsId, int nPartitions) {
+ return resultStateMap.computeIfAbsent(rsId, (k) -> new ResultState[nPartitions]);
+ }
+
+ /**
+ * removes a result partition for a result set
+ *
+ * @param jobId
+ * the id of the job that produced the result set
+ * @param resultSetId
+ * the id of the result set
+ * @param partition
+ * the partition number
+ * @return true, if all partitions for the resultSetId have been removed
+ */
+ boolean removePartition(JobId jobId, ResultSetId resultSetId, int partition) {
+ final ResultState[] resultStates = resultStateMap.get(resultSetId);
+ if (resultStates != null) {
+ final ResultState state = resultStates[partition];
+ if (state != null) {
+ state.closeAndDelete();
+ LOGGER.debug("Removing partition: " + partition + " for JobId: " + jobId);
+ }
+ resultStates[partition] = null;
+ boolean stateEmpty = true;
+ for (ResultState resState : resultStates) {
+ if (resState != null) {
+ stateEmpty = false;
+ break;
+ }
+ }
+ if (stateEmpty) {
+ resultStateMap.remove(resultSetId);
+ }
+ return resultStateMap.isEmpty();
+ }
+ return true;
+ }
+
+ void abortAll() {
+ applyToAllStates((rsId, state, i) -> state.abort());
+ }
+
+ void closeAndDeleteAll() {
+ applyToAllStates((rsId, state, i) -> {
+ state.closeAndDelete();
+ LOGGER.debug("Removing partition: " + i + " for result set " + rsId);
+ });
+ }
+
+ @FunctionalInterface
+ private interface StateModifier {
+ void modify(ResultSetId rsId, ResultState entry, int partition);
+ }
+
+ private void applyToAllStates(StateModifier modifier) {
+ for (Map.Entry<ResultSetId, ResultState[]> entry : resultStateMap.entrySet()) {
+ final ResultSetId rsId = entry.getKey();
+ final ResultState[] resultStates = entry.getValue();
+ if (resultStates == null) {
+ continue;
+ }
+ for (int i = 0; i < resultStates.length; i++) {
+ final ResultState state = resultStates[i];
+ if (state != null) {
+ modifier.modify(rsId, state, i);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java
new file mode 100644
index 0000000..25d3f00
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.result;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hyracks.api.dataflow.state.IStateObject;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IFileHandle;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IWorkspaceFileFactory;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.partitions.ResultSetPartitionId;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class ResultState implements IStateObject {
+ private static final String FILE_PREFIX = "result_";
+
+ private final ResultSetPartitionId resultSetPartitionId;
+
+ private final boolean asyncMode;
+
+ private final int frameSize;
+
+ private final IIOManager ioManager;
+
+ private final IWorkspaceFileFactory fileFactory;
+
+ private final AtomicBoolean eos;
+
+ private final AtomicBoolean failed;
+
+ private final List<Page> localPageList;
+
+ private FileReference fileRef;
+
+ private IFileHandle fileHandle;
+
+ private volatile int referenceCount = 0;
+
+ private long size;
+
+ private long persistentSize;
+ private long remainingReads;
+
+ ResultState(ResultSetPartitionId resultSetPartitionId, boolean asyncMode, IIOManager ioManager,
+ IWorkspaceFileFactory fileFactory, int frameSize, long maxReads) {
+ if (maxReads <= 0) {
+ throw new IllegalArgumentException("maxReads must be > 0");
+ }
+ this.resultSetPartitionId = resultSetPartitionId;
+ this.asyncMode = asyncMode;
+ this.ioManager = ioManager;
+ this.fileFactory = fileFactory;
+ this.frameSize = frameSize;
+ remainingReads = maxReads;
+ eos = new AtomicBoolean(false);
+ failed = new AtomicBoolean(false);
+ localPageList = new ArrayList<>();
+
+ fileRef = null;
+ fileHandle = null;
+ }
+
+ public synchronized void open() {
+ size = 0;
+ persistentSize = 0;
+ referenceCount = 0;
+ }
+
+ public synchronized void close() {
+ eos.set(true);
+ closeWriteFileHandle();
+ notifyAll();
+ }
+
+ public synchronized void closeAndDelete() {
+ // Deleting a job is equivalent to aborting the job for all practical purposes, so the same action, needs
+ // to be taken when there are more requests to these result states.
+ failed.set(true);
+ closeWriteFileHandle();
+ if (fileRef != null) {
+ fileRef.delete();
+ fileRef = null;
+ }
+ }
+
+ private void closeWriteFileHandle() {
+ if (fileHandle != null) {
+ doCloseFileHandle();
+ }
+ }
+
+ private void doCloseFileHandle() {
+ if (--referenceCount == 0) {
+ // close the file if there is no more reference
+ try {
+ ioManager.close(fileHandle);
+ } catch (IOException e) {
+ // Since file handle could not be closed, just ignore.
+ }
+ fileHandle = null;
+ }
+ }
+
+ public synchronized void write(ByteBuffer buffer) throws HyracksDataException {
+ if (fileRef == null) {
+ initWriteFileHandle();
+ }
+
+ size += ioManager.syncWrite(fileHandle, size, buffer);
+ notifyAll();
+ }
+
+ public synchronized void write(ResultMemoryManager resultMemoryManager, ByteBuffer buffer)
+ throws HyracksDataException {
+ int srcOffset = 0;
+ Page destPage = null;
+
+ if (!localPageList.isEmpty()) {
+ destPage = localPageList.get(localPageList.size() - 1);
+ }
+
+ while (srcOffset < buffer.limit()) {
+ if ((destPage == null) || (destPage.getBuffer().remaining() <= 0)) {
+ destPage = resultMemoryManager.requestPage(resultSetPartitionId, this);
+ localPageList.add(destPage);
+ }
+ int srcLength = Math.min(buffer.limit() - srcOffset, destPage.getBuffer().remaining());
+ destPage.getBuffer().put(buffer.array(), srcOffset, srcLength);
+ srcOffset += srcLength;
+ size += srcLength;
+ }
+
+ notifyAll();
+ }
+
+ public synchronized void readOpen() {
+ if (isExhausted()) {
+ throw new IllegalStateException("Result reads exhausted");
+ }
+ remainingReads--;
+ }
+
+ public synchronized void readClose() throws HyracksDataException {
+ if (fileHandle != null) {
+ doCloseFileHandle();
+ }
+ }
+
+ public synchronized long read(long offset, ByteBuffer buffer) throws HyracksDataException {
+ long readSize = 0;
+
+ while (offset >= size && !eos.get() && !failed.get()) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ if ((offset >= size && eos.get()) || failed.get()) {
+ return readSize;
+ }
+
+ if (fileHandle == null) {
+ initReadFileHandle();
+ }
+ readSize = ioManager.syncRead(fileHandle, offset, buffer);
+
+ return readSize;
+ }
+
+ public synchronized long read(ResultMemoryManager resultMemoryManager, long offset, ByteBuffer buffer)
+ throws HyracksDataException {
+ long readSize = 0;
+ while (offset >= size && !eos.get() && !failed.get()) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ if ((offset >= size && eos.get()) || failed.get()) {
+ return readSize;
+ }
+
+ if (offset < persistentSize) {
+ if (fileHandle == null) {
+ initReadFileHandle();
+ }
+ readSize = ioManager.syncRead(fileHandle, offset, buffer);
+ if (readSize < 0) {
+ throw new HyracksDataException("Premature end of file");
+ }
+ }
+
+ if (readSize < buffer.capacity()) {
+ long localPageOffset = offset - persistentSize;
+ int localPageIndex = (int) (localPageOffset / ResultMemoryManager.getPageSize());
+ int pageOffset = (int) (localPageOffset % ResultMemoryManager.getPageSize());
+ Page page = getPage(localPageIndex);
+ if (page == null) {
+ return readSize;
+ }
+ readSize += buffer.remaining();
+ buffer.put(page.getBuffer().array(), pageOffset, buffer.remaining());
+ }
+ resultMemoryManager.pageReferenced(resultSetPartitionId);
+ return readSize;
+ }
+
+ public synchronized void abort() {
+ failed.set(true);
+ notifyAll();
+ }
+
+ public synchronized Page returnPage() throws HyracksDataException {
+ Page page = removePage();
+
+ // If we do not have any pages to be given back close the write channel since we don't write any more, return
+ // null.
+ if (page == null) {
+ ioManager.close(fileHandle);
+ return null;
+ }
+
+ page.getBuffer().flip();
+
+ if (fileRef == null) {
+ initWriteFileHandle();
+ }
+
+ long delta = ioManager.syncWrite(fileHandle, persistentSize, page.getBuffer());
+ persistentSize += delta;
+ return page;
+ }
+
+ public synchronized void setEOS(boolean eos) {
+ this.eos.set(eos);
+ }
+
+ public ResultSetPartitionId getResultSetPartitionId() {
+ return resultSetPartitionId;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public IIOManager getIOManager() {
+ return ioManager;
+ }
+
+ public boolean getAsyncMode() {
+ return asyncMode;
+ }
+
+ @Override
+ public JobId getJobId() {
+ return resultSetPartitionId.getJobId();
+ }
+
+ @Override
+ public Object getId() {
+ return resultSetPartitionId;
+ }
+
+ @Override
+ public long getMemoryOccupancy() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void toBytes(DataOutput out) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void fromBytes(DataInput in) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ private Page getPage(int index) {
+ Page page = null;
+ if (!localPageList.isEmpty()) {
+ page = localPageList.get(index);
+ }
+ return page;
+ }
+
+ private Page removePage() {
+ Page page = null;
+ if (!localPageList.isEmpty()) {
+ page = localPageList.remove(localPageList.size() - 1);
+ }
+ return page;
+ }
+
+ private void initWriteFileHandle() throws HyracksDataException {
+ if (fileHandle == null) {
+ String fName = FILE_PREFIX + String.valueOf(resultSetPartitionId.getPartition());
+ fileRef = fileFactory.createUnmanagedWorkspaceFile(fName);
+ fileHandle = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE,
+ IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
+ if (referenceCount != 0) {
+ throw new IllegalStateException("Illegal reference count " + referenceCount);
+ }
+ referenceCount = 1;
+ notifyAll(); // NOSONAR: always called from a synchronized block
+ }
+ }
+
+ private void initReadFileHandle() throws HyracksDataException {
+ while (fileRef == null && !failed.get()) {
+ // wait for writer to create the file
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ if (failed.get()) {
+ return;
+ }
+ if (fileHandle == null) {
+ // fileHandle has been closed by the writer, create it again
+ fileHandle = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_ONLY,
+ IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
+ }
+ referenceCount++;
+ }
+
+ @Override
+ public String toString() {
+ try {
+ ObjectMapper om = new ObjectMapper();
+ ObjectNode on = om.createObjectNode();
+ on.put("rspid", resultSetPartitionId.toString());
+ on.put("async", asyncMode);
+ on.put("remainingReads", remainingReads);
+ on.put("eos", eos.get());
+ on.put("failed", failed.get());
+ on.put("fileRef", String.valueOf(fileRef));
+ return om.writer(new MinimalPrettyPrinter()).writeValueAsString(on);
+ } catch (JsonProcessingException e) { // NOSONAR
+ return e.getMessage();
+ }
+ }
+
+ public synchronized boolean isExhausted() {
+ return remainingReads == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortAllJobsWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortAllJobsWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortAllJobsWork.java
index c6696fd..b11dada 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortAllJobsWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortAllJobsWork.java
@@ -23,9 +23,9 @@ import java.util.Collection;
import java.util.Deque;
import org.apache.hyracks.api.control.CcId;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobStatus;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.control.common.work.SynchronizableWork;
import org.apache.hyracks.control.nc.Joblet;
import org.apache.hyracks.control.nc.NodeControllerService;
@@ -48,9 +48,9 @@ public class AbortAllJobsWork extends SynchronizableWork {
@Override
protected void doRun() throws Exception {
LOGGER.info("Aborting all tasks for controller {}", ccId);
- IDatasetPartitionManager dpm = ncs.getDatasetPartitionManager();
- if (dpm == null) {
- LOGGER.log(Level.WARN, "DatasetPartitionManager is null on " + ncs.getId());
+ IResultPartitionManager resultPartitionManager = ncs.getResultPartitionManager();
+ if (resultPartitionManager == null) {
+ LOGGER.log(Level.WARN, "ResultPartitionManager is null on " + ncs.getId());
}
Deque<Task> abortedTasks = new ArrayDeque<>();
Collection<Joblet> joblets = ncs.getJobletMap().values();
@@ -61,9 +61,9 @@ public class AbortAllJobsWork extends SynchronizableWork {
abortedTasks.add(task);
});
final JobId jobId = joblet.getJobId();
- if (dpm != null) {
- dpm.abortReader(jobId);
- dpm.sweep(jobId);
+ if (resultPartitionManager != null) {
+ resultPartitionManager.abortReader(jobId);
+ resultPartitionManager.sweep(jobId);
}
ncs.getWorkQueue().schedule(new CleanupJobletWork(ncs, jobId, JobStatus.FAILURE));
});
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortTasksWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortTasksWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortTasksWork.java
index 80f3e98..f47e1ce 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortTasksWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/AbortTasksWork.java
@@ -22,8 +22,8 @@ import java.util.List;
import java.util.Map;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.control.common.work.AbstractWork;
import org.apache.hyracks.control.nc.Joblet;
import org.apache.hyracks.control.nc.NodeControllerService;
@@ -52,9 +52,9 @@ public class AbortTasksWork extends AbstractWork {
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Aborting Tasks: " + jobId + ":" + tasks);
}
- IDatasetPartitionManager dpm = ncs.getDatasetPartitionManager();
- if (dpm != null) {
- ncs.getDatasetPartitionManager().abortReader(jobId);
+ IResultPartitionManager resultPartitionManager = ncs.getResultPartitionManager();
+ if (resultPartitionManager != null) {
+ ncs.getResultPartitionManager().abortReader(jobId);
}
Joblet ji = ncs.getJobletMap().get(jobId);
if (ji != null) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java
index ac80afa..7dbb3c2 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java
@@ -21,8 +21,8 @@ package org.apache.hyracks.control.nc.work;
import java.util.List;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.control.common.work.AbstractWork;
import org.apache.hyracks.control.nc.NodeControllerService;
import org.apache.hyracks.control.nc.Task;
@@ -52,9 +52,9 @@ public class NotifyTaskFailureWork extends AbstractWork {
LOGGER.log(Level.WARN, ncs.getId() + " is sending a notification to cc that task " + taskId + " has failed",
exceptions.get(0));
try {
- IDatasetPartitionManager dpm = ncs.getDatasetPartitionManager();
- if (dpm != null) {
- dpm.abortReader(jobId);
+ IResultPartitionManager resultPartitionManager = ncs.getResultPartitionManager();
+ if (resultPartitionManager != null) {
+ resultPartitionManager.abortReader(jobId);
}
ncs.getClusterController(jobId.getCcId()).notifyTaskFailure(jobId, taskId, ncs.getId(), exceptions);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java
index b0cc40c..9fb9815 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java
@@ -31,11 +31,11 @@ import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import org.apache.hyracks.api.dataflow.value.IResultSerializer;
import org.apache.hyracks.api.dataflow.value.IResultSerializerFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
+import org.apache.hyracks.api.result.IResultPartitionManager;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.dataflow.common.comm.io.FrameOutputStream;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
@@ -67,7 +67,7 @@ public class ResultWriterOperatorDescriptor extends AbstractSingleActivityOperat
public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
throws HyracksDataException {
- final IDatasetPartitionManager dpm = ctx.getDatasetPartitionManager();
+ final IResultPartitionManager resultPartitionManager = ctx.getResultPartitionManager();
final IFrame frame = new VSizeFrame(ctx);
@@ -82,15 +82,15 @@ public class ResultWriterOperatorDescriptor extends AbstractSingleActivityOperat
final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor(outRecordDesc);
return new AbstractUnaryInputSinkOperatorNodePushable() {
- private IFrameWriter datasetPartitionWriter;
+ private IFrameWriter resultPartitionWriter;
private boolean failed = false;
@Override
public void open() throws HyracksDataException {
try {
- datasetPartitionWriter = dpm.createDatasetPartitionWriter(ctx, rsId, ordered, asyncMode, partition,
- nPartitions, maxReads);
- datasetPartitionWriter.open();
+ resultPartitionWriter = resultPartitionManager.createResultPartitionWriter(ctx, rsId, ordered,
+ asyncMode, partition, nPartitions, maxReads);
+ resultPartitionWriter.open();
resultSerializer.init();
} catch (HyracksException e) {
throw HyracksDataException.create(e);
@@ -103,7 +103,7 @@ public class ResultWriterOperatorDescriptor extends AbstractSingleActivityOperat
for (int tIndex = 0; tIndex < frameTupleAccessor.getTupleCount(); tIndex++) {
resultSerializer.appendTuple(frameTupleAccessor, tIndex);
if (!frameOutputStream.appendTuple()) {
- frameOutputStream.flush(datasetPartitionWriter);
+ frameOutputStream.flush(resultPartitionWriter);
resultSerializer.appendTuple(frameTupleAccessor, tIndex);
frameOutputStream.appendTuple();
@@ -114,23 +114,23 @@ public class ResultWriterOperatorDescriptor extends AbstractSingleActivityOperat
@Override
public void fail() throws HyracksDataException {
failed = true;
- if (datasetPartitionWriter != null) {
- datasetPartitionWriter.fail();
+ if (resultPartitionWriter != null) {
+ resultPartitionWriter.fail();
}
}
@Override
public void close() throws HyracksDataException {
- if (datasetPartitionWriter != null) {
+ if (resultPartitionWriter != null) {
try {
if (!failed && frameOutputStream.getTupleCount() > 0) {
- frameOutputStream.flush(datasetPartitionWriter);
+ frameOutputStream.flush(resultPartitionWriter);
}
} catch (Exception e) {
- datasetPartitionWriter.fail();
+ resultPartitionWriter.fail();
throw e;
} finally {
- datasetPartitionWriter.close();
+ resultPartitionWriter.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java
index 0931501..5dcc99a 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractIntegrationTest.java
@@ -36,16 +36,16 @@ import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.api.result.IResultSetReader;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.client.result.ResultSet;
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.common.controllers.CCConfig;
import org.apache.hyracks.control.common.controllers.NCConfig;
@@ -154,8 +154,8 @@ public abstract class AbstractIntegrationTest {
IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
- IHyracksDataset hyracksDataset = new HyracksDataset(hcc, spec.getFrameSize(), nReaders);
- IHyracksDatasetReader reader = hyracksDataset.createReader(jobId, resultSetId);
+ IResultSet resultSet = new ResultSet(hcc, spec.getFrameSize(), nReaders);
+ IResultSetReader reader = resultSet.createReader(jobId, resultSetId);
List<String> resultRecords = new ArrayList<>();
ByteBufferInputStream bbis = new ByteBufferInputStream();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
index 58da8a2..55fd9a0 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
@@ -29,8 +29,6 @@ import org.apache.hyracks.api.client.HyracksConnection;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
import org.apache.hyracks.api.comm.VSizeFrame;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.job.JobFlag;
@@ -38,7 +36,9 @@ import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.api.job.JobStatus;
import org.apache.hyracks.api.job.resource.IJobCapacityController;
-import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.api.result.IResultSetReader;
+import org.apache.hyracks.client.result.ResultSet;
import org.apache.hyracks.control.cc.BaseCCApplication;
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.cc.application.CCServiceContext;
@@ -160,8 +160,8 @@ public abstract class AbstractMultiNCIntegrationTest {
IFrameTupleAccessor frameTupleAccessor = new ResultFrameTupleAccessor();
if (!spec.getResultSetIds().isEmpty()) {
- IHyracksDataset hyracksDataset = new HyracksDataset(hcc, spec.getFrameSize(), nReaders);
- IHyracksDatasetReader reader = hyracksDataset.createReader(jobId, spec.getResultSetIds().get(0));
+ IResultSet resultSet = new ResultSet(hcc, spec.getFrameSize(), nReaders);
+ IResultSetReader reader = resultSet.createReader(jobId, spec.getResultSetIds().get(0));
ObjectMapper om = new ObjectMapper();
ArrayNode resultRecords = om.createArrayNode();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java
index 752c643..39e7a45 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/AggregationTest.java
@@ -28,10 +28,10 @@ import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
import org.apache.hyracks.data.std.accessors.UTF8StringBinaryHashFunctionFamily;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java
index c28a5aa..c542fe9 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/CountOfCountsTest.java
@@ -27,10 +27,10 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java
index 9e795bf..1dce88d 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/HeapSortMergeTest.java
@@ -27,10 +27,10 @@ import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java
index 49dee84..8d3b0dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/LocalityAwareConnectorTest.java
@@ -29,10 +29,10 @@ import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
import org.apache.hyracks.data.std.accessors.UTF8StringBinaryHashFunctionFamily;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java
index 672d2c4..9d79d80 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/NodesAPIIntegrationTest.java
@@ -45,8 +45,8 @@ public class NodesAPIIntegrationTest extends AbstractIntegrationTest {
"nonheap-committed-sizes", "nonheap-max-sizes", "application-memory-budget", "application-cpu-core-budget",
"thread-counts", "peak-thread-counts", "system-load-averages", "gc-names", "gc-collection-counts",
"gc-collection-times", "net-payload-bytes-read", "net-payload-bytes-written", "net-signaling-bytes-read",
- "net-signaling-bytes-written", "dataset-net-payload-bytes-read", "dataset-net-payload-bytes-written",
- "dataset-net-signaling-bytes-read", "dataset-net-signaling-bytes-written", "ipc-messages-sent",
+ "net-signaling-bytes-written", "result-net-payload-bytes-read", "result-net-payload-bytes-written",
+ "result-net-signaling-bytes-read", "result-net-signaling-bytes-written", "ipc-messages-sent",
"ipc-message-bytes-sent", "ipc-messages-received", "ipc-message-bytes-received", "disk-reads",
"disk-writes", "config" };
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java
index 09629b2..bc5de11 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ReplicateOperatorTest.java
@@ -27,10 +27,10 @@ import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java
index 75ba33f..7838a34 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/ScanPrintTest.java
@@ -26,10 +26,10 @@ import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java
index 315b74c..06c0eaa 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/SortMergeTest.java
@@ -26,7 +26,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
index 816f3fa..6154e28 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
@@ -28,10 +28,10 @@ import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import org.apache.hyracks.api.dataflow.value.IMissingWriterFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
index c2b3263..0eb6810 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
@@ -32,11 +32,11 @@ import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITuplePairComparator;
import org.apache.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java
index 81a71eb..48f7837 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/src/test/java/org/apache/hyracks/tests/integration/UnionTest.java
@@ -24,10 +24,10 @@ import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.io.ManagedFileSplit;
import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import org.apache.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
index ac52573..174d5cd 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
@@ -30,7 +30,6 @@ import org.apache.hyracks.api.context.IHyracksJobletContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.dataflow.state.IStateObject;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -39,6 +38,7 @@ import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.profiling.IStatsCollector;
import org.apache.hyracks.api.job.profiling.counters.ICounterContext;
import org.apache.hyracks.api.resources.IDeallocatable;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.control.common.job.profiling.StatsCollector;
import org.apache.hyracks.control.nc.io.WorkspaceFileFactory;
@@ -134,7 +134,7 @@ public class TestTaskContext implements IHyracksTaskContext {
}
@Override
- public IDatasetPartitionManager getDatasetPartitionManager() {
+ public IResultPartitionManager getResultPartitionManager() {
return null;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
index 2f58259..1ceb199 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
@@ -19,7 +19,7 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.util;
-import static org.junit.Assert.*;
+import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.DataInput;
@@ -44,7 +44,6 @@ import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.dataflow.state.IStateObject;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -55,6 +54,7 @@ import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.profiling.IStatsCollector;
import org.apache.hyracks.api.job.profiling.counters.ICounterContext;
import org.apache.hyracks.api.resources.IDeallocatable;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.data.std.primitive.IntegerPointable;
import org.apache.hyracks.data.std.util.GrowableArray;
@@ -726,7 +726,7 @@ public class LSMInvertedIndexTestUtils {
}
@Override
- public IDatasetPartitionManager getDatasetPartitionManager() {
+ public IResultPartitionManager getResultPartitionManager() {
return null;
}
[5/5] asterixdb git commit: [ASTERIXDB-2314][HYR] Dataset in class
names in Hyracks
Posted by ti...@apache.org.
[ASTERIXDB-2314][HYR] Dataset in class names in Hyracks
- user model changes: no
- storage format changes: no
- interface changes: yes
Change-Id: I260add3a2dfb4a722440af5008a1a75f9446579e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2478
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/1d2f00d8
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/1d2f00d8
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/1d2f00d8
Branch: refs/heads/master
Commit: 1d2f00d88313fe393ff877a42fb8684bee9c1b6a
Parents: 6b1b52c
Author: Till Westmann <ti...@apache.org>
Authored: Sat Jul 28 08:53:50 2018 -0700
Committer: Till Westmann <ti...@apache.org>
Committed: Sat Jul 28 12:21:40 2018 -0700
----------------------------------------------------------------------
.../asterix/translator/IRequestParameters.java | 6 +-
.../asterix/translator/IStatementExecutor.java | 2 +-
.../http/server/AbstractQueryApiServlet.java | 26 +-
.../asterix/api/http/server/ApiServlet.java | 19 +-
.../api/http/server/NCQueryServiceServlet.java | 4 +-
.../api/http/server/QueryResultApiServlet.java | 10 +-
.../api/http/server/QueryServiceServlet.java | 4 +-
.../api/http/server/QueryStatusApiServlet.java | 13 +-
.../asterix/api/http/server/RestApiServlet.java | 19 +-
.../api/http/server/ServletConstants.java | 2 +-
.../asterix/api/http/server/ServletUtil.java | 47 +++
.../apache/asterix/app/result/ResultHandle.java | 2 +-
.../apache/asterix/app/result/ResultReader.java | 14 +-
.../asterix/app/translator/QueryTranslator.java | 24 +-
.../app/translator/RequestParameters.java | 12 +-
.../metadata/declared/MetadataProvider.java | 2 +-
.../metadata/declared/ResultSetSinkId.java | 2 +-
.../client/HyracksClientInterfaceFunctions.java | 30 +-
.../HyracksClientInterfaceRemoteProxy.java | 6 +-
.../hyracks/api/client/HyracksConnection.java | 4 +-
.../api/client/IHyracksClientConnection.java | 4 +-
.../api/client/IHyracksClientInterface.java | 2 +-
.../hyracks/api/client/NodeControllerInfo.java | 10 +-
.../api/context/IHyracksTaskContext.java | 4 +-
.../api/dataset/DatasetDirectoryRecord.java | 111 ------
.../hyracks/api/dataset/DatasetJobRecord.java | 177 ---------
.../hyracks/api/dataset/IDatasetManager.java | 38 --
.../api/dataset/IDatasetPartitionManager.java | 44 ---
.../api/dataset/IDatasetStateRecord.java | 23 --
.../hyracks/api/dataset/IHyracksDataset.java | 26 --
...yracksDatasetDirectoryServiceConnection.java | 51 ---
...HyracksDatasetDirectoryServiceInterface.java | 51 ---
.../api/dataset/IHyracksDatasetReader.java | 29 --
.../apache/hyracks/api/dataset/ResultSetId.java | 56 ---
.../hyracks/api/dataset/ResultSetMetaData.java | 46 ---
.../hyracks/api/job/JobSpecification.java | 2 +-
.../api/partitions/ResultSetPartitionId.java | 2 +-
.../hyracks/api/result/IResultDirectory.java | 51 +++
.../hyracks/api/result/IResultManager.java | 38 ++
.../api/result/IResultPartitionManager.java | 44 +++
.../apache/hyracks/api/result/IResultSet.java | 26 ++
.../hyracks/api/result/IResultSetReader.java | 29 ++
.../hyracks/api/result/IResultStateRecord.java | 23 ++
.../api/result/ResultDirectoryRecord.java | 111 ++++++
.../hyracks/api/result/ResultJobRecord.java | 177 +++++++++
.../apache/hyracks/api/result/ResultSetId.java | 56 +++
.../hyracks/api/result/ResultSetMetaData.java | 46 +++
.../hyracks/client/dataset/HyracksDataset.java | 76 ----
...yracksDatasetDirectoryServiceConnection.java | 57 ---
...setDirectoryServiceInterfaceRemoteProxy.java | 55 ---
.../client/dataset/HyracksDatasetReader.java | 244 ------------
.../hyracks/client/result/ResultDirectory.java | 56 +++
.../result/ResultDirectoryRemoteProxy.java | 55 +++
.../apache/hyracks/client/result/ResultSet.java | 74 ++++
.../hyracks/client/result/ResultSetReader.java | 243 ++++++++++++
.../channels/DatasetNetworkInputChannel.java | 154 --------
.../channels/ResultNetworkInputChannel.java | 154 ++++++++
.../hyracks/control/cc/ClientInterfaceIPCI.java | 20 +-
.../control/cc/ClusterControllerService.java | 20 +-
.../hyracks/control/cc/cluster/NodeManager.java | 2 +-
.../cc/dataset/DatasetDirectoryService.java | 316 ---------------
.../cc/dataset/IDatasetDirectoryService.java | 50 ---
.../cc/result/IResultDirectoryService.java | 50 +++
.../cc/result/ResultDirectoryService.java | 316 +++++++++++++++
.../GetDatasetDirectoryServiceInfoWork.java | 45 ---
.../cc/work/GetResultDirectoryAddressWork.java | 45 +++
.../work/GetResultPartitionLocationsWork.java | 14 +-
.../control/cc/work/GetResultStatusWork.java | 6 +-
.../RegisterResultPartitionLocationWork.java | 4 +-
...eportResultPartitionWriteCompletionWork.java | 4 +-
.../control/cc/work/TaskFailureWork.java | 2 +-
.../control/cc/cluster/NodeManagerTest.java | 2 +-
.../control/common/NodeControllerData.java | 40 +-
.../control/common/base/IClusterController.java | 2 +-
.../control/common/controllers/NCConfig.java | 8 +-
.../common/controllers/NodeRegistration.java | 10 +-
.../common/dataset/AbstractDatasetManager.java | 55 ---
.../common/dataset/ResultStateSweeper.java | 53 ---
.../control/common/heartbeat/HeartbeatData.java | 22 +-
.../control/common/ipc/CCNCFunctions.java | 2 +-
.../ipc/ClusterControllerRemoteProxy.java | 2 +-
.../common/result/AbstractResultManager.java | 55 +++
.../common/result/ResultStateSweeper.java | 53 +++
.../control/nc/NodeControllerService.java | 34 +-
.../org/apache/hyracks/control/nc/Task.java | 6 +-
.../nc/dataset/DatasetMemoryManager.java | 245 ------------
.../nc/dataset/DatasetPartitionManager.java | 184 ---------
.../nc/dataset/DatasetPartitionReader.java | 110 ------
.../nc/dataset/DatasetPartitionWriter.java | 142 -------
.../apache/hyracks/control/nc/dataset/Page.java | 37 --
.../control/nc/dataset/ResultSetMap.java | 123 ------
.../hyracks/control/nc/dataset/ResultState.java | 380 ------------------
.../nc/heartbeat/HeartbeatComputeTask.java | 10 +-
.../control/nc/net/DatasetNetworkManager.java | 168 --------
.../control/nc/net/ResultNetworkManager.java | 169 ++++++++
.../apache/hyracks/control/nc/result/Page.java | 37 ++
.../control/nc/result/ResultMemoryManager.java | 247 ++++++++++++
.../nc/result/ResultPartitionManager.java | 184 +++++++++
.../nc/result/ResultPartitionReader.java | 110 ++++++
.../nc/result/ResultPartitionWriter.java | 138 +++++++
.../hyracks/control/nc/result/ResultSetMap.java | 123 ++++++
.../hyracks/control/nc/result/ResultState.java | 381 +++++++++++++++++++
.../control/nc/work/AbortAllJobsWork.java | 14 +-
.../hyracks/control/nc/work/AbortTasksWork.java | 8 +-
.../control/nc/work/NotifyTaskFailureWork.java | 8 +-
.../result/ResultWriterOperatorDescriptor.java | 28 +-
.../integration/AbstractIntegrationTest.java | 12 +-
.../AbstractMultiNCIntegrationTest.java | 10 +-
.../tests/integration/AggregationTest.java | 2 +-
.../tests/integration/CountOfCountsTest.java | 2 +-
.../tests/integration/HeapSortMergeTest.java | 2 +-
.../integration/LocalityAwareConnectorTest.java | 2 +-
.../integration/NodesAPIIntegrationTest.java | 4 +-
.../integration/ReplicateOperatorTest.java | 2 +-
.../tests/integration/ScanPrintTest.java | 2 +-
.../tests/integration/SortMergeTest.java | 2 +-
.../TPCHCustomerOrderHashJoinTest.java | 2 +-
.../TPCHCustomerOrderNestedLoopJoinTest.java | 2 +-
.../hyracks/tests/integration/UnionTest.java | 2 +-
.../hyracks/test/support/TestTaskContext.java | 4 +-
.../util/LSMInvertedIndexTestUtils.java | 6 +-
121 files changed, 3390 insertions(+), 3443 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
index 0dbd3aa..58f0997 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IRequestParameters.java
@@ -22,14 +22,14 @@ import java.util.Map;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.translator.IStatementExecutor.Stats;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.api.result.IResultSet;
public interface IRequestParameters {
/**
- * @return A Hyracks dataset client object that is used to read the results.
+ * @return A Resultset client object that is used to read the results.
*/
- IHyracksDataset getHyracksDataset();
+ IResultSet getResultSet();
/**
* Gets the required result properties of the request.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index fa967f5..db9b743 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -36,7 +36,7 @@ import org.apache.commons.lang3.tuple.Triple;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.client.IClusterInfoCollector;
import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
index c5420ba..9844900 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
@@ -19,19 +19,16 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_DATASET_ATTR;
import java.io.PrintWriter;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;
-import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.http.server.AbstractServlet;
import org.apache.hyracks.ipc.exceptions.IPCException;
import org.apache.logging.log4j.Level;
@@ -105,31 +102,16 @@ public class AbstractQueryApiServlet extends AbstractServlet {
this.appCtx = appCtx;
}
- protected IHyracksDataset getHyracksDataset() throws Exception { // NOSONAR
+ protected IResultSet getResultSet() throws Exception { // NOSONAR
try {
- return doGetHyracksDataset();
+ return ServletUtil.getResultSet(getHyracksClientConnection(), appCtx, ctx);
} catch (IPCException e) {
LOGGER.log(Level.WARN, "Failed getting hyracks dataset connection. Resetting hyracks connection.", e);
ctx.put(HYRACKS_CONNECTION_ATTR, appCtx.getHcc());
- return doGetHyracksDataset();
+ return ServletUtil.getResultSet(getHyracksClientConnection(), appCtx, ctx);
}
}
- protected IHyracksDataset doGetHyracksDataset() throws Exception {
- IHyracksDataset hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
- if (hds == null) {
- synchronized (ctx) {
- hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
- if (hds == null) {
- hds = new HyracksDataset(getHyracksClientConnection(),
- appCtx.getCompilerProperties().getFrameSize(), ResultReader.NUM_READERS);
- ctx.put(HYRACKS_DATASET_ATTR, hds);
- }
- }
- }
- return hds;
- }
-
protected IHyracksClientConnection getHyracksClientConnection() throws Exception { // NOSONAR
IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
if (hcc == null) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java
index 466757e..b2c7c14 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ApiServlet.java
@@ -19,7 +19,6 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_DATASET_ATTR;
import java.awt.image.BufferedImage;
import java.io.BufferedReader;
@@ -33,7 +32,6 @@ import java.util.concurrent.ConcurrentMap;
import javax.imageio.ImageIO;
-import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.app.translator.RequestParameters;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.context.IStorageComponentProvider;
@@ -54,8 +52,7 @@ import org.apache.asterix.translator.SessionConfig.OutputFormat;
import org.apache.asterix.translator.SessionConfig.PlanFormat;
import org.apache.asterix.translator.SessionOutput;
import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.AbstractServlet;
@@ -137,17 +134,7 @@ public class ApiServlet extends AbstractServlet {
}
try {
IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
- IHyracksDataset hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
- if (hds == null) {
- synchronized (ctx) {
- hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
- if (hds == null) {
- hds = new HyracksDataset(hcc, appCtx.getCompilerProperties().getFrameSize(),
- ResultReader.NUM_READERS);
- ctx.put(HYRACKS_DATASET_ATTR, hds);
- }
- }
- }
+ IResultSet resultSet = ServletUtil.getResultSet(hcc, appCtx, ctx);
IParser parser = parserFactory.createParser(query);
List<Statement> aqlStatements = parser.parse();
SessionConfig sessionConfig = new SessionConfig(format, true, isSet(executeQuery), true, planFormat);
@@ -163,7 +150,7 @@ public class ApiServlet extends AbstractServlet {
double duration;
long startTime = System.currentTimeMillis();
final IRequestParameters requestParameters =
- new RequestParameters(hds, new ResultProperties(IStatementExecutor.ResultDelivery.IMMEDIATE),
+ new RequestParameters(resultSet, new ResultProperties(IStatementExecutor.ResultDelivery.IMMEDIATE),
new IStatementExecutor.Stats(), null, null, null, null, true);
translator.compileAndExecute(hcc, null, requestParameters);
long endTime = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 79c17c7..77b8647 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -45,7 +45,7 @@ import org.apache.asterix.translator.ResultProperties;
import org.apache.asterix.translator.SessionOutput;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.hyracks.api.application.INCServiceContext;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.http.api.IChannelClosedHandler;
import org.apache.hyracks.http.api.IServletRequest;
@@ -128,7 +128,7 @@ public class NCQueryServiceServlet extends QueryServiceServlet {
if (delivery == IStatementExecutor.ResultDelivery.IMMEDIATE && !resultMetadata.getResultSets().isEmpty()) {
stats.setProcessedObjects(responseMsg.getStats().getProcessedObjects());
for (Triple<JobId, ResultSetId, ARecordType> rsmd : resultMetadata.getResultSets()) {
- ResultReader resultReader = new ResultReader(getHyracksDataset(), rsmd.getLeft(), rsmd.getMiddle());
+ ResultReader resultReader = new ResultReader(getResultSet(), rsmd.getLeft(), rsmd.getMiddle());
ResultUtil.printResults(appCtx, resultReader, sessionOutput, stats, rsmd.getRight());
}
} else {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
index de96d54..6781f22 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
@@ -26,8 +26,8 @@ import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.translator.IStatementExecutor.Stats;
import org.apache.asterix.translator.SessionOutput;
-import org.apache.hyracks.api.dataset.DatasetJobRecord;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.api.result.ResultJobRecord;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
@@ -59,11 +59,11 @@ public class QueryResultApiServlet extends AbstractQueryApiServlet {
return;
}
- IHyracksDataset hds = getHyracksDataset();
- ResultReader resultReader = new ResultReader(hds, handle.getJobId(), handle.getResultSetId());
+ IResultSet resultSet = getResultSet();
+ ResultReader resultReader = new ResultReader(resultSet, handle.getJobId(), handle.getResultSetId());
try {
- DatasetJobRecord.Status status = resultReader.getStatus();
+ ResultJobRecord.Status status = resultReader.getStatus();
final HttpResponseStatus httpStatus;
if (status == null) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 0c5d216..54b5baa 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -593,8 +593,8 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
Map<String, IAObject> stmtParams =
org.apache.asterix.app.translator.RequestParameters.deserializeParameterValues(statementParameters);
IRequestParameters requestParameters =
- new org.apache.asterix.app.translator.RequestParameters(getHyracksDataset(), resultProperties, stats,
- null, param.getClientContextID(), optionalParameters, stmtParams, param.isMultiStatement());
+ new org.apache.asterix.app.translator.RequestParameters(getResultSet(), resultProperties, stats, null,
+ param.getClientContextID(), optionalParameters, stmtParams, param.isMultiStatement());
translator.compileAndExecute(getHyracksClientConnection(), queryCtx, requestParameters);
execution.end();
printExecutionPlans(sessionOutput, translator.getExecutionPlans());
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
index e55d82a..3773dc5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
@@ -28,8 +28,8 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.asterix.app.result.ResultHandle;
import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.common.api.IApplicationContext;
-import org.apache.hyracks.api.dataset.DatasetJobRecord;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.api.result.ResultJobRecord;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.utils.HttpUtil;
@@ -55,10 +55,9 @@ public class QueryStatusApiServlet extends AbstractQueryApiServlet {
return;
}
- IHyracksDataset hds = getHyracksDataset();
- ResultReader resultReader = new ResultReader(hds, handle.getJobId(), handle.getResultSetId());
+ ResultReader resultReader = new ResultReader(getResultSet(), handle.getJobId(), handle.getResultSetId());
- final DatasetJobRecord.Status resultReaderStatus = resultReader.getStatus();
+ final ResultJobRecord.Status resultReaderStatus = resultReader.getStatus();
if (resultReaderStatus == null) {
LOGGER.log(Level.INFO, "No results for: \"" + strHandle + "\"");
response.setStatus(HttpResponseStatus.NOT_FOUND);
@@ -96,7 +95,7 @@ public class QueryStatusApiServlet extends AbstractQueryApiServlet {
}
}
- ResultStatus resultStatus(DatasetJobRecord.Status status) {
+ ResultStatus resultStatus(ResultJobRecord.Status status) {
switch (status.getState()) {
case IDLE:
case RUNNING:
@@ -110,7 +109,7 @@ public class QueryStatusApiServlet extends AbstractQueryApiServlet {
}
}
- Exception extractException(DatasetJobRecord.Status status) {
+ Exception extractException(ResultJobRecord.Status status) {
switch (status.getState()) {
case FAILED:
List<Exception> exceptions = status.getExceptions();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
index 3c58bc6..99c7308 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/RestApiServlet.java
@@ -19,13 +19,11 @@
package org.apache.asterix.api.http.server;
import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_DATASET_ATTR;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
-import org.apache.asterix.app.result.ResultReader;
import org.apache.asterix.app.translator.QueryTranslator;
import org.apache.asterix.app.translator.RequestParameters;
import org.apache.asterix.common.config.GlobalConfig;
@@ -48,8 +46,7 @@ import org.apache.asterix.translator.SessionConfig.OutputFormat;
import org.apache.asterix.translator.SessionConfig.PlanFormat;
import org.apache.asterix.translator.SessionOutput;
import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.client.dataset.HyracksDataset;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.AbstractServlet;
@@ -192,24 +189,14 @@ public abstract class RestApiServlet extends AbstractServlet {
try {
response.setStatus(HttpResponseStatus.OK);
IHyracksClientConnection hcc = (IHyracksClientConnection) ctx.get(HYRACKS_CONNECTION_ATTR);
- IHyracksDataset hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
- if (hds == null) {
- synchronized (ctx) {
- hds = (IHyracksDataset) ctx.get(HYRACKS_DATASET_ATTR);
- if (hds == null) {
- hds = new HyracksDataset(hcc, appCtx.getCompilerProperties().getFrameSize(),
- ResultReader.NUM_READERS);
- ctx.put(HYRACKS_DATASET_ATTR, hds);
- }
- }
- }
IParser parser = parserFactory.createParser(query);
List<Statement> aqlStatements = parser.parse();
validate(aqlStatements);
MetadataManager.INSTANCE.init();
IStatementExecutor translator = statementExecutorFactory.create(appCtx, aqlStatements, sessionOutput,
compilationProvider, componentProvider);
- final IRequestParameters requestParameters = new RequestParameters(hds,
+ final IResultSet resultSet = ServletUtil.getResultSet(hcc, appCtx, ctx);
+ final IRequestParameters requestParameters = new RequestParameters(resultSet,
new ResultProperties(resultDelivery), new IStatementExecutor.Stats(), null, null, null, null, true);
translator.compileAndExecute(hcc, null, requestParameters);
} catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletConstants.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletConstants.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletConstants.java
index 2fe37c3..f62d0c4 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletConstants.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletConstants.java
@@ -20,7 +20,7 @@ package org.apache.asterix.api.http.server;
public class ServletConstants {
public static final String HYRACKS_CONNECTION_ATTR = "org.apache.asterix.HYRACKS_CONNECTION";
- public static final String HYRACKS_DATASET_ATTR = "org.apache.asterix.HYRACKS_DATASET";
+ public static final String RESULTSET_ATTR = "org.apache.asterix.RESULTSET";
public static final String ASTERIX_APP_CONTEXT_INFO_ATTR = "org.apache.asterix.APP_CONTEXT_INFO";
public static final String EXECUTOR_SERVICE_ATTR = "org.apache.asterix.EXECUTOR_SERVICE";
public static final String RUNNING_QUERIES_ATTR = "org.apache.asterix.RUNINNG_QUERIES";
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletUtil.java
new file mode 100644
index 0000000..17e4c16
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ServletUtil.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.http.server;
+
+import static org.apache.asterix.api.http.server.ServletConstants.RESULTSET_ATTR;
+
+import java.util.Map;
+
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.client.result.ResultSet;
+
+public class ServletUtil {
+ static IResultSet getResultSet(IHyracksClientConnection hcc, IApplicationContext appCtx,
+ final Map<String, Object> ctx) throws Exception {
+ IResultSet resultSet = (IResultSet) ctx.get(RESULTSET_ATTR);
+ if (resultSet == null) {
+ synchronized (ctx) {
+ resultSet = (IResultSet) ctx.get(RESULTSET_ATTR);
+ if (resultSet == null) {
+ resultSet =
+ new ResultSet(hcc, appCtx.getCompilerProperties().getFrameSize(), ResultReader.NUM_READERS);
+ ctx.put(RESULTSET_ATTR, resultSet);
+ }
+ }
+ }
+ return resultSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java
index bbb9b99..a17fa90 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.app.result;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.job.JobId;
public class ResultHandle {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
index eeb01ba..908e3ca 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
@@ -20,24 +20,24 @@ package org.apache.asterix.app.result;
import org.apache.hyracks.api.comm.IFrame;
import org.apache.hyracks.api.comm.IFrameTupleAccessor;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.result.IResultSetReader;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.dataflow.common.comm.io.ResultFrameTupleAccessor;
public class ResultReader {
- private IHyracksDatasetReader reader;
+ private IResultSetReader reader;
private IFrameTupleAccessor frameTupleAccessor;
// Number of parallel result reader buffers
public static final int NUM_READERS = 1;
- public ResultReader(IHyracksDataset hdc, JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
- reader = hdc.createReader(jobId, resultSetId);
+ public ResultReader(IResultSet resultSet, JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
+ reader = resultSet.createReader(jobId, resultSetId);
frameTupleAccessor = new ResultFrameTupleAccessor();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 5b4c198..02941d1 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -186,8 +186,6 @@ import org.apache.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
import org.apache.hyracks.api.client.IClusterInfoCollector;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.io.FileSplit;
@@ -196,6 +194,8 @@ import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.api.job.JobStatus;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.control.cc.ClusterControllerService;
import org.apache.hyracks.control.cc.job.IJobManager;
import org.apache.hyracks.control.cc.job.JobRun;
@@ -281,7 +281,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
String threadName = Thread.currentThread().getName();
Thread.currentThread().setName(QueryTranslator.class.getSimpleName());
Map<String, String> config = new HashMap<>();
- final IHyracksDataset hdc = requestParameters.getHyracksDataset();
+ final IResultSet resultSet = requestParameters.getResultSet();
final ResultDelivery resultDelivery = requestParameters.getResultProperties().getDelivery();
final long maxResultReads = requestParameters.getResultProperties().getMaxReads();
final Stats stats = requestParameters.getStats();
@@ -355,7 +355,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
|| resultDelivery == ResultDelivery.DEFERRED);
metadataProvider.setMaxResultReads(maxResultReads);
}
- handleInsertUpsertStatement(metadataProvider, stmt, hcc, hdc, resultDelivery, outMetadata,
+ handleInsertUpsertStatement(metadataProvider, stmt, hcc, resultSet, resultDelivery, outMetadata,
stats, false, clientContextId, stmtParams, stmtRewriter);
break;
case DELETE:
@@ -390,7 +390,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
metadataProvider.setResultAsyncMode(
resultDelivery == ResultDelivery.ASYNC || resultDelivery == ResultDelivery.DEFERRED);
metadataProvider.setMaxResultReads(maxResultReads);
- handleQuery(metadataProvider, (Query) stmt, hcc, hdc, resultDelivery, outMetadata, stats,
+ handleQuery(metadataProvider, (Query) stmt, hcc, resultSet, resultDelivery, outMetadata, stats,
clientContextId, ctx, stmtParams, stmtRewriter);
break;
case COMPACT:
@@ -1867,7 +1867,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
public JobSpecification handleInsertUpsertStatement(MetadataProvider metadataProvider, Statement stmt,
- IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery,
+ IHyracksClientConnection hcc, IResultSet resultSet, ResultDelivery resultDelivery,
ResultMetadata outMetadata, Stats stats, boolean compileOnly, String clientContextId,
Map<String, IAObject> stmtParams, IStatementRewriter stmtRewriter) throws Exception {
InsertStatement stmtInsertUpsert = (InsertStatement) stmt;
@@ -1912,7 +1912,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
if (stmtInsertUpsert.getReturnExpression() != null) {
- deliverResult(hcc, hdc, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats,
+ deliverResult(hcc, resultSet, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats,
clientContextId, NoOpStatementExecutorContext.INSTANCE);
} else {
locker.lock();
@@ -2465,7 +2465,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
}
protected void handleQuery(MetadataProvider metadataProvider, Query query, IHyracksClientConnection hcc,
- IHyracksDataset hdc, ResultDelivery resultDelivery, ResultMetadata outMetadata, Stats stats,
+ IResultSet resultSet, ResultDelivery resultDelivery, ResultMetadata outMetadata, Stats stats,
String clientContextId, IStatementExecutorContext ctx, Map<String, IAObject> stmtParams,
IStatementRewriter stmtRewriter) throws Exception {
final IMetadataLocker locker = new IMetadataLocker() {
@@ -2499,11 +2499,11 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
throw e;
}
};
- deliverResult(hcc, hdc, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats, clientContextId,
- ctx);
+ deliverResult(hcc, resultSet, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats,
+ clientContextId, ctx);
}
- private void deliverResult(IHyracksClientConnection hcc, IHyracksDataset hdc, IStatementCompiler compiler,
+ private void deliverResult(IHyracksClientConnection hcc, IResultSet resultSet, IStatementCompiler compiler,
MetadataProvider metadataProvider, IMetadataLocker locker, ResultDelivery resultDelivery,
ResultMetadata outMetadata, Stats stats, String clientContextId, IStatementExecutorContext ctx)
throws Exception {
@@ -2521,7 +2521,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
break;
case IMMEDIATE:
createAndRunJob(hcc, jobFlags, null, compiler, locker, resultDelivery, id -> {
- final ResultReader resultReader = new ResultReader(hdc, id, resultSetId);
+ final ResultReader resultReader = new ResultReader(resultSet, id, resultSetId);
updateJobStats(id, stats);
// stop buffering and allow for streaming result delivery
sessionOutput.release();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
index ad12125..5f76568 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/RequestParameters.java
@@ -33,15 +33,15 @@ import org.apache.asterix.translator.IStatementExecutor;
import org.apache.asterix.translator.IStatementExecutor.Stats;
import org.apache.asterix.translator.ResultProperties;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.result.IResultSet;
import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
import com.fasterxml.jackson.databind.JsonNode;
public class RequestParameters implements IRequestParameters {
- private final IHyracksDataset hdc;
+ private final IResultSet resultSet;
private final ResultProperties resultProperties;
private final Stats stats;
private final Map<String, String> optionalParameters;
@@ -50,10 +50,10 @@ public class RequestParameters implements IRequestParameters {
private final Map<String, IAObject> statementParameters;
private final boolean multiStatement;
- public RequestParameters(IHyracksDataset hdc, ResultProperties resultProperties, Stats stats,
+ public RequestParameters(IResultSet resultSet, ResultProperties resultProperties, Stats stats,
IStatementExecutor.ResultMetadata outMetadata, String clientContextId,
Map<String, String> optionalParameters, Map<String, IAObject> statementParameters, boolean multiStatement) {
- this.hdc = hdc;
+ this.resultSet = resultSet;
this.resultProperties = resultProperties;
this.stats = stats;
this.outMetadata = outMetadata;
@@ -64,8 +64,8 @@ public class RequestParameters implements IRequestParameters {
}
@Override
- public IHyracksDataset getHyracksDataset() {
- return hdc;
+ public IResultSet getResultSet() {
+ return resultSet;
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index f3f5c56..695dcb8 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -122,7 +122,7 @@ import org.apache.hyracks.api.dataflow.value.IResultSerializerFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.io.FileSplit;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.data.std.primitive.ShortPointable;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ResultSetSinkId.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ResultSetSinkId.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ResultSetSinkId.java
index 51feb56..2fb7344 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ResultSetSinkId.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/ResultSetSinkId.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.metadata.declared;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
public class ResultSetSinkId {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceFunctions.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceFunctions.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceFunctions.java
index ad2e77a..72bdc3e 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceFunctions.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceFunctions.java
@@ -25,12 +25,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.hyracks.api.dataset.DatasetDirectoryRecord;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.job.DeployedJobSpecId;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultSetId;
public class HyracksClientInterfaceFunctions {
public enum FunctionId {
@@ -43,9 +43,9 @@ public class HyracksClientInterfaceFunctions {
UNDEPLOY_JOB,
REDEPLOY_JOB,
CANCEL_JOB,
- GET_DATASET_DIRECTORY_SERIVICE_INFO,
- GET_DATASET_RESULT_STATUS,
- GET_DATASET_RESULT_LOCATIONS,
+ GET_RESULT_DIRECTORY_ADDRESS,
+ GET_RESULT_STATUS,
+ GET_RESULT_LOCATIONS,
WAIT_FOR_COMPLETION,
GET_NODE_CONTROLLERS_INFO,
CLI_DEPLOY_BINARY,
@@ -250,30 +250,30 @@ public class HyracksClientInterfaceFunctions {
}
}
- public static class GetDatasetDirectoryServiceInfoFunction extends Function {
+ public static class GetResultDirectoryAddressFunction extends Function {
private static final long serialVersionUID = 1L;
@Override
public FunctionId getFunctionId() {
- return FunctionId.GET_DATASET_DIRECTORY_SERIVICE_INFO;
+ return FunctionId.GET_RESULT_DIRECTORY_ADDRESS;
}
}
- public static class GetDatasetResultStatusFunction extends Function {
+ public static class GetResultStatusFunction extends Function {
private static final long serialVersionUID = 1L;
private final JobId jobId;
private final ResultSetId rsId;
- public GetDatasetResultStatusFunction(JobId jobId, ResultSetId rsId) {
+ public GetResultStatusFunction(JobId jobId, ResultSetId rsId) {
this.jobId = jobId;
this.rsId = rsId;
}
@Override
public FunctionId getFunctionId() {
- return FunctionId.GET_DATASET_RESULT_STATUS;
+ return FunctionId.GET_RESULT_STATUS;
}
public JobId getJobId() {
@@ -285,16 +285,16 @@ public class HyracksClientInterfaceFunctions {
}
}
- public static class GetDatasetResultLocationsFunction extends Function {
+ public static class GetResultLocationsFunction extends Function {
private static final long serialVersionUID = 1L;
private final JobId jobId;
private final ResultSetId rsId;
- private final DatasetDirectoryRecord[] knownRecords;
+ private final ResultDirectoryRecord[] knownRecords;
- public GetDatasetResultLocationsFunction(JobId jobId, ResultSetId rsId, DatasetDirectoryRecord[] knownRecords) {
+ public GetResultLocationsFunction(JobId jobId, ResultSetId rsId, ResultDirectoryRecord[] knownRecords) {
this.jobId = jobId;
this.rsId = rsId;
this.knownRecords = knownRecords;
@@ -302,7 +302,7 @@ public class HyracksClientInterfaceFunctions {
@Override
public FunctionId getFunctionId() {
- return FunctionId.GET_DATASET_RESULT_LOCATIONS;
+ return FunctionId.GET_RESULT_LOCATIONS;
}
public JobId getJobId() {
@@ -313,7 +313,7 @@ public class HyracksClientInterfaceFunctions {
return rsId;
}
- public DatasetDirectoryRecord[] getKnownRecords() {
+ public ResultDirectoryRecord[] getKnownRecords() {
return knownRecords;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
index 1ee9bd8..63a32f7 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
@@ -112,9 +112,9 @@ public class HyracksClientInterfaceRemoteProxy implements IHyracksClientInterfac
}
@Override
- public NetworkAddress getDatasetDirectoryServiceInfo() throws Exception {
- HyracksClientInterfaceFunctions.GetDatasetDirectoryServiceInfoFunction gddsf =
- new HyracksClientInterfaceFunctions.GetDatasetDirectoryServiceInfoFunction();
+ public NetworkAddress getResultDirectoryAddress() throws Exception {
+ HyracksClientInterfaceFunctions.GetResultDirectoryAddressFunction gddsf =
+ new HyracksClientInterfaceFunctions.GetResultDirectoryAddressFunction();
return (NetworkAddress) rpci.call(ipcHandle, gddsf);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksConnection.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksConnection.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksConnection.java
index eaec3c3..48c656f 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksConnection.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/HyracksConnection.java
@@ -171,8 +171,8 @@ public final class HyracksConnection implements IHyracksClientConnection {
}
@Override
- public NetworkAddress getDatasetDirectoryServiceInfo() throws Exception {
- return hci.getDatasetDirectoryServiceInfo();
+ public NetworkAddress getResultDirectoryAddress() throws Exception {
+ return hci.getResultDirectoryAddress();
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientConnection.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientConnection.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientConnection.java
index b3b7677..89f2ad4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientConnection.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientConnection.java
@@ -141,12 +141,12 @@ public interface IHyracksClientConnection extends IClusterInfoCollector {
JobId startJob(IActivityClusterGraphGeneratorFactory acggf, EnumSet<JobFlag> jobFlags) throws Exception;
/**
- * Gets the IP Address and port for the DatasetDirectoryService wrapped in NetworkAddress
+ * Gets the IP Address and port for the ResultDirectoryService wrapped in NetworkAddress
*
* @return {@link NetworkAddress}
* @throws Exception
*/
- NetworkAddress getDatasetDirectoryServiceInfo() throws Exception;
+ NetworkAddress getResultDirectoryAddress() throws Exception;
/**
* Waits until the specified job has completed, either successfully or has
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientInterface.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientInterface.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientInterface.java
index 6a75806..4cc47d2 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientInterface.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/IHyracksClientInterface.java
@@ -49,7 +49,7 @@ public interface IHyracksClientInterface {
public void undeployJobSpec(DeployedJobSpecId deployedJobSpecId) throws Exception;
- public NetworkAddress getDatasetDirectoryServiceInfo() throws Exception;
+ public NetworkAddress getResultDirectoryAddress() throws Exception;
public void waitForCompletion(JobId jobId) throws Exception;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeControllerInfo.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeControllerInfo.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeControllerInfo.java
index 57f389f..aab6b2b 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeControllerInfo.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/client/NodeControllerInfo.java
@@ -31,18 +31,18 @@ public class NodeControllerInfo implements Serializable {
private final NetworkAddress netAddress;
- private final NetworkAddress datasetNetworkAddress;
+ private final NetworkAddress resultNetworkAddress;
private final NetworkAddress messagingNetworkAddress;
private final int numAvailableCores;
public NodeControllerInfo(String nodeId, NodeStatus status, NetworkAddress netAddress,
- NetworkAddress datasetNetworkAddress, NetworkAddress messagingNetworkAddress, int numAvailableCores) {
+ NetworkAddress resultNetworkAddress, NetworkAddress messagingNetworkAddress, int numAvailableCores) {
this.nodeId = nodeId;
this.status = status;
this.netAddress = netAddress;
- this.datasetNetworkAddress = datasetNetworkAddress;
+ this.resultNetworkAddress = resultNetworkAddress;
this.messagingNetworkAddress = messagingNetworkAddress;
this.numAvailableCores = numAvailableCores;
}
@@ -59,8 +59,8 @@ public class NodeControllerInfo implements Serializable {
return netAddress;
}
- public NetworkAddress getDatasetNetworkAddress() {
- return datasetNetworkAddress;
+ public NetworkAddress getResultNetworkAddress() {
+ return resultNetworkAddress;
}
public NetworkAddress getMessagingNetworkAddress() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
index bf42d0c..09b6506 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
@@ -23,7 +23,7 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksException;
import org.apache.hyracks.api.io.IWorkspaceFileFactory;
@@ -43,7 +43,7 @@ public interface IHyracksTaskContext
ExecutorService getExecutorService();
- IDatasetPartitionManager getDatasetPartitionManager();
+ IResultPartitionManager getResultPartitionManager();
void sendApplicationMessageToCC(Serializable message, DeploymentId deploymentId) throws Exception;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetDirectoryRecord.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetDirectoryRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetDirectoryRecord.java
deleted file mode 100644
index e47b1e2..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetDirectoryRecord.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import java.io.Serializable;
-
-import org.apache.hyracks.api.comm.NetworkAddress;
-
-public class DatasetDirectoryRecord implements Serializable {
- public enum Status {
- IDLE,
- RUNNING,
- SUCCESS,
- FAILED
- }
-
- private static final long serialVersionUID = 1L;
-
- private NetworkAddress address;
-
- private boolean readEOS;
-
- private Status status;
-
- private boolean empty;
-
- public DatasetDirectoryRecord() {
- this.address = null;
- this.readEOS = false;
- this.status = Status.IDLE;
- }
-
- public void setNetworkAddress(NetworkAddress address) {
- this.address = address;
- }
-
- public NetworkAddress getNetworkAddress() {
- return address;
- }
-
- public void setEmpty(boolean empty) {
- this.empty = empty;
- }
-
- public boolean isEmpty() {
- return empty;
- }
-
- public void readEOS() {
- this.readEOS = true;
- }
-
- public boolean hasReachedReadEOS() {
- return readEOS;
- }
-
- public void start() {
- updateStatus(Status.RUNNING);
- }
-
- public void writeEOS() {
- updateStatus(Status.SUCCESS);
- }
-
- public void fail() {
- status = Status.FAILED;
- }
-
- private void updateStatus(final DatasetDirectoryRecord.Status newStatus) {
- // FAILED is a stable status
- if (status != Status.FAILED) {
- status = newStatus;
- }
- }
-
- public Status getStatus() {
- return status;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
- if (!(o instanceof DatasetDirectoryRecord)) {
- return false;
- }
- return address.equals(((DatasetDirectoryRecord) o).address);
- }
-
- @Override
- public String toString() {
- return String.valueOf(address) + " " + status + (empty ? " (empty)" : "") + (readEOS ? " (EOS)" : "");
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java
deleted file mode 100644
index 4e7ddda..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/DatasetJobRecord.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class DatasetJobRecord implements IDatasetStateRecord {
- public enum State {
- IDLE,
- RUNNING,
- SUCCESS,
- FAILED
- }
-
- public static class Status implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- State state = State.IDLE;
-
- private List<Exception> exceptions;
-
- public State getState() {
- return state;
- }
-
- void setState(State state) {
- this.state = state;
- }
-
- public List<Exception> getExceptions() {
- return exceptions;
- }
-
- void setExceptions(List<Exception> exceptions) {
- this.exceptions = exceptions;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{ \"state\": \"").append(state.name()).append("\"");
- if (exceptions != null && !exceptions.isEmpty()) {
- sb.append(", \"exceptions\": ");
- List<String> msgs = new ArrayList<>();
- exceptions.forEach(e -> msgs.add("\"" + e.getMessage() + "\""));
- sb.append(Arrays.toString(msgs.toArray()));
- }
- sb.append(" }");
- return sb.toString();
- }
- }
-
- private static final long serialVersionUID = 1L;
-
- private final long timestamp;
-
- private Status status;
-
- private Map<ResultSetId, ResultSetMetaData> resultSetMetadataMap = new HashMap<>();
-
- public DatasetJobRecord() {
- this.timestamp = System.nanoTime();
- this.status = new Status();
- }
-
- private void updateState(State newStatus) {
- // FAILED is a stable status
- if (status.state != State.FAILED) {
- status.setState(newStatus);
- }
- }
-
- public void start() {
- updateState(State.RUNNING);
- }
-
- public void success() {
- updateState(State.SUCCESS);
- }
-
- public void fail(ResultSetId rsId, int partition) {
- getOrCreateDirectoryRecord(rsId, partition).fail();
- }
-
- public void fail(List<Exception> exceptions) {
- updateState(State.FAILED);
- status.setExceptions(exceptions);
- }
-
- @Override
- public long getTimestamp() {
- return timestamp;
- }
-
- public Status getStatus() {
- return status;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{ \"status\": ").append(status.toString()).append(", ");
- sb.append("\"timestamp\": ").append(timestamp).append(", ");
- sb.append("\"resultsets\": ").append(Arrays.toString(resultSetMetadataMap.entrySet().toArray())).append(" }");
- return sb.toString();
- }
-
- public void setResultSetMetaData(ResultSetId rsId, boolean orderedResult, int nPartitions)
- throws HyracksDataException {
- ResultSetMetaData rsMd = resultSetMetadataMap.get(rsId);
- if (rsMd == null) {
- resultSetMetadataMap.put(rsId, new ResultSetMetaData(nPartitions, orderedResult));
- } else if (rsMd.getOrderedResult() != orderedResult || rsMd.getRecords().length != nPartitions) {
- throw HyracksDataException.create(ErrorCode.INCONSISTENT_RESULT_METADATA, rsId.toString());
- }
- //TODO(tillw) throwing a HyracksDataException here hangs the execution tests
- }
-
- public ResultSetMetaData getResultSetMetaData(ResultSetId rsId) {
- return resultSetMetadataMap.get(rsId);
- }
-
- public synchronized DatasetDirectoryRecord getOrCreateDirectoryRecord(ResultSetId rsId, int partition) {
- DatasetDirectoryRecord[] records = getResultSetMetaData(rsId).getRecords();
- if (records[partition] == null) {
- records[partition] = new DatasetDirectoryRecord();
- }
- return records[partition];
- }
-
- public synchronized DatasetDirectoryRecord getDirectoryRecord(ResultSetId rsId, int partition)
- throws HyracksDataException {
- DatasetDirectoryRecord[] records = getResultSetMetaData(rsId).getRecords();
- if (records[partition] == null) {
- throw HyracksDataException.create(ErrorCode.RESULT_NO_RECORD, partition, rsId);
- }
- return records[partition];
- }
-
- public synchronized void updateState(ResultSetId rsId) {
- int successCount = 0;
- DatasetDirectoryRecord[] records = getResultSetMetaData(rsId).getRecords();
- for (DatasetDirectoryRecord record : records) {
- if ((record != null) && (record.getStatus() == DatasetDirectoryRecord.Status.SUCCESS)) {
- successCount++;
- }
- }
- if (successCount == records.length) {
- success();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetManager.java
deleted file mode 100644
index a0c1f78..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetManager.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import java.util.Set;
-
-import org.apache.hyracks.api.job.JobId;
-
-public interface IDatasetManager {
-
- Set<JobId> getJobIds();
-
- IDatasetStateRecord getState(JobId jobId);
-
- void sweep(JobId jobId);
-
- /**
- * Removes all references and deletes persisted files for
- * all expired datasets.
- */
- void sweepExpiredDatasets();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetPartitionManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetPartitionManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetPartitionManager.java
deleted file mode 100644
index b1e203f..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetPartitionManager.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.job.JobId;
-
-public interface IDatasetPartitionManager extends IDatasetManager {
- IFrameWriter createDatasetPartitionWriter(IHyracksTaskContext ctx, ResultSetId rsId, boolean orderedResult,
- boolean asyncMode, int partition, int nPartitions, long maxReads) throws HyracksException;
-
- void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, int partition, int nPartitions,
- boolean orderedResult, boolean emptyResult) throws HyracksException;
-
- void reportPartitionWriteCompletion(JobId jobId, ResultSetId resultSetId, int partition) throws HyracksException;
-
- void initializeDatasetPartitionReader(JobId jobId, ResultSetId resultSetId, int partition, IFrameWriter noc)
- throws HyracksException;
-
- void removePartition(JobId jobId, ResultSetId resultSetId, int partition);
-
- void abortReader(JobId jobId);
-
- void close();
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetStateRecord.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetStateRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetStateRecord.java
deleted file mode 100644
index d18e6cf..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IDatasetStateRecord.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-public interface IDatasetStateRecord {
- public long getTimestamp();
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDataset.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDataset.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDataset.java
deleted file mode 100644
index 0fa5c75..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDataset.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobId;
-
-public interface IHyracksDataset {
- public IHyracksDatasetReader createReader(JobId jobId, ResultSetId resultSetId) throws HyracksDataException;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceConnection.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceConnection.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceConnection.java
deleted file mode 100644
index 200f5e5..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceConnection.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.job.JobId;
-
-public interface IHyracksDatasetDirectoryServiceConnection {
- /**
- * Gets the result status for the given result set.
- *
- * @param jobId
- * ID of the job
- * @param rsId
- * ID of the result set
- * @return {@link Status}
- * @throws Exception
- */
- public Status getDatasetResultStatus(JobId jobId, ResultSetId rsId) throws Exception;
-
- /**
- * Gets the IP Addresses and ports for the partition generating the result for each location.
- *
- * @param jobId
- * ID of the job
- * @param rsId
- * ID of the result set
- * @param knownRecords
- * Locations that are already known to the client
- * @return {@link NetworkAddress[]}
- * @throws Exception
- */
- public DatasetDirectoryRecord[] getDatasetResultLocations(JobId jobId, ResultSetId rsId,
- DatasetDirectoryRecord[] knownRecords) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceInterface.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceInterface.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceInterface.java
deleted file mode 100644
index 56c0af9..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetDirectoryServiceInterface.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.job.JobId;
-
-public interface IHyracksDatasetDirectoryServiceInterface {
- /**
- * Gets the result status for the given result set.
- *
- * @param jobId
- * ID of the job
- * @param rsId
- * ID of the result set
- * @return {@link Status}
- * @throws Exception
- */
- public Status getDatasetResultStatus(JobId jobId, ResultSetId rsId) throws Exception;
-
- /**
- * Gets the IP Addresses and ports for the partition generating the result for each location.
- *
- * @param jobId
- * ID of the job
- * @param rsId
- * ID of the result set
- * @param knownRecords
- * Locations from the dataset directory that are already known to the client
- * @return {@link NetworkAddress[]}
- * @throws Exception
- */
- public DatasetDirectoryRecord[] getDatasetResultLocations(JobId jobId, ResultSetId rsId,
- DatasetDirectoryRecord[] knownRecords) throws Exception;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetReader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetReader.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetReader.java
deleted file mode 100644
index 858faaa..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/IHyracksDatasetReader.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public interface IHyracksDatasetReader {
- public Status getResultStatus();
-
- public int read(IFrame frame) throws HyracksDataException;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetId.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetId.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetId.java
deleted file mode 100644
index 1e21e19..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetId.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import java.io.Serializable;
-
-public class ResultSetId implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private final long id;
-
- public ResultSetId(long id) {
- this.id = id;
- }
-
- public long getId() {
- return id;
- }
-
- @Override
- public int hashCode() {
- return (int) id;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o == this) {
- return true;
- }
- if (!(o instanceof ResultSetId)) {
- return false;
- }
- return ((ResultSetId) o).id == id;
- }
-
- @Override
- public String toString() {
- return "RSID:" + id;
- }
-}
[4/5] asterixdb git commit: [ASTERIXDB-2314][HYR] Dataset in class
names in Hyracks
Posted by ti...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetMetaData.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetMetaData.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetMetaData.java
deleted file mode 100644
index 8e9e3dc..0000000
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/dataset/ResultSetMetaData.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.api.dataset;
-
-import java.util.Arrays;
-
-public class ResultSetMetaData {
- private final DatasetDirectoryRecord[] records;
- private final boolean ordered;
-
- ResultSetMetaData(int len, boolean ordered) {
- this.records = new DatasetDirectoryRecord[len];
- this.ordered = ordered;
- }
-
- public boolean getOrderedResult() {
- return ordered;
- }
-
- public DatasetDirectoryRecord[] getRecords() {
- return records;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{ordered: ").append(ordered).append(", records: ").append(Arrays.toString(records));
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
index c4c7320..b0a8017 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/JobSpecification.java
@@ -40,7 +40,7 @@ import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.OperatorDescriptorId;
import org.apache.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.job.resource.ClusterCapacity;
import org.apache.hyracks.api.job.resource.IClusterCapacity;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/partitions/ResultSetPartitionId.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/partitions/ResultSetPartitionId.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/partitions/ResultSetPartitionId.java
index 8a3e15a..8343fe0 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/partitions/ResultSetPartitionId.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/partitions/ResultSetPartitionId.java
@@ -20,7 +20,7 @@ package org.apache.hyracks.api.partitions;
import java.io.Serializable;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.job.JobId;
public final class ResultSetPartitionId implements Serializable {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultDirectory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultDirectory.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultDirectory.java
new file mode 100644
index 0000000..41b9d1a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultDirectory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.job.JobId;
+
+public interface IResultDirectory {
+ /**
+ * Gets the result status for the given result set.
+ *
+ * @param jobId
+ * ID of the job
+ * @param rsId
+ * ID of the result set
+ * @return {@link Status}
+ * @throws Exception
+ */
+ Status getResultStatus(JobId jobId, ResultSetId rsId) throws Exception;
+
+ /**
+ * Gets the IP Addresses and ports for the partition generating the result for each location.
+ *
+ * @param jobId
+ * ID of the job
+ * @param rsId
+ * ID of the result set
+ * @param knownRecords
+ * Locations that are already known to the client
+ * @return {@link ResultDirectoryRecord[]}
+ * @throws Exception
+ */
+ ResultDirectoryRecord[] getResultLocations(JobId jobId, ResultSetId rsId, ResultDirectoryRecord[] knownRecords)
+ throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultManager.java
new file mode 100644
index 0000000..d45cf44
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultManager.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import java.util.Set;
+
+import org.apache.hyracks.api.job.JobId;
+
+public interface IResultManager {
+
+ Set<JobId> getJobIds();
+
+ IResultStateRecord getState(JobId jobId);
+
+ void sweep(JobId jobId);
+
+ /**
+ * Removes all references and deletes persisted files for
+ * all expired results.
+ */
+ void sweepExpiredResultSets();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java
new file mode 100644
index 0000000..a539d37
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import org.apache.hyracks.api.comm.IFrameWriter;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.job.JobId;
+
+public interface IResultPartitionManager extends IResultManager {
+ IFrameWriter createResultPartitionWriter(IHyracksTaskContext ctx, ResultSetId rsId, boolean orderedResult,
+ boolean asyncMode, int partition, int nPartitions, long maxReads) throws HyracksException;
+
+ void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, int partition, int nPartitions,
+ boolean orderedResult, boolean emptyResult) throws HyracksException;
+
+ void reportPartitionWriteCompletion(JobId jobId, ResultSetId resultSetId, int partition) throws HyracksException;
+
+ void initializeResultPartitionReader(JobId jobId, ResultSetId resultSetId, int partition, IFrameWriter noc)
+ throws HyracksException;
+
+ void removePartition(JobId jobId, ResultSetId resultSetId, int partition);
+
+ void abortReader(JobId jobId);
+
+ void close();
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
new file mode 100644
index 0000000..ff2e48a
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+
+public interface IResultSet {
+ IResultSetReader createReader(JobId jobId, ResultSetId resultSetId) throws HyracksDataException;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
new file mode 100644
index 0000000..0884c53
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IResultSetReader {
+ Status getResultStatus();
+
+ int read(IFrame frame) throws HyracksDataException;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultStateRecord.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultStateRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultStateRecord.java
new file mode 100644
index 0000000..fe6bc15
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultStateRecord.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+public interface IResultStateRecord {
+ long getTimestamp();
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java
new file mode 100644
index 0000000..71792be
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import java.io.Serializable;
+
+import org.apache.hyracks.api.comm.NetworkAddress;
+
+public class ResultDirectoryRecord implements Serializable {
+ public enum Status {
+ IDLE,
+ RUNNING,
+ SUCCESS,
+ FAILED
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ private NetworkAddress address;
+
+ private boolean readEOS;
+
+ private Status status;
+
+ private boolean empty;
+
+ public ResultDirectoryRecord() {
+ this.address = null;
+ this.readEOS = false;
+ this.status = Status.IDLE;
+ }
+
+ public void setNetworkAddress(NetworkAddress address) {
+ this.address = address;
+ }
+
+ public NetworkAddress getNetworkAddress() {
+ return address;
+ }
+
+ public void setEmpty(boolean empty) {
+ this.empty = empty;
+ }
+
+ public boolean isEmpty() {
+ return empty;
+ }
+
+ public void readEOS() {
+ this.readEOS = true;
+ }
+
+ public boolean hasReachedReadEOS() {
+ return readEOS;
+ }
+
+ public void start() {
+ updateStatus(Status.RUNNING);
+ }
+
+ public void writeEOS() {
+ updateStatus(Status.SUCCESS);
+ }
+
+ public void fail() {
+ status = Status.FAILED;
+ }
+
+ private void updateStatus(final ResultDirectoryRecord.Status newStatus) {
+ // FAILED is a stable status
+ if (status != Status.FAILED) {
+ status = newStatus;
+ }
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof ResultDirectoryRecord)) {
+ return false;
+ }
+ return address.equals(((ResultDirectoryRecord) o).address);
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(address) + " " + status + (empty ? " (empty)" : "") + (readEOS ? " (EOS)" : "");
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
new file mode 100644
index 0000000..b8ddbd2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class ResultJobRecord implements IResultStateRecord {
+ public enum State {
+ IDLE,
+ RUNNING,
+ SUCCESS,
+ FAILED
+ }
+
+ public static class Status implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ State state = State.IDLE;
+
+ private List<Exception> exceptions;
+
+ public State getState() {
+ return state;
+ }
+
+ void setState(State state) {
+ this.state = state;
+ }
+
+ public List<Exception> getExceptions() {
+ return exceptions;
+ }
+
+ void setExceptions(List<Exception> exceptions) {
+ this.exceptions = exceptions;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{ \"state\": \"").append(state.name()).append("\"");
+ if (exceptions != null && !exceptions.isEmpty()) {
+ sb.append(", \"exceptions\": ");
+ List<String> msgs = new ArrayList<>();
+ exceptions.forEach(e -> msgs.add("\"" + e.getMessage() + "\""));
+ sb.append(Arrays.toString(msgs.toArray()));
+ }
+ sb.append(" }");
+ return sb.toString();
+ }
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ private final long timestamp;
+
+ private Status status;
+
+ private Map<ResultSetId, ResultSetMetaData> resultSetMetadataMap = new HashMap<>();
+
+ public ResultJobRecord() {
+ this.timestamp = System.nanoTime();
+ this.status = new Status();
+ }
+
+ private void updateState(State newStatus) {
+ // FAILED is a stable status
+ if (status.state != State.FAILED) {
+ status.setState(newStatus);
+ }
+ }
+
+ public void start() {
+ updateState(State.RUNNING);
+ }
+
+ public void success() {
+ updateState(State.SUCCESS);
+ }
+
+ public void fail(ResultSetId rsId, int partition) {
+ getOrCreateDirectoryRecord(rsId, partition).fail();
+ }
+
+ public void fail(List<Exception> exceptions) {
+ updateState(State.FAILED);
+ status.setExceptions(exceptions);
+ }
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{ \"status\": ").append(status.toString()).append(", ");
+ sb.append("\"timestamp\": ").append(timestamp).append(", ");
+ sb.append("\"resultsets\": ").append(Arrays.toString(resultSetMetadataMap.entrySet().toArray())).append(" }");
+ return sb.toString();
+ }
+
+ public void setResultSetMetaData(ResultSetId rsId, boolean orderedResult, int nPartitions)
+ throws HyracksDataException {
+ ResultSetMetaData rsMd = resultSetMetadataMap.get(rsId);
+ if (rsMd == null) {
+ resultSetMetadataMap.put(rsId, new ResultSetMetaData(nPartitions, orderedResult));
+ } else if (rsMd.getOrderedResult() != orderedResult || rsMd.getRecords().length != nPartitions) {
+ throw HyracksDataException.create(ErrorCode.INCONSISTENT_RESULT_METADATA, rsId.toString());
+ }
+ //TODO(tillw) throwing a HyracksDataException here hangs the execution tests
+ }
+
+ public ResultSetMetaData getResultSetMetaData(ResultSetId rsId) {
+ return resultSetMetadataMap.get(rsId);
+ }
+
+ public synchronized ResultDirectoryRecord getOrCreateDirectoryRecord(ResultSetId rsId, int partition) {
+ ResultDirectoryRecord[] records = getResultSetMetaData(rsId).getRecords();
+ if (records[partition] == null) {
+ records[partition] = new ResultDirectoryRecord();
+ }
+ return records[partition];
+ }
+
+ public synchronized ResultDirectoryRecord getDirectoryRecord(ResultSetId rsId, int partition)
+ throws HyracksDataException {
+ ResultDirectoryRecord[] records = getResultSetMetaData(rsId).getRecords();
+ if (records[partition] == null) {
+ throw HyracksDataException.create(ErrorCode.RESULT_NO_RECORD, partition, rsId);
+ }
+ return records[partition];
+ }
+
+ public synchronized void updateState(ResultSetId rsId) {
+ int successCount = 0;
+ ResultDirectoryRecord[] records = getResultSetMetaData(rsId).getRecords();
+ for (ResultDirectoryRecord record : records) {
+ if ((record != null) && (record.getStatus() == ResultDirectoryRecord.Status.SUCCESS)) {
+ successCount++;
+ }
+ }
+ if (successCount == records.length) {
+ success();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetId.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetId.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetId.java
new file mode 100644
index 0000000..ffd4ced
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetId.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import java.io.Serializable;
+
+public class ResultSetId implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final long id;
+
+ public ResultSetId(long id) {
+ this.id = id;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == this) {
+ return true;
+ }
+ if (!(o instanceof ResultSetId)) {
+ return false;
+ }
+ return ((ResultSetId) o).id == id;
+ }
+
+ @Override
+ public String toString() {
+ return "RSID:" + id;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetMetaData.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetMetaData.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetMetaData.java
new file mode 100644
index 0000000..b7b8f1c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultSetMetaData.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.result;
+
+import java.util.Arrays;
+
+public class ResultSetMetaData {
+ private final ResultDirectoryRecord[] records;
+ private final boolean ordered;
+
+ ResultSetMetaData(int len, boolean ordered) {
+ this.records = new ResultDirectoryRecord[len];
+ this.ordered = ordered;
+ }
+
+ public boolean getOrderedResult() {
+ return ordered;
+ }
+
+ public ResultDirectoryRecord[] getRecords() {
+ return records;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{ordered: ").append(ordered).append(", records: ").append(Arrays.toString(records));
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDataset.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDataset.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDataset.java
deleted file mode 100644
index 7f549ca..0000000
--- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDataset.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.client.dataset;
-
-import org.apache.hyracks.api.client.IHyracksClientConnection;
-import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
-import org.apache.hyracks.api.dataset.IHyracksDataset;
-import org.apache.hyracks.api.dataset.IHyracksDatasetDirectoryServiceConnection;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.client.net.ClientNetworkManager;
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-
-public class HyracksDataset implements IHyracksDataset {
- private final IHyracksDatasetDirectoryServiceConnection datasetDirectoryServiceConnection;
-
- private final ClientNetworkManager netManager;
-
- private final IHyracksCommonContext datasetClientCtx;
-
- public HyracksDataset(IHyracksClientConnection hcc, int frameSize, int nReaders) throws Exception {
- NetworkAddress ddsAddress = hcc.getDatasetDirectoryServiceInfo();
- datasetDirectoryServiceConnection =
- new HyracksDatasetDirectoryServiceConnection(ddsAddress.getAddress(), ddsAddress.getPort());
-
- netManager = new ClientNetworkManager(nReaders);
- netManager.start();
-
- datasetClientCtx = new DatasetClientContext(frameSize);
- }
-
- @Override
- public IHyracksDatasetReader createReader(JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
- IHyracksDatasetReader reader = null;
- try {
- reader = new HyracksDatasetReader(datasetDirectoryServiceConnection, netManager, datasetClientCtx, jobId,
- resultSetId);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- return reader;
- }
-
- static class DatasetClientContext extends FrameManager implements IHyracksCommonContext {
-
- DatasetClientContext(int frameSize) {
- super(frameSize);
- }
-
- @Override
- public IIOManager getIoManager() {
- return null;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceConnection.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceConnection.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceConnection.java
deleted file mode 100644
index 63139d9..0000000
--- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceConnection.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.client.dataset;
-
-import java.net.InetSocketAddress;
-
-import org.apache.hyracks.api.dataset.DatasetDirectoryRecord;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.dataset.IHyracksDatasetDirectoryServiceConnection;
-import org.apache.hyracks.api.dataset.IHyracksDatasetDirectoryServiceInterface;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.ipc.api.IIPCHandle;
-import org.apache.hyracks.ipc.api.RPCInterface;
-import org.apache.hyracks.ipc.impl.IPCSystem;
-import org.apache.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
-
-//TODO(madhusudancs): Should this implementation be moved to org.apache.hyracks.client?
-public class HyracksDatasetDirectoryServiceConnection implements IHyracksDatasetDirectoryServiceConnection {
- private final IPCSystem ipc;
- private final IHyracksDatasetDirectoryServiceInterface ddsi;
-
- public HyracksDatasetDirectoryServiceConnection(String ddsHost, int ddsPort) throws Exception {
- RPCInterface rpci = new RPCInterface();
- ipc = new IPCSystem(new InetSocketAddress(0), rpci, new JavaSerializationBasedPayloadSerializerDeserializer());
- ipc.start();
- IIPCHandle ddsIpchandle = ipc.getReconnectingHandle(new InetSocketAddress(ddsHost, ddsPort));
- this.ddsi = new HyracksDatasetDirectoryServiceInterfaceRemoteProxy(ddsIpchandle, rpci);
- }
-
- @Override
- public Status getDatasetResultStatus(JobId jobId, ResultSetId rsId) throws Exception {
- return ddsi.getDatasetResultStatus(jobId, rsId);
- }
-
- @Override
- public DatasetDirectoryRecord[] getDatasetResultLocations(JobId jobId, ResultSetId rsId,
- DatasetDirectoryRecord[] knownRecords) throws Exception {
- return ddsi.getDatasetResultLocations(jobId, rsId, knownRecords);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceInterfaceRemoteProxy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceInterfaceRemoteProxy.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceInterfaceRemoteProxy.java
deleted file mode 100644
index 7eeb913..0000000
--- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetDirectoryServiceInterfaceRemoteProxy.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.client.dataset;
-
-import org.apache.hyracks.api.client.HyracksClientInterfaceFunctions;
-import org.apache.hyracks.api.dataset.DatasetDirectoryRecord;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.dataset.IHyracksDatasetDirectoryServiceInterface;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.ipc.api.IIPCHandle;
-import org.apache.hyracks.ipc.api.RPCInterface;
-
-//TODO(madhusudancs): Should this implementation be moved to org.apache.hyracks.client?
-public class HyracksDatasetDirectoryServiceInterfaceRemoteProxy implements IHyracksDatasetDirectoryServiceInterface {
- private final IIPCHandle ipcHandle;
-
- private final RPCInterface rpci;
-
- public HyracksDatasetDirectoryServiceInterfaceRemoteProxy(IIPCHandle ipcHandle, RPCInterface rpci) {
- this.ipcHandle = ipcHandle;
- this.rpci = rpci;
- }
-
- @Override
- public Status getDatasetResultStatus(JobId jobId, ResultSetId rsId) throws Exception {
- HyracksClientInterfaceFunctions.GetDatasetResultStatusFunction gdrlf =
- new HyracksClientInterfaceFunctions.GetDatasetResultStatusFunction(jobId, rsId);
- return (Status) rpci.call(ipcHandle, gdrlf);
- }
-
- @Override
- public DatasetDirectoryRecord[] getDatasetResultLocations(JobId jobId, ResultSetId rsId,
- DatasetDirectoryRecord[] knownRecords) throws Exception {
- HyracksClientInterfaceFunctions.GetDatasetResultLocationsFunction gdrlf =
- new HyracksClientInterfaceFunctions.GetDatasetResultLocationsFunction(jobId, rsId, knownRecords);
- return (DatasetDirectoryRecord[]) rpci.call(ipcHandle, gdrlf);
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetReader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetReader.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetReader.java
deleted file mode 100644
index b1566f4..0000000
--- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/dataset/HyracksDatasetReader.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.client.dataset;
-
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.api.channels.IInputChannel;
-import org.apache.hyracks.api.channels.IInputChannelMonitor;
-import org.apache.hyracks.api.comm.FrameHelper;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
-import org.apache.hyracks.api.dataset.DatasetDirectoryRecord;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.dataset.IHyracksDatasetDirectoryServiceConnection;
-import org.apache.hyracks.api.dataset.IHyracksDatasetReader;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.client.net.ClientNetworkManager;
-import org.apache.hyracks.comm.channels.DatasetNetworkInputChannel;
-import org.apache.hyracks.util.annotations.NotThreadSafe;
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-@NotThreadSafe
-public class HyracksDatasetReader implements IHyracksDatasetReader {
-
- private static final Logger LOGGER = LogManager.getLogger();
- private static final int NUM_READ_BUFFERS = 1;
- private final IHyracksDatasetDirectoryServiceConnection datasetDirectory;
- private final ClientNetworkManager netManager;
- private final IHyracksCommonContext datasetClientCtx;
- private final JobId jobId;
- private final ResultSetId resultSetId;
- private DatasetDirectoryRecord[] knownRecords;
- private DatasetInputChannelMonitor[] monitors;
- private DatasetInputChannelMonitor currentRecordMonitor;
- private DatasetNetworkInputChannel currentRecordChannel;
- private int currentRecord;
-
- public HyracksDatasetReader(IHyracksDatasetDirectoryServiceConnection datasetDirectory,
- ClientNetworkManager netManager, IHyracksCommonContext datasetClientCtx, JobId jobId,
- ResultSetId resultSetId) {
- this.datasetDirectory = datasetDirectory;
- this.netManager = netManager;
- this.datasetClientCtx = datasetClientCtx;
- this.jobId = jobId;
- this.resultSetId = resultSetId;
- currentRecord = -1;
- }
-
- @Override
- public Status getResultStatus() {
- try {
- return datasetDirectory.getDatasetResultStatus(jobId, resultSetId);
- } catch (HyracksDataException e) {
- if (e.getErrorCode() != ErrorCode.NO_RESULT_SET) {
- LOGGER.log(Level.WARN, "Exception retrieving result set for job " + jobId, e);
- }
- } catch (Exception e) {
- LOGGER.log(Level.WARN, "Exception retrieving result set for job " + jobId, e);
- }
- return null;
- }
-
- @Override
- public int read(IFrame frame) throws HyracksDataException {
- frame.reset();
- int readSize = 0;
- if (isFirstRead() && !hasNextRecord()) {
- return readSize;
- }
- // read until frame is full or all dataset records have been read
- while (readSize < frame.getFrameSize()) {
- if (currentRecordMonitor.hasMoreFrames()) {
- final ByteBuffer readBuffer = currentRecordChannel.getNextBuffer();
- if (readBuffer == null) {
- throw new IllegalStateException("Unexpected empty frame");
- }
- currentRecordMonitor.notifyFrameRead();
- if (readSize == 0) {
- final int nBlocks = FrameHelper.deserializeNumOfMinFrame(readBuffer);
- frame.ensureFrameSize(frame.getMinSize() * nBlocks);
- frame.getBuffer().clear();
- }
- frame.getBuffer().put(readBuffer);
- currentRecordChannel.recycleBuffer(readBuffer);
- readSize = frame.getBuffer().position();
- } else {
- currentRecordChannel.close();
- if (currentRecordMonitor.failed()) {
- throw HyracksDataException.create(ErrorCode.FAILED_TO_READ_RESULT, jobId);
- }
- if (isLastRecord() || !hasNextRecord()) {
- break;
- }
- }
- }
- frame.getBuffer().flip();
- return readSize;
- }
-
- private SocketAddress getSocketAddress(DatasetDirectoryRecord record) throws HyracksDataException {
- try {
- final NetworkAddress netAddr = record.getNetworkAddress();
- return new InetSocketAddress(InetAddress.getByAddress(netAddr.lookupIpAddress()), netAddr.getPort());
- } catch (UnknownHostException e) {
- throw HyracksDataException.create(e);
- }
- }
-
- private DatasetInputChannelMonitor getMonitor(int partition) {
- if (knownRecords == null || knownRecords[partition] == null) {
- throw new IllegalStateException("Accessing monitors before obtaining the corresponding addresses");
- }
- if (monitors == null) {
- monitors = new DatasetInputChannelMonitor[knownRecords.length];
- }
- if (monitors[partition] == null) {
- monitors[partition] = new DatasetInputChannelMonitor();
- }
- return monitors[partition];
- }
-
- private boolean hasNextRecord() throws HyracksDataException {
- currentRecord++;
- DatasetDirectoryRecord record = getRecord(currentRecord);
- // skip empty records
- while (record.isEmpty() && ++currentRecord < knownRecords.length) {
- record = getRecord(currentRecord);
- }
- if (currentRecord == knownRecords.length) {
- // exhausted all known records
- return false;
- }
- requestRecordData(record);
- return true;
- }
-
- private DatasetDirectoryRecord getRecord(int recordNum) throws HyracksDataException {
- try {
- while (knownRecords == null || knownRecords[recordNum] == null) {
- knownRecords = datasetDirectory.getDatasetResultLocations(jobId, resultSetId, knownRecords);
- }
- return knownRecords[recordNum];
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- private void requestRecordData(DatasetDirectoryRecord record) throws HyracksDataException {
- currentRecordChannel = new DatasetNetworkInputChannel(netManager, getSocketAddress(record), jobId, resultSetId,
- currentRecord, NUM_READ_BUFFERS);
- currentRecordMonitor = getMonitor(currentRecord);
- currentRecordChannel.registerMonitor(currentRecordMonitor);
- currentRecordChannel.open(datasetClientCtx);
- }
-
- private boolean isFirstRead() {
- return currentRecord == -1;
- }
-
- private boolean isLastRecord() {
- return knownRecords != null && currentRecord == knownRecords.length - 1;
- }
-
- private static class DatasetInputChannelMonitor implements IInputChannelMonitor {
-
- private int availableFrames;
- private boolean eos;
- private boolean failed;
-
- DatasetInputChannelMonitor() {
- eos = false;
- failed = false;
- }
-
- @Override
- public synchronized void notifyFailure(IInputChannel channel, int errorCode) {
- failed = true;
- notifyAll();
- }
-
- @Override
- public synchronized void notifyDataAvailability(IInputChannel channel, int nFrames) {
- availableFrames += nFrames;
- notifyAll();
- }
-
- @Override
- public synchronized void notifyEndOfStream(IInputChannel channel) {
- eos = true;
- notifyAll();
- }
-
- synchronized boolean failed() {
- return failed;
- }
-
- synchronized void notifyFrameRead() {
- availableFrames--;
- notifyAll();
- }
-
- synchronized boolean hasMoreFrames() throws HyracksDataException {
- while (!failed && !eos && availableFrames == 0) {
- try {
- wait();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw HyracksDataException.create(e);
- }
- }
- return !failed && !isFullyConsumed();
- }
-
- private synchronized boolean isFullyConsumed() {
- return availableFrames == 0 && eos;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectory.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectory.java
new file mode 100644
index 0000000..6f8c4d0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.client.result;
+
+import java.net.InetSocketAddress;
+
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.result.IResultDirectory;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.ipc.api.IIPCHandle;
+import org.apache.hyracks.ipc.api.RPCInterface;
+import org.apache.hyracks.ipc.impl.IPCSystem;
+import org.apache.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
+
+//TODO(madhusudancs): Should this implementation be moved to org.apache.hyracks.client?
+public class ResultDirectory implements IResultDirectory {
+ private final IPCSystem ipc;
+ private final IResultDirectory remoteResultDirectory;
+
+ public ResultDirectory(String resultHost, int resultPort) throws Exception {
+ RPCInterface rpci = new RPCInterface();
+ ipc = new IPCSystem(new InetSocketAddress(0), rpci, new JavaSerializationBasedPayloadSerializerDeserializer());
+ ipc.start();
+ IIPCHandle ddsIpchandle = ipc.getReconnectingHandle(new InetSocketAddress(resultHost, resultPort));
+ this.remoteResultDirectory = new ResultDirectoryRemoteProxy(ddsIpchandle, rpci);
+ }
+
+ @Override
+ public Status getResultStatus(JobId jobId, ResultSetId rsId) throws Exception {
+ return remoteResultDirectory.getResultStatus(jobId, rsId);
+ }
+
+ @Override
+ public ResultDirectoryRecord[] getResultLocations(JobId jobId, ResultSetId rsId,
+ ResultDirectoryRecord[] knownRecords) throws Exception {
+ return remoteResultDirectory.getResultLocations(jobId, rsId, knownRecords);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectoryRemoteProxy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectoryRemoteProxy.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectoryRemoteProxy.java
new file mode 100644
index 0000000..77c6e4b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultDirectoryRemoteProxy.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.client.result;
+
+import org.apache.hyracks.api.client.HyracksClientInterfaceFunctions;
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.result.IResultDirectory;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.ipc.api.IIPCHandle;
+import org.apache.hyracks.ipc.api.RPCInterface;
+
+//TODO(madhusudancs): Should this implementation be moved to org.apache.hyracks.client?
+public class ResultDirectoryRemoteProxy implements IResultDirectory {
+ private final IIPCHandle ipcHandle;
+
+ private final RPCInterface rpci;
+
+ public ResultDirectoryRemoteProxy(IIPCHandle ipcHandle, RPCInterface rpci) {
+ this.ipcHandle = ipcHandle;
+ this.rpci = rpci;
+ }
+
+ @Override
+ public Status getResultStatus(JobId jobId, ResultSetId rsId) throws Exception {
+ HyracksClientInterfaceFunctions.GetResultStatusFunction gdrlf =
+ new HyracksClientInterfaceFunctions.GetResultStatusFunction(jobId, rsId);
+ return (Status) rpci.call(ipcHandle, gdrlf);
+ }
+
+ @Override
+ public ResultDirectoryRecord[] getResultLocations(JobId jobId, ResultSetId rsId,
+ ResultDirectoryRecord[] knownRecords) throws Exception {
+ HyracksClientInterfaceFunctions.GetResultLocationsFunction gdrlf =
+ new HyracksClientInterfaceFunctions.GetResultLocationsFunction(jobId, rsId, knownRecords);
+ return (ResultDirectoryRecord[]) rpci.call(ipcHandle, gdrlf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java
new file mode 100644
index 0000000..ef93cce
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.client.result;
+
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.comm.NetworkAddress;
+import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.api.result.IResultDirectory;
+import org.apache.hyracks.api.result.IResultSetReader;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.client.net.ClientNetworkManager;
+import org.apache.hyracks.control.nc.resources.memory.FrameManager;
+
+public class ResultSet implements IResultSet {
+ private final IResultDirectory resultDirectory;
+
+ private final ClientNetworkManager netManager;
+
+ private final IHyracksCommonContext resultClientCtx;
+
+ public ResultSet(IHyracksClientConnection hcc, int frameSize, int nReaders) throws Exception {
+ NetworkAddress ddsAddress = hcc.getResultDirectoryAddress();
+ resultDirectory = new ResultDirectory(ddsAddress.getAddress(), ddsAddress.getPort());
+
+ netManager = new ClientNetworkManager(nReaders);
+ netManager.start();
+
+ resultClientCtx = new ResultClientContext(frameSize);
+ }
+
+ @Override
+ public IResultSetReader createReader(JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
+ IResultSetReader reader = null;
+ try {
+ reader = new ResultSetReader(resultDirectory, netManager, resultClientCtx, jobId, resultSetId);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ return reader;
+ }
+
+ static class ResultClientContext extends FrameManager implements IHyracksCommonContext {
+
+ ResultClientContext(int frameSize) {
+ super(frameSize);
+ }
+
+ @Override
+ public IIOManager getIoManager() {
+ return null;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java
new file mode 100644
index 0000000..092d860
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.client.result;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.channels.IInputChannel;
+import org.apache.hyracks.api.channels.IInputChannelMonitor;
+import org.apache.hyracks.api.comm.FrameHelper;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.NetworkAddress;
+import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.result.IResultDirectory;
+import org.apache.hyracks.api.result.IResultSetReader;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.client.net.ClientNetworkManager;
+import org.apache.hyracks.comm.channels.ResultNetworkInputChannel;
+import org.apache.hyracks.util.annotations.NotThreadSafe;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+@NotThreadSafe
+public class ResultSetReader implements IResultSetReader {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+ private static final int NUM_READ_BUFFERS = 1;
+ private final IResultDirectory resultDirectory;
+ private final ClientNetworkManager netManager;
+ private final IHyracksCommonContext resultClientCtx;
+ private final JobId jobId;
+ private final ResultSetId resultSetId;
+ private ResultDirectoryRecord[] knownRecords;
+ private ResultInputChannelMonitor[] monitors;
+ private ResultInputChannelMonitor currentRecordMonitor;
+ private ResultNetworkInputChannel currentRecordChannel;
+ private int currentRecord;
+
+ public ResultSetReader(IResultDirectory resultDirectory, ClientNetworkManager netManager,
+ IHyracksCommonContext resultClientCtx, JobId jobId, ResultSetId resultSetId) {
+ this.resultDirectory = resultDirectory;
+ this.netManager = netManager;
+ this.resultClientCtx = resultClientCtx;
+ this.jobId = jobId;
+ this.resultSetId = resultSetId;
+ currentRecord = -1;
+ }
+
+ @Override
+ public Status getResultStatus() {
+ try {
+ return resultDirectory.getResultStatus(jobId, resultSetId);
+ } catch (HyracksDataException e) {
+ if (e.getErrorCode() != ErrorCode.NO_RESULT_SET) {
+ LOGGER.log(Level.WARN, "Exception retrieving result set for job " + jobId, e);
+ }
+ } catch (Exception e) {
+ LOGGER.log(Level.WARN, "Exception retrieving result set for job " + jobId, e);
+ }
+ return null;
+ }
+
+ @Override
+ public int read(IFrame frame) throws HyracksDataException {
+ frame.reset();
+ int readSize = 0;
+ if (isFirstRead() && !hasNextRecord()) {
+ return readSize;
+ }
+ // read until frame is full or all result records have been read
+ while (readSize < frame.getFrameSize()) {
+ if (currentRecordMonitor.hasMoreFrames()) {
+ final ByteBuffer readBuffer = currentRecordChannel.getNextBuffer();
+ if (readBuffer == null) {
+ throw new IllegalStateException("Unexpected empty frame");
+ }
+ currentRecordMonitor.notifyFrameRead();
+ if (readSize == 0) {
+ final int nBlocks = FrameHelper.deserializeNumOfMinFrame(readBuffer);
+ frame.ensureFrameSize(frame.getMinSize() * nBlocks);
+ frame.getBuffer().clear();
+ }
+ frame.getBuffer().put(readBuffer);
+ currentRecordChannel.recycleBuffer(readBuffer);
+ readSize = frame.getBuffer().position();
+ } else {
+ currentRecordChannel.close();
+ if (currentRecordMonitor.failed()) {
+ throw HyracksDataException.create(ErrorCode.FAILED_TO_READ_RESULT, jobId);
+ }
+ if (isLastRecord() || !hasNextRecord()) {
+ break;
+ }
+ }
+ }
+ frame.getBuffer().flip();
+ return readSize;
+ }
+
+ private SocketAddress getSocketAddress(ResultDirectoryRecord record) throws HyracksDataException {
+ try {
+ final NetworkAddress netAddr = record.getNetworkAddress();
+ return new InetSocketAddress(InetAddress.getByAddress(netAddr.lookupIpAddress()), netAddr.getPort());
+ } catch (UnknownHostException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private ResultInputChannelMonitor getMonitor(int partition) {
+ if (knownRecords == null || knownRecords[partition] == null) {
+ throw new IllegalStateException("Accessing monitors before obtaining the corresponding addresses");
+ }
+ if (monitors == null) {
+ monitors = new ResultInputChannelMonitor[knownRecords.length];
+ }
+ if (monitors[partition] == null) {
+ monitors[partition] = new ResultInputChannelMonitor();
+ }
+ return monitors[partition];
+ }
+
+ private boolean hasNextRecord() throws HyracksDataException {
+ currentRecord++;
+ ResultDirectoryRecord record = getRecord(currentRecord);
+ // skip empty records
+ while (record.isEmpty() && ++currentRecord < knownRecords.length) {
+ record = getRecord(currentRecord);
+ }
+ if (currentRecord == knownRecords.length) {
+ // exhausted all known records
+ return false;
+ }
+ requestRecordData(record);
+ return true;
+ }
+
+ private ResultDirectoryRecord getRecord(int recordNum) throws HyracksDataException {
+ try {
+ while (knownRecords == null || knownRecords[recordNum] == null) {
+ knownRecords = resultDirectory.getResultLocations(jobId, resultSetId, knownRecords);
+ }
+ return knownRecords[recordNum];
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+
+ private void requestRecordData(ResultDirectoryRecord record) throws HyracksDataException {
+ currentRecordChannel = new ResultNetworkInputChannel(netManager, getSocketAddress(record), jobId, resultSetId,
+ currentRecord, NUM_READ_BUFFERS);
+ currentRecordMonitor = getMonitor(currentRecord);
+ currentRecordChannel.registerMonitor(currentRecordMonitor);
+ currentRecordChannel.open(resultClientCtx);
+ }
+
+ private boolean isFirstRead() {
+ return currentRecord == -1;
+ }
+
+ private boolean isLastRecord() {
+ return knownRecords != null && currentRecord == knownRecords.length - 1;
+ }
+
+ private static class ResultInputChannelMonitor implements IInputChannelMonitor {
+
+ private int availableFrames;
+ private boolean eos;
+ private boolean failed;
+
+ ResultInputChannelMonitor() {
+ eos = false;
+ failed = false;
+ }
+
+ @Override
+ public synchronized void notifyFailure(IInputChannel channel, int errorCode) {
+ failed = true;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void notifyDataAvailability(IInputChannel channel, int nFrames) {
+ availableFrames += nFrames;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void notifyEndOfStream(IInputChannel channel) {
+ eos = true;
+ notifyAll();
+ }
+
+ synchronized boolean failed() {
+ return failed;
+ }
+
+ synchronized void notifyFrameRead() {
+ availableFrames--;
+ notifyAll();
+ }
+
+ synchronized boolean hasMoreFrames() throws HyracksDataException {
+ while (!failed && !eos && availableFrames == 0) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw HyracksDataException.create(e);
+ }
+ }
+ return !failed && !isFullyConsumed();
+ }
+
+ private synchronized boolean isFullyConsumed() {
+ return availableFrames == 0 && eos;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/DatasetNetworkInputChannel.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/DatasetNetworkInputChannel.java b/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/DatasetNetworkInputChannel.java
deleted file mode 100644
index 0f96a6e..0000000
--- a/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/DatasetNetworkInputChannel.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.comm.channels;
-
-import java.net.SocketAddress;
-import java.nio.ByteBuffer;
-import java.util.ArrayDeque;
-import java.util.Queue;
-
-import org.apache.hyracks.api.channels.IInputChannel;
-import org.apache.hyracks.api.channels.IInputChannelMonitor;
-import org.apache.hyracks.api.comm.IBufferAcceptor;
-import org.apache.hyracks.api.comm.IChannelControlBlock;
-import org.apache.hyracks.api.comm.ICloseableBufferAcceptor;
-import org.apache.hyracks.api.context.IHyracksCommonContext;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class DatasetNetworkInputChannel implements IInputChannel {
- private static final Logger LOGGER = LogManager.getLogger();
-
- static final int INITIAL_MESSAGE_SIZE = 20;
-
- private final IChannelConnectionFactory netManager;
-
- private final SocketAddress remoteAddress;
-
- private final JobId jobId;
-
- private final ResultSetId resultSetId;
-
- private final int partition;
-
- private final Queue<ByteBuffer> fullQueue;
-
- private final int nBuffers;
-
- private IChannelControlBlock ccb;
-
- private IInputChannelMonitor monitor;
-
- private Object attachment;
-
- public DatasetNetworkInputChannel(IChannelConnectionFactory netManager, SocketAddress remoteAddress, JobId jobId,
- ResultSetId resultSetId, int partition, int nBuffers) {
- this.netManager = netManager;
- this.remoteAddress = remoteAddress;
- this.jobId = jobId;
- this.resultSetId = resultSetId;
- this.partition = partition;
- fullQueue = new ArrayDeque<ByteBuffer>(nBuffers);
- this.nBuffers = nBuffers;
- }
-
- @Override
- public void registerMonitor(IInputChannelMonitor monitor) {
- this.monitor = monitor;
- }
-
- @Override
- public void setAttachment(Object attachment) {
- this.attachment = attachment;
- }
-
- @Override
- public Object getAttachment() {
- return attachment;
- }
-
- @Override
- public synchronized ByteBuffer getNextBuffer() {
- return fullQueue.poll();
- }
-
- @Override
- public void recycleBuffer(ByteBuffer buffer) {
- buffer.clear();
- ccb.getReadInterface().getEmptyBufferAcceptor().accept(buffer);
- }
-
- @Override
- public void open(IHyracksCommonContext ctx) throws HyracksDataException {
- try {
- ccb = netManager.connect(remoteAddress);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- ccb.getReadInterface().setFullBufferAcceptor(new ReadFullBufferAcceptor());
- ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
- for (int i = 0; i < nBuffers; ++i) {
- ccb.getReadInterface().getEmptyBufferAcceptor().accept(ctx.allocateFrame());
- }
- ByteBuffer writeBuffer = ByteBuffer.allocate(INITIAL_MESSAGE_SIZE);
- writeBuffer.putLong(jobId.getId());
- writeBuffer.putLong(resultSetId.getId());
- writeBuffer.putInt(partition);
- writeBuffer.flip();
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Sending partition request for JobId: " + jobId + " partition: " + partition + " on channel: "
- + ccb);
- }
- ccb.getWriteInterface().getFullBufferAcceptor().accept(writeBuffer);
- ccb.getWriteInterface().getFullBufferAcceptor().close();
- }
-
- @Override
- public void close() throws HyracksDataException {
-
- }
-
- private class ReadFullBufferAcceptor implements ICloseableBufferAcceptor {
- @Override
- public void accept(ByteBuffer buffer) {
- fullQueue.add(buffer);
- monitor.notifyDataAvailability(DatasetNetworkInputChannel.this, 1);
- }
-
- @Override
- public void close() {
- monitor.notifyEndOfStream(DatasetNetworkInputChannel.this);
- }
-
- @Override
- public void error(int ecode) {
- monitor.notifyFailure(DatasetNetworkInputChannel.this, ecode);
- }
- }
-
- private class WriteEmptyBufferAcceptor implements IBufferAcceptor {
- @Override
- public void accept(ByteBuffer buffer) {
- // do nothing
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/ResultNetworkInputChannel.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/ResultNetworkInputChannel.java b/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/ResultNetworkInputChannel.java
new file mode 100644
index 0000000..37540cd
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-comm/src/main/java/org/apache/hyracks/comm/channels/ResultNetworkInputChannel.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.comm.channels;
+
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Queue;
+
+import org.apache.hyracks.api.channels.IInputChannel;
+import org.apache.hyracks.api.channels.IInputChannelMonitor;
+import org.apache.hyracks.api.comm.IBufferAcceptor;
+import org.apache.hyracks.api.comm.IChannelControlBlock;
+import org.apache.hyracks.api.comm.ICloseableBufferAcceptor;
+import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ResultNetworkInputChannel implements IInputChannel {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ static final int INITIAL_MESSAGE_SIZE = 20;
+
+ private final IChannelConnectionFactory netManager;
+
+ private final SocketAddress remoteAddress;
+
+ private final JobId jobId;
+
+ private final ResultSetId resultSetId;
+
+ private final int partition;
+
+ private final Queue<ByteBuffer> fullQueue;
+
+ private final int nBuffers;
+
+ private IChannelControlBlock ccb;
+
+ private IInputChannelMonitor monitor;
+
+ private Object attachment;
+
+ public ResultNetworkInputChannel(IChannelConnectionFactory netManager, SocketAddress remoteAddress, JobId jobId,
+ ResultSetId resultSetId, int partition, int nBuffers) {
+ this.netManager = netManager;
+ this.remoteAddress = remoteAddress;
+ this.jobId = jobId;
+ this.resultSetId = resultSetId;
+ this.partition = partition;
+ fullQueue = new ArrayDeque<ByteBuffer>(nBuffers);
+ this.nBuffers = nBuffers;
+ }
+
+ @Override
+ public void registerMonitor(IInputChannelMonitor monitor) {
+ this.monitor = monitor;
+ }
+
+ @Override
+ public void setAttachment(Object attachment) {
+ this.attachment = attachment;
+ }
+
+ @Override
+ public Object getAttachment() {
+ return attachment;
+ }
+
+ @Override
+ public synchronized ByteBuffer getNextBuffer() {
+ return fullQueue.poll();
+ }
+
+ @Override
+ public void recycleBuffer(ByteBuffer buffer) {
+ buffer.clear();
+ ccb.getReadInterface().getEmptyBufferAcceptor().accept(buffer);
+ }
+
+ @Override
+ public void open(IHyracksCommonContext ctx) throws HyracksDataException {
+ try {
+ ccb = netManager.connect(remoteAddress);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ ccb.getReadInterface().setFullBufferAcceptor(new ReadFullBufferAcceptor());
+ ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
+ for (int i = 0; i < nBuffers; ++i) {
+ ccb.getReadInterface().getEmptyBufferAcceptor().accept(ctx.allocateFrame());
+ }
+ ByteBuffer writeBuffer = ByteBuffer.allocate(INITIAL_MESSAGE_SIZE);
+ writeBuffer.putLong(jobId.getId());
+ writeBuffer.putLong(resultSetId.getId());
+ writeBuffer.putInt(partition);
+ writeBuffer.flip();
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Sending partition request for JobId: " + jobId + " partition: " + partition + " on channel: "
+ + ccb);
+ }
+ ccb.getWriteInterface().getFullBufferAcceptor().accept(writeBuffer);
+ ccb.getWriteInterface().getFullBufferAcceptor().close();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+
+ }
+
+ private class ReadFullBufferAcceptor implements ICloseableBufferAcceptor {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ fullQueue.add(buffer);
+ monitor.notifyDataAvailability(ResultNetworkInputChannel.this, 1);
+ }
+
+ @Override
+ public void close() {
+ monitor.notifyEndOfStream(ResultNetworkInputChannel.this);
+ }
+
+ @Override
+ public void error(int ecode) {
+ monitor.notifyFailure(ResultNetworkInputChannel.this, ecode);
+ }
+ }
+
+ private class WriteEmptyBufferAcceptor implements IBufferAcceptor {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ // do nothing
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClientInterfaceIPCI.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClientInterfaceIPCI.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClientInterfaceIPCI.java
index a669402..2edbab8 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClientInterfaceIPCI.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClientInterfaceIPCI.java
@@ -20,21 +20,21 @@ package org.apache.hyracks.control.cc;
import org.apache.hyracks.api.client.HyracksClientInterfaceFunctions;
import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
import org.apache.hyracks.api.job.DeployedJobSpecId;
import org.apache.hyracks.api.job.DeployedJobSpecIdFactory;
import org.apache.hyracks.api.job.JobIdFactory;
import org.apache.hyracks.api.job.JobInfo;
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
import org.apache.hyracks.control.cc.work.CancelJobWork;
import org.apache.hyracks.control.cc.work.CliDeployBinaryWork;
import org.apache.hyracks.control.cc.work.CliUnDeployBinaryWork;
import org.apache.hyracks.control.cc.work.ClusterShutdownWork;
import org.apache.hyracks.control.cc.work.DeployJobSpecWork;
-import org.apache.hyracks.control.cc.work.GetDatasetDirectoryServiceInfoWork;
import org.apache.hyracks.control.cc.work.GetJobInfoWork;
import org.apache.hyracks.control.cc.work.GetJobStatusWork;
import org.apache.hyracks.control.cc.work.GetNodeControllersInfoWork;
import org.apache.hyracks.control.cc.work.GetNodeDetailsJSONWork;
+import org.apache.hyracks.control.cc.work.GetResultDirectoryAddressWork;
import org.apache.hyracks.control.cc.work.GetResultPartitionLocationsWork;
import org.apache.hyracks.control.cc.work.GetResultStatusWork;
import org.apache.hyracks.control.cc.work.GetThreadDumpWork;
@@ -121,19 +121,19 @@ class ClientInterfaceIPCI implements IIPCI {
ccs.getWorkQueue().schedule(new JobStartWork(ccs, sjf.getDeploymentId(), acggfBytes, sjf.getJobFlags(),
jobIdFactory, sjf.getJobParameters(), new IPCResponder<>(handle, mid), id));
break;
- case GET_DATASET_DIRECTORY_SERIVICE_INFO:
+ case GET_RESULT_DIRECTORY_ADDRESS:
ccs.getWorkQueue().schedule(
- new GetDatasetDirectoryServiceInfoWork(ccs, new IPCResponder<NetworkAddress>(handle, mid)));
+ new GetResultDirectoryAddressWork(ccs, new IPCResponder<NetworkAddress>(handle, mid)));
break;
- case GET_DATASET_RESULT_STATUS:
- HyracksClientInterfaceFunctions.GetDatasetResultStatusFunction gdrsf =
- (HyracksClientInterfaceFunctions.GetDatasetResultStatusFunction) fn;
+ case GET_RESULT_STATUS:
+ HyracksClientInterfaceFunctions.GetResultStatusFunction gdrsf =
+ (HyracksClientInterfaceFunctions.GetResultStatusFunction) fn;
ccs.getWorkQueue().schedule(new GetResultStatusWork(ccs, gdrsf.getJobId(), gdrsf.getResultSetId(),
new IPCResponder<Status>(handle, mid)));
break;
- case GET_DATASET_RESULT_LOCATIONS:
- HyracksClientInterfaceFunctions.GetDatasetResultLocationsFunction gdrlf =
- (HyracksClientInterfaceFunctions.GetDatasetResultLocationsFunction) fn;
+ case GET_RESULT_LOCATIONS:
+ HyracksClientInterfaceFunctions.GetResultLocationsFunction gdrlf =
+ (HyracksClientInterfaceFunctions.GetResultLocationsFunction) fn;
ccs.getWorkQueue().schedule(new GetResultPartitionLocationsWork(ccs, gdrlf.getJobId(),
gdrlf.getResultSetId(), gdrlf.getKnownRecords(), new IPCResponder<>(handle, mid)));
break;
[3/5] asterixdb git commit: [ASTERIXDB-2314][HYR] Dataset in class
names in Hyracks
Posted by ti...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
index 153a32d..8ed015c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
@@ -55,10 +55,10 @@ import org.apache.hyracks.api.topology.TopologyDefinitionParser;
import org.apache.hyracks.control.cc.application.CCServiceContext;
import org.apache.hyracks.control.cc.cluster.INodeManager;
import org.apache.hyracks.control.cc.cluster.NodeManager;
-import org.apache.hyracks.control.cc.dataset.DatasetDirectoryService;
-import org.apache.hyracks.control.cc.dataset.IDatasetDirectoryService;
import org.apache.hyracks.control.cc.job.IJobManager;
import org.apache.hyracks.control.cc.job.JobManager;
+import org.apache.hyracks.control.cc.result.IResultDirectoryService;
+import org.apache.hyracks.control.cc.result.ResultDirectoryService;
import org.apache.hyracks.control.cc.scheduler.IResourceManager;
import org.apache.hyracks.control.cc.scheduler.ResourceManager;
import org.apache.hyracks.control.cc.web.WebServer;
@@ -122,7 +122,7 @@ public class ClusterControllerService implements IControllerService {
private final DeadNodeSweeper sweeper;
- private final IDatasetDirectoryService datasetDirectoryService;
+ private final IResultDirectoryService resultDirectoryService;
private final Map<DeploymentId, DeploymentRun> deploymentRunMap;
@@ -171,8 +171,8 @@ public class ClusterControllerService implements IControllerService {
final ClusterTopology topology = computeClusterTopology(ccConfig);
ccContext = new ClusterControllerContext(topology);
sweeper = new DeadNodeSweeper();
- datasetDirectoryService =
- new DatasetDirectoryService(ccConfig.getResultTTL(), ccConfig.getResultSweepThreshold());
+ resultDirectoryService =
+ new ResultDirectoryService(ccConfig.getResultTTL(), ccConfig.getResultSweepThreshold());
deploymentRunMap = new HashMap<>();
stateDumpRunMap = new HashMap<>();
@@ -221,7 +221,7 @@ public class ClusterControllerService implements IControllerService {
jobLog.open();
startApplication();
- datasetDirectoryService.init(executor);
+ resultDirectoryService.init(executor);
workQueue.start();
connectNCs();
LOGGER.log(Level.INFO, "Started ClusterControllerService");
@@ -230,7 +230,7 @@ public class ClusterControllerService implements IControllerService {
private void startApplication() throws Exception {
serviceCtx = new CCServiceContext(this, serverCtx, ccContext, ccConfig.getAppConfig());
- serviceCtx.addJobLifecycleListener(datasetDirectoryService);
+ serviceCtx.addJobLifecycleListener(resultDirectoryService);
application.init(serviceCtx);
executor = MaintainedThreadNameExecutorService.newCachedThreadPool(serviceCtx.getThreadFactory());
application.start(ccConfig.getAppArgsArray());
@@ -408,7 +408,7 @@ public class ClusterControllerService implements IControllerService {
return clusterIPC;
}
- public NetworkAddress getDatasetDirectoryServiceInfo() {
+ public NetworkAddress getResultDirectoryAddress() {
return new NetworkAddress(ccConfig.getClientPublicAddress(), ccConfig.getClientPublicPort());
}
@@ -461,8 +461,8 @@ public class ClusterControllerService implements IControllerService {
}
}
- public IDatasetDirectoryService getDatasetDirectoryService() {
- return datasetDirectoryService;
+ public IResultDirectoryService getResultDirectoryService() {
+ return resultDirectoryService;
}
public synchronized void addStateDumpRun(String id, StateDumpRun sdr) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java
index 2b03324..31f989b 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java
@@ -144,7 +144,7 @@ public class NodeManager implements INodeManager {
Map<String, NodeControllerInfo> result = new LinkedHashMap<>();
nodeRegistry.forEach(
(key, ncState) -> result.put(key, new NodeControllerInfo(key, NodeStatus.ACTIVE, ncState.getDataPort(),
- ncState.getDatasetPort(), ncState.getMessagingPort(), ncState.getCapacity().getCores())));
+ ncState.getResultPort(), ncState.getMessagingPort(), ncState.getCapacity().getCores())));
return result;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java
deleted file mode 100644
index 04aaddd..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java
+++ /dev/null
@@ -1,316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.cc.dataset;
-
-import java.io.PrintWriter;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.api.dataset.DatasetDirectoryRecord;
-import org.apache.hyracks.api.dataset.DatasetJobRecord;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.State;
-import org.apache.hyracks.api.dataset.IDatasetStateRecord;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.dataset.ResultSetMetaData;
-import org.apache.hyracks.api.exceptions.ErrorCode;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.api.job.JobStatus;
-import org.apache.hyracks.control.common.dataset.AbstractDatasetManager;
-import org.apache.hyracks.control.common.dataset.ResultStateSweeper;
-import org.apache.hyracks.control.common.work.IResultCallback;
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-/**
- * TODO(madhusudancs): The potential perils of this global dataset directory service implementation is that, the jobs
- * location information is never evicted from the memory and the memory usage grows as the number of jobs in the system
- * grows. What we should possibly do is, add an API call for the client to say that it received everything it has to for
- * the job (after it receives all the results) completely. Then we can just get rid of the location information for that
- * job.
- */
-public class DatasetDirectoryService extends AbstractDatasetManager implements IDatasetDirectoryService {
-
- private static final Logger LOGGER = LogManager.getLogger();
-
- private final long resultSweepThreshold;
-
- private final Map<JobId, JobResultInfo> jobResultLocations;
-
- public DatasetDirectoryService(long resultTTL, long resultSweepThreshold) {
- super(resultTTL);
- this.resultSweepThreshold = resultSweepThreshold;
- jobResultLocations = new LinkedHashMap<>();
- }
-
- @Override
- public void init(ExecutorService executor) {
- executor.execute(new ResultStateSweeper(this, resultSweepThreshold, LOGGER));
- }
-
- @Override
- public synchronized void notifyJobCreation(JobId jobId, JobSpecification spec) throws HyracksException {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info(getClass().getSimpleName() + " notified of new job " + jobId);
- }
- if (jobResultLocations.get(jobId) != null) {
- throw HyracksDataException.create(ErrorCode.MORE_THAN_ONE_RESULT, jobId);
- }
- jobResultLocations.put(jobId, new JobResultInfo(new DatasetJobRecord(), null));
- }
-
- @Override
- public synchronized void notifyJobStart(JobId jobId) throws HyracksException {
- jobResultLocations.get(jobId).getRecord().start();
- }
-
- @Override
- public void notifyJobFinish(JobId jobId, JobStatus jobStatus, List<Exception> exceptions) throws HyracksException {
- // Auto-generated method stub
- }
-
- private DatasetJobRecord getDatasetJobRecord(JobId jobId) {
- final JobResultInfo jri = jobResultLocations.get(jobId);
- return jri == null ? null : jri.getRecord();
- }
-
- private DatasetJobRecord getNonNullDatasetJobRecord(JobId jobId) throws HyracksDataException {
- final DatasetJobRecord djr = getDatasetJobRecord(jobId);
- if (djr == null) {
- throw HyracksDataException.create(ErrorCode.NO_RESULT_SET, jobId);
- }
- return djr;
- }
-
- @Override
- public synchronized void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, boolean orderedResult,
- boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress)
- throws HyracksDataException {
- DatasetJobRecord djr = getNonNullDatasetJobRecord(jobId);
- djr.setResultSetMetaData(rsId, orderedResult, nPartitions);
- DatasetDirectoryRecord record = djr.getOrCreateDirectoryRecord(rsId, partition);
-
- record.setNetworkAddress(networkAddress);
- record.setEmpty(emptyResult);
- record.start();
-
- final JobResultInfo jobResultInfo = jobResultLocations.get(jobId);
- Waiter waiter = jobResultInfo.getWaiter(rsId);
- if (waiter != null) {
- try {
- DatasetDirectoryRecord[] updatedRecords = updatedRecords(jobId, rsId, waiter.knownRecords);
- if (updatedRecords != null) {
- jobResultInfo.removeWaiter(rsId);
- waiter.callback.setValue(updatedRecords);
- }
- } catch (Exception e) {
- waiter.callback.setException(e);
- }
- }
- notifyAll();
- }
-
- @Override
- public synchronized void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition)
- throws HyracksDataException {
- DatasetJobRecord djr = getNonNullDatasetJobRecord(jobId);
- djr.getDirectoryRecord(rsId, partition).writeEOS();
- djr.updateState(rsId);
- notifyAll();
- }
-
- @Override
- public synchronized void reportJobFailure(JobId jobId, List<Exception> exceptions) {
- LOGGER.log(Level.INFO, "job " + jobId + " failed and is being reported to " + getClass().getSimpleName(),
- exceptions.get(0));
- DatasetJobRecord djr = getDatasetJobRecord(jobId);
- LOGGER.log(Level.INFO, "Dataset job record is " + djr);
- if (djr != null) {
- LOGGER.log(Level.INFO, "Setting exceptions in Dataset job record");
- djr.fail(exceptions);
- }
- final JobResultInfo jobResultInfo = jobResultLocations.get(jobId);
- LOGGER.log(Level.INFO, "Job result info is " + jobResultInfo);
- if (jobResultInfo != null) {
- LOGGER.log(Level.INFO, "Setting exceptions in Job result info");
- jobResultInfo.setException(exceptions.isEmpty() ? null : exceptions.get(0));
- }
- notifyAll();
- }
-
- @Override
- public synchronized DatasetJobRecord.Status getResultStatus(JobId jobId, ResultSetId rsId)
- throws HyracksDataException {
- return getNonNullDatasetJobRecord(jobId).getStatus();
- }
-
- @Override
- public synchronized Set<JobId> getJobIds() {
- return jobResultLocations.keySet();
- }
-
- @Override
- public IDatasetStateRecord getState(JobId jobId) {
- return getDatasetJobRecord(jobId);
- }
-
- @Override
- public synchronized void sweep(JobId jobId) {
- jobResultLocations.remove(jobId);
- }
-
- @Override
- public synchronized void getResultPartitionLocations(JobId jobId, ResultSetId rsId,
- DatasetDirectoryRecord[] knownRecords, IResultCallback<DatasetDirectoryRecord[]> callback)
- throws HyracksDataException {
- DatasetDirectoryRecord[] updatedRecords = updatedRecords(jobId, rsId, knownRecords);
- if (updatedRecords == null) {
- jobResultLocations.get(jobId).addWaiter(rsId, knownRecords, callback);
- } else {
- callback.setValue(updatedRecords);
- }
- }
-
- /**
- * Compares the records already known by the client for the given job's result set id with the records that the
- * dataset directory service knows and if there are any newly discovered records returns a whole array with the
- * new records filled in.
- *
- * @param jobId
- * - Id of the job for which the directory records should be retrieved.
- * @param rsId
- * - Id of the result set for which the directory records should be retrieved.
- * @param knownRecords
- * - An array of directory records that the client is already aware of.
- * @return
- * Returns the updated records if new record were discovered, null otherwise
- * @throws HyracksDataException
- * TODO(madhusudancs): Think about caching (and still be stateless) instead of this ugly O(n) iterations for
- * every check. This already looks very expensive.
- */
- private DatasetDirectoryRecord[] updatedRecords(JobId jobId, ResultSetId rsId,
- DatasetDirectoryRecord[] knownRecords) throws HyracksDataException {
- DatasetJobRecord djr = getNonNullDatasetJobRecord(jobId);
-
- if (djr.getStatus().getState() == State.FAILED) {
- List<Exception> caughtExceptions = djr.getStatus().getExceptions();
- if (caughtExceptions != null && !caughtExceptions.isEmpty()) {
- final Exception cause = caughtExceptions.get(caughtExceptions.size() - 1);
- if (cause instanceof HyracksDataException) {
- throw (HyracksDataException) cause;
- }
- throw HyracksDataException.create(ErrorCode.RESULT_FAILURE_EXCEPTION, cause, rsId, jobId);
- } else {
- throw HyracksDataException.create(ErrorCode.RESULT_FAILURE_NO_EXCEPTION, rsId, jobId);
- }
- }
-
- final ResultSetMetaData resultSetMetaData = djr.getResultSetMetaData(rsId);
- if (resultSetMetaData == null) {
- return null;
- }
- DatasetDirectoryRecord[] records = resultSetMetaData.getRecords();
-
- return Arrays.equals(records, knownRecords) ? null : records;
- }
-
- public PrintWriter print(PrintWriter pw) {
- for (JobId jId : getJobIds()) {
- pw.print(jId.toString());
- pw.print(" - ");
- pw.println(String.valueOf(getDatasetJobRecord(jId)));
- }
- pw.flush();
- return pw;
- }
-}
-
-class JobResultInfo {
-
- private DatasetJobRecord record;
- private Waiters waiters;
- private Exception exception;
-
- JobResultInfo(DatasetJobRecord record, Waiters waiters) {
- this.record = record;
- this.waiters = waiters;
- }
-
- DatasetJobRecord getRecord() {
- return record;
- }
-
- void addWaiter(ResultSetId rsId, DatasetDirectoryRecord[] knownRecords,
- IResultCallback<DatasetDirectoryRecord[]> callback) {
- if (waiters == null) {
- waiters = new Waiters();
- }
- waiters.put(rsId, new Waiter(knownRecords, callback));
- if (exception != null) {
- // Exception was set before the waiter is added.
- setException(exception);
- }
- }
-
- Waiter removeWaiter(ResultSetId rsId) {
- return waiters.remove(rsId);
- }
-
- Waiter getWaiter(ResultSetId rsId) {
- return waiters != null ? waiters.get(rsId) : null;
- }
-
- void setException(Exception exception) {
- if (waiters != null) {
- for (ResultSetId rsId : waiters.keySet()) {
- waiters.remove(rsId).callback.setException(exception);
- }
- }
- // Caches the exception anyway for future added waiters.
- this.exception = exception;
- }
-
- @Override
- public String toString() {
- return record.toString();
- }
-}
-
-class Waiters extends HashMap<ResultSetId, Waiter> {
- private static final long serialVersionUID = 1L;
-}
-
-class Waiter {
- DatasetDirectoryRecord[] knownRecords;
- IResultCallback<DatasetDirectoryRecord[]> callback;
-
- Waiter(DatasetDirectoryRecord[] knownRecords, IResultCallback<DatasetDirectoryRecord[]> callback) {
- this.knownRecords = knownRecords;
- this.callback = callback;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/IDatasetDirectoryService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/IDatasetDirectoryService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/IDatasetDirectoryService.java
deleted file mode 100644
index 68d6c16..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/IDatasetDirectoryService.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.cc.dataset;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.api.dataset.DatasetDirectoryRecord;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.dataset.IDatasetManager;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.IJobLifecycleListener;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.control.common.work.IResultCallback;
-
-public interface IDatasetDirectoryService extends IJobLifecycleListener, IDatasetManager {
- public void init(ExecutorService executor);
-
- public void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, boolean orderedResult,
- boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress)
- throws HyracksDataException;
-
- public void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition)
- throws HyracksDataException;
-
- public void reportJobFailure(JobId jobId, List<Exception> exceptions);
-
- public Status getResultStatus(JobId jobId, ResultSetId rsId) throws HyracksDataException;
-
- public void getResultPartitionLocations(JobId jobId, ResultSetId rsId, DatasetDirectoryRecord[] knownLocations,
- IResultCallback<DatasetDirectoryRecord[]> callback) throws HyracksDataException;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java
new file mode 100644
index 0000000..c8f576c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.cc.result;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.hyracks.api.comm.NetworkAddress;
+import org.apache.hyracks.api.result.IResultManager;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.IJobLifecycleListener;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.control.common.work.IResultCallback;
+
+public interface IResultDirectoryService extends IJobLifecycleListener, IResultManager {
+ public void init(ExecutorService executor);
+
+ public void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, boolean orderedResult,
+ boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress)
+ throws HyracksDataException;
+
+ public void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition)
+ throws HyracksDataException;
+
+ public void reportJobFailure(JobId jobId, List<Exception> exceptions);
+
+ public Status getResultStatus(JobId jobId, ResultSetId rsId) throws HyracksDataException;
+
+ public void getResultPartitionLocations(JobId jobId, ResultSetId rsId, ResultDirectoryRecord[] knownLocations,
+ IResultCallback<ResultDirectoryRecord[]> callback) throws HyracksDataException;
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
new file mode 100644
index 0000000..a65ce4c
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.cc.result;
+
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.hyracks.api.comm.NetworkAddress;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultJobRecord;
+import org.apache.hyracks.api.result.ResultJobRecord.State;
+import org.apache.hyracks.api.result.IResultStateRecord;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetMetaData;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.apache.hyracks.api.job.JobStatus;
+import org.apache.hyracks.control.common.result.AbstractResultManager;
+import org.apache.hyracks.control.common.result.ResultStateSweeper;
+import org.apache.hyracks.control.common.work.IResultCallback;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * TODO(madhusudancs): The potential perils of this global result directory service implementation is that, the jobs
+ * location information is never evicted from the memory and the memory usage grows as the number of jobs in the system
+ * grows. What we should possibly do is, add an API call for the client to say that it received everything it has to for
+ * the job (after it receives all the results) completely. Then we can just get rid of the location information for that
+ * job.
+ */
+public class ResultDirectoryService extends AbstractResultManager implements IResultDirectoryService {
+
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private final long resultSweepThreshold;
+
+ private final Map<JobId, JobResultInfo> jobResultLocations;
+
+ public ResultDirectoryService(long resultTTL, long resultSweepThreshold) {
+ super(resultTTL);
+ this.resultSweepThreshold = resultSweepThreshold;
+ jobResultLocations = new LinkedHashMap<>();
+ }
+
+ @Override
+ public void init(ExecutorService executor) {
+ executor.execute(new ResultStateSweeper(this, resultSweepThreshold, LOGGER));
+ }
+
+ @Override
+ public synchronized void notifyJobCreation(JobId jobId, JobSpecification spec) throws HyracksException {
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info(getClass().getSimpleName() + " notified of new job " + jobId);
+ }
+ if (jobResultLocations.get(jobId) != null) {
+ throw HyracksDataException.create(ErrorCode.MORE_THAN_ONE_RESULT, jobId);
+ }
+ jobResultLocations.put(jobId, new JobResultInfo(new ResultJobRecord(), null));
+ }
+
+ @Override
+ public synchronized void notifyJobStart(JobId jobId) throws HyracksException {
+ jobResultLocations.get(jobId).getRecord().start();
+ }
+
+ @Override
+ public void notifyJobFinish(JobId jobId, JobStatus jobStatus, List<Exception> exceptions) throws HyracksException {
+ // Auto-generated method stub
+ }
+
+ private ResultJobRecord getResultJobRecord(JobId jobId) {
+ final JobResultInfo jri = jobResultLocations.get(jobId);
+ return jri == null ? null : jri.getRecord();
+ }
+
+ private ResultJobRecord getNonNullResultJobRecord(JobId jobId) throws HyracksDataException {
+ final ResultJobRecord djr = getResultJobRecord(jobId);
+ if (djr == null) {
+ throw HyracksDataException.create(ErrorCode.NO_RESULT_SET, jobId);
+ }
+ return djr;
+ }
+
+ @Override
+ public synchronized void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, boolean orderedResult,
+ boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress)
+ throws HyracksDataException {
+ ResultJobRecord djr = getNonNullResultJobRecord(jobId);
+ djr.setResultSetMetaData(rsId, orderedResult, nPartitions);
+ ResultDirectoryRecord record = djr.getOrCreateDirectoryRecord(rsId, partition);
+
+ record.setNetworkAddress(networkAddress);
+ record.setEmpty(emptyResult);
+ record.start();
+
+ final JobResultInfo jobResultInfo = jobResultLocations.get(jobId);
+ Waiter waiter = jobResultInfo.getWaiter(rsId);
+ if (waiter != null) {
+ try {
+ ResultDirectoryRecord[] updatedRecords = updatedRecords(jobId, rsId, waiter.knownRecords);
+ if (updatedRecords != null) {
+ jobResultInfo.removeWaiter(rsId);
+ waiter.callback.setValue(updatedRecords);
+ }
+ } catch (Exception e) {
+ waiter.callback.setException(e);
+ }
+ }
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition)
+ throws HyracksDataException {
+ ResultJobRecord djr = getNonNullResultJobRecord(jobId);
+ djr.getDirectoryRecord(rsId, partition).writeEOS();
+ djr.updateState(rsId);
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void reportJobFailure(JobId jobId, List<Exception> exceptions) {
+ LOGGER.log(Level.INFO, "job " + jobId + " failed and is being reported to " + getClass().getSimpleName(),
+ exceptions.get(0));
+ ResultJobRecord rjr = getResultJobRecord(jobId);
+ LOGGER.log(Level.INFO, "Result job record is " + rjr);
+ if (rjr != null) {
+ LOGGER.log(Level.INFO, "Setting exceptions in Result job record");
+ rjr.fail(exceptions);
+ }
+ final JobResultInfo jobResultInfo = jobResultLocations.get(jobId);
+ LOGGER.log(Level.INFO, "Job result info is " + jobResultInfo);
+ if (jobResultInfo != null) {
+ LOGGER.log(Level.INFO, "Setting exceptions in Job result info");
+ jobResultInfo.setException(exceptions.isEmpty() ? null : exceptions.get(0));
+ }
+ notifyAll();
+ }
+
+ @Override
+ public synchronized ResultJobRecord.Status getResultStatus(JobId jobId, ResultSetId rsId)
+ throws HyracksDataException {
+ return getNonNullResultJobRecord(jobId).getStatus();
+ }
+
+ @Override
+ public synchronized Set<JobId> getJobIds() {
+ return jobResultLocations.keySet();
+ }
+
+ @Override
+ public IResultStateRecord getState(JobId jobId) {
+ return getResultJobRecord(jobId);
+ }
+
+ @Override
+ public synchronized void sweep(JobId jobId) {
+ jobResultLocations.remove(jobId);
+ }
+
+ @Override
+ public synchronized void getResultPartitionLocations(JobId jobId, ResultSetId rsId,
+ ResultDirectoryRecord[] knownRecords, IResultCallback<ResultDirectoryRecord[]> callback)
+ throws HyracksDataException {
+ ResultDirectoryRecord[] updatedRecords = updatedRecords(jobId, rsId, knownRecords);
+ if (updatedRecords == null) {
+ jobResultLocations.get(jobId).addWaiter(rsId, knownRecords, callback);
+ } else {
+ callback.setValue(updatedRecords);
+ }
+ }
+
+ /**
+ * Compares the records already known by the client for the given job's result set id with the records that the
+ * result directory service knows and if there are any newly discovered records returns a whole array with the
+ * new records filled in.
+ *
+ * @param jobId
+ * - Id of the job for which the directory records should be retrieved.
+ * @param rsId
+ * - Id of the result set for which the directory records should be retrieved.
+ * @param knownRecords
+ * - An array of directory records that the client is already aware of.
+ * @return
+ * Returns the updated records if new record were discovered, null otherwise
+ * @throws HyracksDataException
+ * TODO(madhusudancs): Think about caching (and still be stateless) instead of this ugly O(n)
+ * iterations for every check. This already looks very expensive.
+ */
+ private ResultDirectoryRecord[] updatedRecords(JobId jobId, ResultSetId rsId, ResultDirectoryRecord[] knownRecords)
+ throws HyracksDataException {
+ ResultJobRecord djr = getNonNullResultJobRecord(jobId);
+
+ if (djr.getStatus().getState() == State.FAILED) {
+ List<Exception> caughtExceptions = djr.getStatus().getExceptions();
+ if (caughtExceptions != null && !caughtExceptions.isEmpty()) {
+ final Exception cause = caughtExceptions.get(caughtExceptions.size() - 1);
+ if (cause instanceof HyracksDataException) {
+ throw (HyracksDataException) cause;
+ }
+ throw HyracksDataException.create(ErrorCode.RESULT_FAILURE_EXCEPTION, cause, rsId, jobId);
+ } else {
+ throw HyracksDataException.create(ErrorCode.RESULT_FAILURE_NO_EXCEPTION, rsId, jobId);
+ }
+ }
+
+ final ResultSetMetaData resultSetMetaData = djr.getResultSetMetaData(rsId);
+ if (resultSetMetaData == null) {
+ return null;
+ }
+ ResultDirectoryRecord[] records = resultSetMetaData.getRecords();
+
+ return Arrays.equals(records, knownRecords) ? null : records;
+ }
+
+ public PrintWriter print(PrintWriter pw) {
+ for (JobId jId : getJobIds()) {
+ pw.print(jId.toString());
+ pw.print(" - ");
+ pw.println(String.valueOf(getResultJobRecord(jId)));
+ }
+ pw.flush();
+ return pw;
+ }
+}
+
+class JobResultInfo {
+
+ private ResultJobRecord record;
+ private Waiters waiters;
+ private Exception exception;
+
+ JobResultInfo(ResultJobRecord record, Waiters waiters) {
+ this.record = record;
+ this.waiters = waiters;
+ }
+
+ ResultJobRecord getRecord() {
+ return record;
+ }
+
+ void addWaiter(ResultSetId rsId, ResultDirectoryRecord[] knownRecords,
+ IResultCallback<ResultDirectoryRecord[]> callback) {
+ if (waiters == null) {
+ waiters = new Waiters();
+ }
+ waiters.put(rsId, new Waiter(knownRecords, callback));
+ if (exception != null) {
+ // Exception was set before the waiter is added.
+ setException(exception);
+ }
+ }
+
+ Waiter removeWaiter(ResultSetId rsId) {
+ return waiters.remove(rsId);
+ }
+
+ Waiter getWaiter(ResultSetId rsId) {
+ return waiters != null ? waiters.get(rsId) : null;
+ }
+
+ void setException(Exception exception) {
+ if (waiters != null) {
+ for (ResultSetId rsId : waiters.keySet()) {
+ waiters.remove(rsId).callback.setException(exception);
+ }
+ }
+ // Caches the exception anyway for future added waiters.
+ this.exception = exception;
+ }
+
+ @Override
+ public String toString() {
+ return record.toString();
+ }
+}
+
+class Waiters extends HashMap<ResultSetId, Waiter> {
+ private static final long serialVersionUID = 1L;
+}
+
+class Waiter {
+ ResultDirectoryRecord[] knownRecords;
+ IResultCallback<ResultDirectoryRecord[]> callback;
+
+ Waiter(ResultDirectoryRecord[] knownRecords, IResultCallback<ResultDirectoryRecord[]> callback) {
+ this.knownRecords = knownRecords;
+ this.callback = callback;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetDatasetDirectoryServiceInfoWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetDatasetDirectoryServiceInfoWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetDatasetDirectoryServiceInfoWork.java
deleted file mode 100644
index fb1febf..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetDatasetDirectoryServiceInfoWork.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.cc.work;
-
-import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.common.work.IResultCallback;
-import org.apache.hyracks.control.common.work.SynchronizableWork;
-
-public class GetDatasetDirectoryServiceInfoWork extends SynchronizableWork {
- private final ClusterControllerService ccs;
-
- private final IResultCallback<NetworkAddress> callback;
-
- public GetDatasetDirectoryServiceInfoWork(ClusterControllerService ccs, IResultCallback<NetworkAddress> callback) {
- this.ccs = ccs;
- this.callback = callback;
- }
-
- @Override
- public void doRun() {
- try {
- NetworkAddress addr = ccs.getDatasetDirectoryServiceInfo();
- callback.setValue(addr);
- } catch (Exception e) {
- callback.setException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultDirectoryAddressWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultDirectoryAddressWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultDirectoryAddressWork.java
new file mode 100644
index 0000000..bf95ff2
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultDirectoryAddressWork.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.cc.work;
+
+import org.apache.hyracks.api.comm.NetworkAddress;
+import org.apache.hyracks.control.cc.ClusterControllerService;
+import org.apache.hyracks.control.common.work.IResultCallback;
+import org.apache.hyracks.control.common.work.SynchronizableWork;
+
+public class GetResultDirectoryAddressWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+
+ private final IResultCallback<NetworkAddress> callback;
+
+ public GetResultDirectoryAddressWork(ClusterControllerService ccs, IResultCallback<NetworkAddress> callback) {
+ this.ccs = ccs;
+ this.callback = callback;
+ }
+
+ @Override
+ public void doRun() {
+ try {
+ NetworkAddress addr = ccs.getResultDirectoryAddress();
+ callback.setValue(addr);
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultPartitionLocationsWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultPartitionLocationsWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultPartitionLocationsWork.java
index ce82ec8..68382f7 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultPartitionLocationsWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultPartitionLocationsWork.java
@@ -20,12 +20,12 @@ package org.apache.hyracks.control.cc.work;
import java.util.Arrays;
-import org.apache.hyracks.api.dataset.DatasetDirectoryRecord;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.control.cc.ClusterControllerService;
-import org.apache.hyracks.control.cc.dataset.IDatasetDirectoryService;
+import org.apache.hyracks.control.cc.result.IResultDirectoryService;
import org.apache.hyracks.control.common.work.IResultCallback;
import org.apache.hyracks.control.common.work.SynchronizableWork;
@@ -36,12 +36,12 @@ public class GetResultPartitionLocationsWork extends SynchronizableWork {
private final ResultSetId rsId;
- private final DatasetDirectoryRecord[] knownRecords;
+ private final ResultDirectoryRecord[] knownRecords;
- private final IResultCallback<DatasetDirectoryRecord[]> callback;
+ private final IResultCallback<ResultDirectoryRecord[]> callback;
public GetResultPartitionLocationsWork(ClusterControllerService ccs, JobId jobId, ResultSetId rsId,
- DatasetDirectoryRecord[] knownRecords, IResultCallback<DatasetDirectoryRecord[]> callback) {
+ ResultDirectoryRecord[] knownRecords, IResultCallback<ResultDirectoryRecord[]> callback) {
this.ccs = ccs;
this.jobId = jobId;
this.rsId = rsId;
@@ -51,7 +51,7 @@ public class GetResultPartitionLocationsWork extends SynchronizableWork {
@Override
public void doRun() {
- final IDatasetDirectoryService dds = ccs.getDatasetDirectoryService();
+ final IResultDirectoryService dds = ccs.getResultDirectoryService();
ccs.getExecutor().execute(new Runnable() {
@Override
public void run() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultStatusWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultStatusWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultStatusWork.java
index 676276a..560f505 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultStatusWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultStatusWork.java
@@ -18,8 +18,8 @@
*/
package org.apache.hyracks.control.cc.work;
-import org.apache.hyracks.api.dataset.DatasetJobRecord.Status;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.control.cc.ClusterControllerService;
@@ -46,7 +46,7 @@ public class GetResultStatusWork extends SynchronizableWork {
@Override
public void doRun() {
try {
- Status status = ccs.getDatasetDirectoryService().getResultStatus(jobId, rsId);
+ Status status = ccs.getResultDirectoryService().getResultStatus(jobId, rsId);
callback.setValue(status);
} catch (HyracksDataException e) {
callback.setException(e);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java
index ad36701..98fe722 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.control.cc.ClusterControllerService;
@@ -68,7 +68,7 @@ public class RegisterResultPartitionLocationWork extends AbstractWork {
@Override
public void run() {
try {
- ccs.getDatasetDirectoryService().registerResultPartitionLocation(jobId, rsId, orderedResult, emptyResult,
+ ccs.getResultDirectoryService().registerResultPartitionLocation(jobId, rsId, orderedResult, emptyResult,
partition, nPartitions, networkAddress);
} catch (HyracksDataException e) {
LOGGER.log(Level.WARN, "Failed to register partition location", e);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java
index d63bc8a..b36c917 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java
@@ -18,7 +18,7 @@
*/
package org.apache.hyracks.control.cc.work;
-import org.apache.hyracks.api.dataset.ResultSetId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.control.cc.ClusterControllerService;
@@ -44,7 +44,7 @@ public class ReportResultPartitionWriteCompletionWork extends AbstractWork {
@Override
public void run() {
try {
- ccs.getDatasetDirectoryService().reportResultPartitionWriteCompletion(jobId, rsId, partition);
+ ccs.getResultDirectoryService().reportResultPartitionWriteCompletion(jobId, rsId, partition);
} catch (HyracksDataException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java
index e7af300..8c86aff 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java
@@ -48,7 +48,7 @@ public class TaskFailureWork extends AbstractTaskLifecycleWork {
if (run == null) {
return;
}
- ccs.getDatasetDirectoryService().reportJobFailure(jobId, exceptions);
+ ccs.getResultDirectoryService().reportJobFailure(jobId, exceptions);
run.getExecutor().notifyTaskFailure(ta, exceptions);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/cluster/NodeManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/cluster/NodeManagerTest.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/cluster/NodeManagerTest.java
index cc2ed46..43b00f3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/cluster/NodeManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/test/java/org/apache/hyracks/control/cc/cluster/NodeManagerTest.java
@@ -172,7 +172,7 @@ public class NodeManagerTest {
NetworkAddress msgAddr = new NetworkAddress(ipAddr, 1003);
when(ncState.getCapacity()).thenReturn(new NodeCapacity(NODE_MEMORY_SIZE, NODE_CORES));
when(ncState.getDataPort()).thenReturn(dataAddr);
- when(ncState.getDatasetPort()).thenReturn(resultAddr);
+ when(ncState.getResultPort()).thenReturn(resultAddr);
when(ncState.getMessagingPort()).thenReturn(msgAddr);
NCConfig ncConfig = new NCConfig(nodeId);
ncConfig.setDataPublicAddress(ipAddr);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
index ddab504..1926da6 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/NodeControllerData.java
@@ -46,7 +46,7 @@ public class NodeControllerData {
private final NetworkAddress dataPort;
- private final NetworkAddress datasetPort;
+ private final NetworkAddress resultPort;
private final NetworkAddress messagingPort;
@@ -118,13 +118,13 @@ public class NodeControllerData {
private final long[] netSignalingBytesWritten;
- private final long[] datasetNetPayloadBytesRead;
+ private final long[] resultNetPayloadBytesRead;
- private final long[] datasetNetPayloadBytesWritten;
+ private final long[] resultNetPayloadBytesWritten;
- private final long[] datasetNetSignalingBytesRead;
+ private final long[] resultNetSignalingBytesRead;
- private final long[] datasetNetSignalingBytesWritten;
+ private final long[] resultNetSignalingBytesWritten;
private final long[] ipcMessagesSent;
@@ -147,7 +147,7 @@ public class NodeControllerData {
public NodeControllerData(NodeRegistration reg) {
ncConfig = reg.getNCConfig();
dataPort = reg.getDataPort();
- datasetPort = reg.getDatasetPort();
+ resultPort = reg.getResultPort();
messagingPort = reg.getMessagingPort();
activeJobIds = new HashSet<>();
@@ -191,10 +191,10 @@ public class NodeControllerData {
netPayloadBytesWritten = new long[RRD_SIZE];
netSignalingBytesRead = new long[RRD_SIZE];
netSignalingBytesWritten = new long[RRD_SIZE];
- datasetNetPayloadBytesRead = new long[RRD_SIZE];
- datasetNetPayloadBytesWritten = new long[RRD_SIZE];
- datasetNetSignalingBytesRead = new long[RRD_SIZE];
- datasetNetSignalingBytesWritten = new long[RRD_SIZE];
+ resultNetPayloadBytesRead = new long[RRD_SIZE];
+ resultNetPayloadBytesWritten = new long[RRD_SIZE];
+ resultNetSignalingBytesRead = new long[RRD_SIZE];
+ resultNetSignalingBytesWritten = new long[RRD_SIZE];
ipcMessagesSent = new long[RRD_SIZE];
ipcMessageBytesSent = new long[RRD_SIZE];
ipcMessagesReceived = new long[RRD_SIZE];
@@ -231,10 +231,10 @@ public class NodeControllerData {
netPayloadBytesWritten[rrdPtr] = hbData.netPayloadBytesWritten;
netSignalingBytesRead[rrdPtr] = hbData.netSignalingBytesRead;
netSignalingBytesWritten[rrdPtr] = hbData.netSignalingBytesWritten;
- datasetNetPayloadBytesRead[rrdPtr] = hbData.datasetNetPayloadBytesRead;
- datasetNetPayloadBytesWritten[rrdPtr] = hbData.datasetNetPayloadBytesWritten;
- datasetNetSignalingBytesRead[rrdPtr] = hbData.datasetNetSignalingBytesRead;
- datasetNetSignalingBytesWritten[rrdPtr] = hbData.datasetNetSignalingBytesWritten;
+ resultNetPayloadBytesRead[rrdPtr] = hbData.resultNetPayloadBytesRead;
+ resultNetPayloadBytesWritten[rrdPtr] = hbData.resultNetPayloadBytesWritten;
+ resultNetSignalingBytesRead[rrdPtr] = hbData.resultNetSignalingBytesRead;
+ resultNetSignalingBytesWritten[rrdPtr] = hbData.resultNetSignalingBytesWritten;
ipcMessagesSent[rrdPtr] = hbData.ipcMessagesSent;
ipcMessageBytesSent[rrdPtr] = hbData.ipcMessageBytesSent;
ipcMessagesReceived[rrdPtr] = hbData.ipcMessagesReceived;
@@ -264,8 +264,8 @@ public class NodeControllerData {
return dataPort;
}
- public NetworkAddress getDatasetPort() {
- return datasetPort;
+ public NetworkAddress getResultPort() {
+ return resultPort;
}
public NetworkAddress getMessagingPort() {
@@ -332,10 +332,10 @@ public class NodeControllerData {
put(o, "net-payload-bytes-written", netPayloadBytesWritten);
put(o, "net-signaling-bytes-read", netSignalingBytesRead);
put(o, "net-signaling-bytes-written", netSignalingBytesWritten);
- put(o, "dataset-net-payload-bytes-read", datasetNetPayloadBytesRead);
- put(o, "dataset-net-payload-bytes-written", datasetNetPayloadBytesWritten);
- put(o, "dataset-net-signaling-bytes-read", datasetNetSignalingBytesRead);
- put(o, "dataset-net-signaling-bytes-written", datasetNetSignalingBytesWritten);
+ put(o, "result-net-payload-bytes-read", resultNetPayloadBytesRead);
+ put(o, "result-net-payload-bytes-written", resultNetPayloadBytesWritten);
+ put(o, "result-net-signaling-bytes-read", resultNetSignalingBytesRead);
+ put(o, "result-net-signaling-bytes-written", resultNetSignalingBytesWritten);
put(o, "ipc-messages-sent", ipcMessagesSent);
put(o, "ipc-message-bytes-sent", ipcMessageBytesSent);
put(o, "ipc-messages-received", ipcMessagesReceived);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java
index 6230f1d..fc0154e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java
@@ -22,10 +22,10 @@ import java.util.List;
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.job.DeployedJobSpecId;
import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.control.common.controllers.NodeRegistration;
import org.apache.hyracks.control.common.deployment.DeploymentStatus;
import org.apache.hyracks.control.common.heartbeat.HeartbeatData;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
index aa3260d..95c67df 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NCConfig.java
@@ -157,13 +157,13 @@ public class NCConfig extends ControllerConfig {
case DATA_PUBLIC_PORT:
return "Public IP port to announce data listener";
case RESULT_LISTEN_ADDRESS:
- return "IP Address to bind dataset result distribution listener";
+ return "IP Address to bind result distribution listener";
case RESULT_LISTEN_PORT:
- return "IP port to bind dataset result distribution listener";
+ return "IP port to bind result distribution listener";
case RESULT_PUBLIC_ADDRESS:
- return "Public IP Address to announce dataset result distribution listener";
+ return "Public IP Address to announce result distribution listener";
case RESULT_PUBLIC_PORT:
- return "Public IP port to announce dataset result distribution listener";
+ return "Public IP port to announce result distribution listener";
case MESSAGING_LISTEN_ADDRESS:
return "IP Address to bind messaging listener";
case MESSAGING_LISTEN_PORT:
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
index c2f2e1a..b4d835d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/controllers/NodeRegistration.java
@@ -43,7 +43,7 @@ public final class NodeRegistration implements Serializable {
private final NetworkAddress dataPort;
- private final NetworkAddress datasetPort;
+ private final NetworkAddress resultPort;
private final String osName;
@@ -78,12 +78,12 @@ public final class NodeRegistration implements Serializable {
private final NodeCapacity capacity;
public NodeRegistration(InetSocketAddress ncAddress, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
- NetworkAddress datasetPort, HeartbeatSchema hbSchema, NetworkAddress messagingPort, NodeCapacity capacity) {
+ NetworkAddress resultPort, HeartbeatSchema hbSchema, NetworkAddress messagingPort, NodeCapacity capacity) {
this.ncAddress = ncAddress;
this.nodeId = nodeId;
this.ncConfig = ncConfig;
this.dataPort = dataPort;
- this.datasetPort = datasetPort;
+ this.resultPort = resultPort;
this.hbSchema = hbSchema;
this.messagingPort = messagingPort;
this.capacity = capacity;
@@ -122,8 +122,8 @@ public final class NodeRegistration implements Serializable {
return dataPort;
}
- public NetworkAddress getDatasetPort() {
- return datasetPort;
+ public NetworkAddress getResultPort() {
+ return resultPort;
}
public String getOSName() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/AbstractDatasetManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/AbstractDatasetManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/AbstractDatasetManager.java
deleted file mode 100644
index f95229e..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/AbstractDatasetManager.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.common.dataset;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hyracks.api.dataset.IDatasetManager;
-import org.apache.hyracks.api.dataset.IDatasetStateRecord;
-import org.apache.hyracks.api.job.JobId;
-
-public abstract class AbstractDatasetManager implements IDatasetManager {
-
- private final long nanoResultTTL;
-
- protected AbstractDatasetManager(long resultTTL) {
- this.nanoResultTTL = TimeUnit.MILLISECONDS.toNanos(resultTTL);
- }
-
- @Override
- public synchronized void sweepExpiredDatasets() {
- final List<JobId> expiredDatasets = new ArrayList<>();
- final long sweepTime = System.nanoTime();
- for (JobId jobId : getJobIds()) {
- final IDatasetStateRecord state = getState(jobId);
- if (state != null && hasExpired(state, sweepTime, nanoResultTTL)) {
- expiredDatasets.add(jobId);
- }
- }
- for (JobId jobId : expiredDatasets) {
- sweep(jobId);
- }
- }
-
- private static boolean hasExpired(IDatasetStateRecord dataset, long currentTime, long ttl) {
- return currentTime - dataset.getTimestamp() - ttl > 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/ResultStateSweeper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/ResultStateSweeper.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/ResultStateSweeper.java
deleted file mode 100644
index 901ec67..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/dataset/ResultStateSweeper.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.control.common.dataset;
-
-import org.apache.hyracks.api.dataset.IDatasetManager;
-import org.apache.logging.log4j.Logger;
-
-/**
- * Sweeper to clean up the stale result distribution files and result states.
- */
-public class ResultStateSweeper implements Runnable {
-
- private final IDatasetManager datasetManager;
- private final long resultSweepThreshold;
- private final Logger logger;
-
- public ResultStateSweeper(IDatasetManager datasetManager, long resultSweepThreshold, Logger logger) {
- this.datasetManager = datasetManager;
- this.resultSweepThreshold = resultSweepThreshold;
- this.logger = logger;
- }
-
- @Override
- public void run() {
- while (!Thread.currentThread().isInterrupted()) {
- try {
- Thread.sleep(resultSweepThreshold);
- datasetManager.sweepExpiredDatasets();
- logger.trace("Result state cleanup instance successfully completed.");
- } catch (InterruptedException e) {
- logger.warn("Result cleaner thread interrupted, shutting down.");
- Thread.currentThread().interrupt();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
index 76b68bb..b8ba173 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
@@ -44,10 +44,10 @@ public class HeartbeatData {
public long netPayloadBytesWritten;
public long netSignalingBytesRead;
public long netSignalingBytesWritten;
- public long datasetNetPayloadBytesRead;
- public long datasetNetPayloadBytesWritten;
- public long datasetNetSignalingBytesRead;
- public long datasetNetSignalingBytesWritten;
+ public long resultNetPayloadBytesRead;
+ public long resultNetPayloadBytesWritten;
+ public long resultNetSignalingBytesRead;
+ public long resultNetSignalingBytesWritten;
public long ipcMessagesSent;
public long ipcMessageBytesSent;
public long ipcMessagesReceived;
@@ -79,9 +79,9 @@ public class HeartbeatData {
netSignalingBytesRead = dis.readLong();
netSignalingBytesWritten = dis.readLong();
netSignalingBytesWritten = dis.readLong();
- datasetNetPayloadBytesWritten = dis.readLong();
- datasetNetSignalingBytesRead = dis.readLong();
- datasetNetSignalingBytesWritten = dis.readLong();
+ resultNetPayloadBytesWritten = dis.readLong();
+ resultNetSignalingBytesRead = dis.readLong();
+ resultNetSignalingBytesWritten = dis.readLong();
ipcMessagesSent = dis.readLong();
ipcMessageBytesSent = dis.readLong();
ipcMessagesReceived = dis.readLong();
@@ -119,10 +119,10 @@ public class HeartbeatData {
dos.writeLong(netPayloadBytesWritten);
dos.writeLong(netSignalingBytesRead);
dos.writeLong(netSignalingBytesWritten);
- dos.writeLong(datasetNetPayloadBytesRead);
- dos.writeLong(datasetNetPayloadBytesWritten);
- dos.writeLong(datasetNetSignalingBytesRead);
- dos.writeLong(datasetNetSignalingBytesWritten);
+ dos.writeLong(resultNetPayloadBytesRead);
+ dos.writeLong(resultNetPayloadBytesWritten);
+ dos.writeLong(resultNetSignalingBytesRead);
+ dos.writeLong(resultNetSignalingBytesWritten);
dos.writeLong(ipcMessagesSent);
dos.writeLong(ipcMessageBytesSent);
dos.writeLong(ipcMessagesReceived);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java
index dea5198..ce4578d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java
@@ -46,13 +46,13 @@ import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.dataflow.TaskId;
import org.apache.hyracks.api.dataflow.connectors.ConnectorPolicyFactory;
import org.apache.hyracks.api.dataflow.connectors.IConnectorPolicy;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.job.DeployedJobSpecId;
import org.apache.hyracks.api.job.JobFlag;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobStatus;
import org.apache.hyracks.api.partitions.PartitionId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.control.common.controllers.NodeParameters;
import org.apache.hyracks.control.common.controllers.NodeRegistration;
import org.apache.hyracks.control.common.deployment.DeploymentStatus;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
index bf35e6b..8e2ec22 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
@@ -22,10 +22,10 @@ import java.util.List;
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
-import org.apache.hyracks.api.dataset.ResultSetId;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.job.DeployedJobSpecId;
import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultSetId;
import org.apache.hyracks.control.common.base.IClusterController;
import org.apache.hyracks.control.common.controllers.NodeRegistration;
import org.apache.hyracks.control.common.deployment.DeploymentStatus;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/AbstractResultManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/AbstractResultManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/AbstractResultManager.java
new file mode 100644
index 0000000..e314a6d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/AbstractResultManager.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.common.result;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hyracks.api.result.IResultManager;
+import org.apache.hyracks.api.result.IResultStateRecord;
+import org.apache.hyracks.api.job.JobId;
+
+public abstract class AbstractResultManager implements IResultManager {
+
+ private final long nanoResultTTL;
+
+ protected AbstractResultManager(long resultTTL) {
+ this.nanoResultTTL = TimeUnit.MILLISECONDS.toNanos(resultTTL);
+ }
+
+ @Override
+ public synchronized void sweepExpiredResultSets() {
+ final List<JobId> expiredResultSets = new ArrayList<>();
+ final long sweepTime = System.nanoTime();
+ for (JobId jobId : getJobIds()) {
+ final IResultStateRecord state = getState(jobId);
+ if (state != null && hasExpired(state, sweepTime, nanoResultTTL)) {
+ expiredResultSets.add(jobId);
+ }
+ }
+ for (JobId jobId : expiredResultSets) {
+ sweep(jobId);
+ }
+ }
+
+ private static boolean hasExpired(IResultStateRecord state, long currentTime, long ttl) {
+ return currentTime - state.getTimestamp() - ttl > 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/ResultStateSweeper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/ResultStateSweeper.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/ResultStateSweeper.java
new file mode 100644
index 0000000..76bbabb
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/result/ResultStateSweeper.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.control.common.result;
+
+import org.apache.hyracks.api.result.IResultManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * Sweeper to clean up the stale result distribution files and result states.
+ */
+public class ResultStateSweeper implements Runnable {
+
+ private final IResultManager resultManager;
+ private final long resultSweepThreshold;
+ private final Logger logger;
+
+ public ResultStateSweeper(IResultManager resultManager, long resultSweepThreshold, Logger logger) {
+ this.resultManager = resultManager;
+ this.resultSweepThreshold = resultSweepThreshold;
+ this.logger = logger;
+ }
+
+ @Override
+ public void run() {
+ while (!Thread.currentThread().isInterrupted()) {
+ try {
+ Thread.sleep(resultSweepThreshold);
+ resultManager.sweepExpiredResultSets();
+ logger.trace("Result state cleanup instance successfully completed.");
+ } catch (InterruptedException e) {
+ logger.warn("Result cleaner thread interrupted, shutting down.");
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+}
[2/5] asterixdb git commit: [ASTERIXDB-2314][HYR] Dataset in class
names in Hyracks
Posted by ti...@apache.org.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 804bacd..169e5ea 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -46,7 +46,6 @@ import org.apache.hyracks.api.client.NodeControllerInfo;
import org.apache.hyracks.api.client.NodeStatus;
import org.apache.hyracks.api.comm.NetworkAddress;
import org.apache.hyracks.api.control.CcId;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -57,6 +56,7 @@ import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.job.JobParameterByteStore;
import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
import org.apache.hyracks.api.lifecycle.LifeCycleComponentManager;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.api.service.IControllerService;
import org.apache.hyracks.api.util.CleanupUtils;
import org.apache.hyracks.api.util.InvokeUtil;
@@ -74,15 +74,15 @@ import org.apache.hyracks.control.common.job.profiling.om.JobProfile;
import org.apache.hyracks.control.common.work.FutureValue;
import org.apache.hyracks.control.common.work.WorkQueue;
import org.apache.hyracks.control.nc.application.NCServiceContext;
-import org.apache.hyracks.control.nc.dataset.DatasetPartitionManager;
import org.apache.hyracks.control.nc.heartbeat.HeartbeatComputeTask;
import org.apache.hyracks.control.nc.heartbeat.HeartbeatTask;
import org.apache.hyracks.control.nc.io.IOManager;
-import org.apache.hyracks.control.nc.net.DatasetNetworkManager;
import org.apache.hyracks.control.nc.net.MessagingNetworkManager;
import org.apache.hyracks.control.nc.net.NetworkManager;
+import org.apache.hyracks.control.nc.net.ResultNetworkManager;
import org.apache.hyracks.control.nc.partitions.PartitionManager;
import org.apache.hyracks.control.nc.resources.memory.MemoryManager;
+import org.apache.hyracks.control.nc.result.ResultPartitionManager;
import org.apache.hyracks.control.nc.work.AbortAllJobsWork;
import org.apache.hyracks.control.nc.work.BuildJobProfilesWork;
import org.apache.hyracks.ipc.api.IIPCEventListener;
@@ -118,9 +118,9 @@ public class NodeControllerService implements IControllerService {
private NetworkManager netManager;
- private IDatasetPartitionManager datasetPartitionManager;
+ private IResultPartitionManager resultPartitionManager;
- private DatasetNetworkManager datasetNetworkManager;
+ private ResultNetworkManager resultNetworkManager;
private final WorkQueue workQueue;
@@ -262,10 +262,10 @@ public class NodeControllerService implements IControllerService {
}
private void init() {
- datasetPartitionManager = new DatasetPartitionManager(this, executor, ncConfig.getResultManagerMemory(),
+ resultPartitionManager = new ResultPartitionManager(this, executor, ncConfig.getResultManagerMemory(),
ncConfig.getResultTTL(), ncConfig.getResultSweepThreshold());
- datasetNetworkManager = new DatasetNetworkManager(ncConfig.getResultListenAddress(),
- ncConfig.getResultListenPort(), datasetPartitionManager, ncConfig.getNetThreadCount(),
+ resultNetworkManager = new ResultNetworkManager(ncConfig.getResultListenAddress(),
+ ncConfig.getResultListenPort(), resultPartitionManager, ncConfig.getNetThreadCount(),
ncConfig.getNetBufferCount(), ncConfig.getResultPublicAddress(), ncConfig.getResultPublicPort(),
FullFrameChannelInterfaceFactory.INSTANCE);
if (ncConfig.getMessagingListenAddress() != null && serviceCtx.getMessagingChannelInterfaceFactory() != null) {
@@ -289,7 +289,7 @@ public class NodeControllerService implements IControllerService {
netManager.start();
startApplication();
init();
- datasetNetworkManager.start();
+ resultNetworkManager.start();
if (messagingNetManager != null) {
messagingNetManager.start();
}
@@ -323,11 +323,11 @@ public class NodeControllerService implements IControllerService {
}
HeartbeatSchema hbSchema = new HeartbeatSchema(gcInfos);
- NetworkAddress datasetAddress = datasetNetworkManager.getPublicNetworkAddress();
+ NetworkAddress resultAddress = resultNetworkManager.getPublicNetworkAddress();
NetworkAddress netAddress = netManager.getPublicNetworkAddress();
NetworkAddress messagingAddress =
messagingNetManager != null ? messagingNetManager.getPublicNetworkAddress() : null;
- nodeRegistration = new NodeRegistration(ncAddress, id, ncConfig, netAddress, datasetAddress, hbSchema,
+ nodeRegistration = new NodeRegistration(ncAddress, id, ncConfig, netAddress, resultAddress, hbSchema,
messagingAddress, application.getCapacity());
ncData = new NodeControllerData(nodeRegistration);
@@ -494,9 +494,9 @@ public class NodeControllerService implements IControllerService {
LOGGER.log(Level.ERROR, "Some jobs failed to exit, continuing with abnormal shutdown");
}
partitionManager.close();
- datasetPartitionManager.close();
+ resultPartitionManager.close();
netManager.stop();
- datasetNetworkManager.stop();
+ resultNetworkManager.stop();
if (messagingNetManager != null) {
messagingNetManager.stop();
}
@@ -582,8 +582,8 @@ public class NodeControllerService implements IControllerService {
return netManager;
}
- public DatasetNetworkManager getDatasetNetworkManager() {
- return datasetNetworkManager;
+ public ResultNetworkManager getResultNetworkManager() {
+ return resultNetworkManager;
}
public PartitionManager getPartitionManager() {
@@ -645,8 +645,8 @@ public class NodeControllerService implements IControllerService {
getClusterController(ccId).sendApplicationMessageToCC(data, deploymentId, id);
}
- public IDatasetPartitionManager getDatasetPartitionManager() {
- return datasetPartitionManager;
+ public IResultPartitionManager getResultPartitionManager() {
+ return resultPartitionManager;
}
public MessagingNetworkManager getMessagingNetworkManager() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
index 340924d..f6531d7 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
@@ -43,7 +43,6 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.dataflow.state.IStateObject;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
@@ -57,6 +56,7 @@ import org.apache.hyracks.api.job.profiling.counters.ICounter;
import org.apache.hyracks.api.job.profiling.counters.ICounterContext;
import org.apache.hyracks.api.partitions.PartitionId;
import org.apache.hyracks.api.resources.IDeallocatable;
+import org.apache.hyracks.api.result.IResultPartitionManager;
import org.apache.hyracks.api.util.ExceptionUtils;
import org.apache.hyracks.api.util.JavaSerializationUtils;
import org.apache.hyracks.control.common.job.PartitionState;
@@ -427,8 +427,8 @@ public class Task implements IHyracksTaskContext, ICounterContext, Runnable {
}
@Override
- public IDatasetPartitionManager getDatasetPartitionManager() {
- return ncs.getDatasetPartitionManager();
+ public IResultPartitionManager getResultPartitionManager() {
+ return ncs.getResultPartitionManager();
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetMemoryManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetMemoryManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetMemoryManager.java
deleted file mode 100644
index 37cddb8..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetMemoryManager.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.dataset;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.partitions.ResultSetPartitionId;
-
-public class DatasetMemoryManager {
- private int availableMemory;
-
- private final Set<Page> availPages;
-
- private final LeastRecentlyUsedList leastRecentlyUsedList;
-
- private final Map<ResultSetPartitionId, PartitionNode> resultPartitionNodesMap;
-
- private final static int FRAME_SIZE = 32768;
-
- public DatasetMemoryManager(int availableMemory) {
- this.availableMemory = availableMemory;
-
- availPages = new HashSet<Page>();
-
- // Atleast have one page for temporarily storing the results.
- if (this.availableMemory <= FRAME_SIZE)
- this.availableMemory = FRAME_SIZE;
-
- leastRecentlyUsedList = new LeastRecentlyUsedList();
- resultPartitionNodesMap = new HashMap<ResultSetPartitionId, PartitionNode>();
- }
-
- public synchronized Page requestPage(ResultSetPartitionId resultSetPartitionId, ResultState resultState)
- throws HyracksDataException {
- Page page;
- if (availPages.isEmpty()) {
- if (availableMemory >= FRAME_SIZE) {
- /* TODO(madhusudancs): Should we have some way of accounting this memory usage by using Hyrack's allocateFrame()
- * instead of direct ByteBuffer.allocate()?
- */
- availPages.add(new Page(ByteBuffer.allocate(FRAME_SIZE)));
- availableMemory -= FRAME_SIZE;
- page = getAvailablePage();
- } else {
- page = evictPage();
- }
- } else {
- page = getAvailablePage();
- }
-
- page.clear();
-
- /*
- * It is extremely important to update the reference after obtaining the page because, in the cases where memory
- * manager is allocated only one page of memory, the front of the LRU list should not be created by the
- * update reference call before a page is pushed on to the element of the LRU list. So we first obtain the page,
- * then make a updateReference call which in turn creates a new node in the LRU list and then add the page to it.
- */
- PartitionNode pn = updateReference(resultSetPartitionId, resultState);
- pn.add(page);
- return page;
- }
-
- public void pageReferenced(ResultSetPartitionId resultSetPartitionId) {
- // When a page is referenced the dataset partition writer should already be known, so we pass null.
- updateReference(resultSetPartitionId, null);
- }
-
- public static int getPageSize() {
- return FRAME_SIZE;
- }
-
- protected void insertPartitionNode(ResultSetPartitionId resultSetPartitionId, PartitionNode pn) {
- leastRecentlyUsedList.add(pn);
- resultPartitionNodesMap.put(resultSetPartitionId, pn);
- }
-
- protected PartitionNode updateReference(ResultSetPartitionId resultSetPartitionId, ResultState resultState) {
- PartitionNode pn = null;
-
- if (!resultPartitionNodesMap.containsKey(resultSetPartitionId)) {
- if (resultState != null) {
- pn = new PartitionNode(resultSetPartitionId, resultState);
- insertPartitionNode(resultSetPartitionId, pn);
- }
- return pn;
- }
- synchronized (this) {
- pn = resultPartitionNodesMap.get(resultSetPartitionId);
- leastRecentlyUsedList.remove(pn);
- insertPartitionNode(resultSetPartitionId, pn);
- }
-
- return pn;
- }
-
- protected Page evictPage() throws HyracksDataException {
- PartitionNode pn = leastRecentlyUsedList.getFirst();
- ResultState resultState = pn.getResultState();
- Page page = resultState.returnPage();
-
- /* If the partition holding the pages breaks the contract by not returning the page or it has no page, just take
- * away all the pages allocated to it and add to the available pages set.
- */
- if (page == null) {
- availPages.addAll(pn);
- pn.clear();
- resultPartitionNodesMap.remove(pn.getResultSetPartitionId());
- leastRecentlyUsedList.remove(pn);
-
- /* Based on the assumption that if the dataset partition writer returned a null page, it should be lying about
- * the number of pages it holds in which case we just evict all the pages it holds and should thus be able to
- * add all those pages to available set and we have at least one page to allocate back.
- */
- page = getAvailablePage();
- } else {
- pn.remove(page);
-
- // If the partition no more holds any pages, remove it from the linked list and the hash map.
- if (pn.isEmpty()) {
- resultPartitionNodesMap.remove(pn.getResultSetPartitionId());
- leastRecentlyUsedList.remove(pn);
- }
- }
-
- return page;
- }
-
- protected Page getAvailablePage() {
- Iterator<Page> iter = availPages.iterator();
- Page page = iter.next();
- iter.remove();
- return page;
- }
-
- private class LeastRecentlyUsedList {
- private PartitionNode head;
-
- private PartitionNode tail;
-
- public LeastRecentlyUsedList() {
- head = null;
- tail = null;
- }
-
- public void add(PartitionNode node) {
- if (head == null) {
- head = tail = node;
- return;
- }
- tail.setNext(node);
- node.setPrev(tail);
- tail = node;
- }
-
- public void remove(PartitionNode node) {
- if ((node == head) && (node == tail)) {
- head = tail = null;
- return;
- } else if (node == head) {
- head = head.getNext();
- head.setPrev(null);
- return;
- } else if (node == tail) {
- tail = tail.getPrev();
- tail.setNext(null);
- return;
- } else {
- PartitionNode prev = node.getPrev();
- PartitionNode next = node.getNext();
- prev.setNext(next);
- next.setPrev(prev);
- }
- }
-
- public PartitionNode getFirst() {
- return head;
- }
- }
-
- private class PartitionNode extends HashSet<Page> {
- private static final long serialVersionUID = 1L;
-
- private final ResultSetPartitionId resultSetPartitionId;
-
- private final ResultState resultState;
-
- private PartitionNode prev;
-
- private PartitionNode next;
-
- public PartitionNode(ResultSetPartitionId resultSetPartitionId, ResultState resultState) {
- this.resultSetPartitionId = resultSetPartitionId;
- this.resultState = resultState;
- prev = null;
- next = null;
- }
-
- public ResultSetPartitionId getResultSetPartitionId() {
- return resultSetPartitionId;
- }
-
- public ResultState getResultState() {
- return resultState;
- }
-
- public void setPrev(PartitionNode node) {
- prev = node;
- }
-
- public PartitionNode getPrev() {
- return prev;
- }
-
- public void setNext(PartitionNode node) {
- next = node;
- }
-
- public PartitionNode getNext() {
- return next;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionManager.java
deleted file mode 100644
index b7cf9a4..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionManager.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.dataset;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executor;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.io.IWorkspaceFileFactory;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.control.common.dataset.AbstractDatasetManager;
-import org.apache.hyracks.control.common.dataset.ResultStateSweeper;
-import org.apache.hyracks.control.nc.NodeControllerService;
-import org.apache.hyracks.control.nc.io.WorkspaceFileFactory;
-import org.apache.hyracks.control.nc.resources.DefaultDeallocatableRegistry;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class DatasetPartitionManager extends AbstractDatasetManager implements IDatasetPartitionManager {
- private static final Logger LOGGER = LogManager.getLogger();
-
- private final NodeControllerService ncs;
-
- private final Executor executor;
-
- private final Map<JobId, ResultSetMap> partitionResultStateMap;
-
- private final DefaultDeallocatableRegistry deallocatableRegistry;
-
- private final IWorkspaceFileFactory fileFactory;
-
- private final DatasetMemoryManager datasetMemoryManager;
-
- public DatasetPartitionManager(NodeControllerService ncs, Executor executor, int availableMemory, long resultTTL,
- long resultSweepThreshold) {
- super(resultTTL);
- this.ncs = ncs;
- this.executor = executor;
- deallocatableRegistry = new DefaultDeallocatableRegistry();
- fileFactory = new WorkspaceFileFactory(deallocatableRegistry, ncs.getIoManager());
- if (availableMemory >= DatasetMemoryManager.getPageSize()) {
- datasetMemoryManager = new DatasetMemoryManager(availableMemory);
- } else {
- datasetMemoryManager = null;
- }
- partitionResultStateMap = new HashMap<>();
- executor.execute(new ResultStateSweeper(this, resultSweepThreshold, LOGGER));
- }
-
- @Override
- public IFrameWriter createDatasetPartitionWriter(IHyracksTaskContext ctx, ResultSetId rsId, boolean orderedResult,
- boolean asyncMode, int partition, int nPartitions, long maxReads) {
- DatasetPartitionWriter dpw;
- JobId jobId = ctx.getJobletContext().getJobId();
- synchronized (this) {
- dpw = new DatasetPartitionWriter(ctx, this, jobId, rsId, asyncMode, orderedResult, partition, nPartitions,
- datasetMemoryManager, fileFactory, maxReads);
- ResultSetMap rsIdMap = partitionResultStateMap.computeIfAbsent(jobId, k -> new ResultSetMap());
- ResultState[] resultStates = rsIdMap.createOrGetResultStates(rsId, nPartitions);
- resultStates[partition] = dpw.getResultState();
- }
- LOGGER.debug("Initialized partition writer: JobId: {}:partition: {}", jobId, partition);
- return dpw;
- }
-
- @Override
- public void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, int partition, int nPartitions,
- boolean orderedResult, boolean emptyResult) throws HyracksException {
- try {
- // Be sure to send the *public* network address to the CC
- ncs.getClusterController(jobId.getCcId()).registerResultPartitionLocation(jobId, rsId, orderedResult,
- emptyResult, partition, nPartitions, ncs.getDatasetNetworkManager().getPublicNetworkAddress());
- } catch (Exception e) {
- throw HyracksException.create(e);
- }
- }
-
- @Override
- public void reportPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition) throws HyracksException {
- try {
- LOGGER.debug("Reporting partition write completion: JobId: {}:ResultSetId: {}:partition: {}", jobId, rsId,
- partition);
- ncs.getClusterController(jobId.getCcId()).reportResultPartitionWriteCompletion(jobId, rsId, partition);
- } catch (Exception e) {
- throw HyracksException.create(e);
- }
- }
-
- @Override
- public void initializeDatasetPartitionReader(JobId jobId, ResultSetId resultSetId, int partition,
- IFrameWriter writer) throws HyracksException {
- ResultState resultState = getResultState(jobId, resultSetId, partition);
- DatasetPartitionReader dpr = new DatasetPartitionReader(this, datasetMemoryManager, executor, resultState);
- dpr.writeTo(writer);
- LOGGER.debug("Initialized partition reader: JobId: {}:ResultSetId: {}:partition: {}", jobId, resultSetId,
- partition);
- }
-
- private synchronized ResultState getResultState(JobId jobId, ResultSetId resultSetId, int partition)
- throws HyracksException {
- ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
- if (rsIdMap == null) {
- throw new HyracksException("Unknown JobId " + jobId);
- }
- ResultState[] resultStates = rsIdMap.getResultStates(resultSetId);
- if (resultStates == null) {
- throw new HyracksException("Unknown JobId: " + jobId + " ResultSetId: " + resultSetId);
- }
- ResultState resultState = resultStates[partition];
- if (resultState == null) {
- throw new HyracksException("No DatasetPartitionWriter for partition " + partition);
- }
- return resultState;
- }
-
- @Override
- public synchronized void removePartition(JobId jobId, ResultSetId resultSetId, int partition) {
- ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
- if (rsIdMap != null && rsIdMap.removePartition(jobId, resultSetId, partition)) {
- partitionResultStateMap.remove(jobId);
- }
- }
-
- @Override
- public synchronized void abortReader(JobId jobId) {
- ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
- if (rsIdMap != null) {
- rsIdMap.abortAll();
- }
- }
-
- @Override
- public synchronized void close() {
- for (JobId jobId : getJobIds()) {
- deinit(jobId);
- }
- deallocatableRegistry.close();
- }
-
- @Override
- public synchronized Set<JobId> getJobIds() {
- return partitionResultStateMap.keySet();
- }
-
- @Override
- public synchronized ResultSetMap getState(JobId jobId) {
- return partitionResultStateMap.get(jobId);
- }
-
- @Override
- public synchronized void sweep(JobId jobId) {
- deinit(jobId);
- partitionResultStateMap.remove(jobId);
- }
-
- private synchronized void deinit(JobId jobId) {
- ResultSetMap rsIdMap = partitionResultStateMap.get(jobId);
- if (rsIdMap != null) {
- rsIdMap.closeAndDeleteAll();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionReader.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionReader.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionReader.java
deleted file mode 100644
index 8c4fcb0..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionReader.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.dataset;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.Executor;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.partitions.ResultSetPartitionId;
-import org.apache.hyracks.comm.channels.NetworkOutputChannel;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class DatasetPartitionReader {
- private static final Logger LOGGER = LogManager.getLogger();
-
- private final DatasetPartitionManager datasetPartitionManager;
- private final DatasetMemoryManager datasetMemoryManager;
- private final Executor executor;
- private final ResultState resultState;
-
- public DatasetPartitionReader(DatasetPartitionManager datasetPartitionManager,
- DatasetMemoryManager datasetMemoryManager, Executor executor, ResultState resultState) {
- this.datasetPartitionManager = datasetPartitionManager;
- this.datasetMemoryManager = datasetMemoryManager;
- this.executor = executor;
- this.resultState = resultState;
- }
-
- public void writeTo(final IFrameWriter writer) {
- executor.execute(new ResultPartitionSender((NetworkOutputChannel) writer));
- }
-
- private class ResultPartitionSender implements Runnable {
-
- private final NetworkOutputChannel channel;
-
- ResultPartitionSender(final NetworkOutputChannel channel) {
- this.channel = channel;
- }
-
- @Override
- public void run() {
- channel.setFrameSize(resultState.getFrameSize());
- channel.open();
- try {
- resultState.readOpen();
- long offset = 0;
- final ByteBuffer buffer = ByteBuffer.allocate(resultState.getFrameSize());
- while (true) {
- buffer.clear();
- final long size = read(offset, buffer);
- if (size <= 0) {
- break;
- } else if (size < buffer.limit()) {
- throw new IllegalStateException(
- "Premature end of file - readSize: " + size + " buffer limit: " + buffer.limit());
- }
- offset += size;
- buffer.flip();
- channel.nextFrame(buffer);
- }
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("result reading successful(" + resultState.getResultSetPartitionId() + ")");
- }
- } catch (Exception e) {
- LOGGER.error(() -> "failed to send result partition " + resultState.getResultSetPartitionId(), e);
- channel.abort();
- } finally {
- close();
- }
- }
-
- private long read(long offset, ByteBuffer buffer) throws HyracksDataException {
- return datasetMemoryManager != null ? resultState.read(datasetMemoryManager, offset, buffer)
- : resultState.read(offset, buffer);
- }
-
- private void close() {
- try {
- channel.close();
- resultState.readClose();
- if (resultState.isExhausted()) {
- final ResultSetPartitionId partitionId = resultState.getResultSetPartitionId();
- datasetPartitionManager.removePartition(partitionId.getJobId(), partitionId.getResultSetId(),
- partitionId.getPartition());
- }
- } catch (HyracksDataException e) {
- LOGGER.error("unexpected failure in partition reader clean up", e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java
deleted file mode 100644
index b593bb5..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.dataset;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.api.comm.IFrameWriter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.io.IWorkspaceFileFactory;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.partitions.ResultSetPartitionId;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class DatasetPartitionWriter implements IFrameWriter {
- private static final Logger LOGGER = LogManager.getLogger();
-
- private final IDatasetPartitionManager manager;
-
- private final JobId jobId;
-
- private final ResultSetId resultSetId;
-
- private final boolean orderedResult;
-
- private final int partition;
-
- private final int nPartitions;
-
- private final DatasetMemoryManager datasetMemoryManager;
-
- private final ResultSetPartitionId resultSetPartitionId;
-
- private final ResultState resultState;
-
- private boolean partitionRegistered;
-
- private boolean failed = false;
-
- public DatasetPartitionWriter(IHyracksTaskContext ctx, IDatasetPartitionManager manager, JobId jobId,
- ResultSetId rsId, boolean asyncMode, boolean orderedResult, int partition, int nPartitions,
- DatasetMemoryManager datasetMemoryManager, IWorkspaceFileFactory fileFactory, long maxReads) {
- this.manager = manager;
- this.jobId = jobId;
- this.resultSetId = rsId;
- this.orderedResult = orderedResult;
- this.partition = partition;
- this.nPartitions = nPartitions;
- this.datasetMemoryManager = datasetMemoryManager;
-
- resultSetPartitionId = new ResultSetPartitionId(jobId, rsId, partition);
- resultState = new ResultState(resultSetPartitionId, asyncMode, ctx.getIoManager(), fileFactory,
- ctx.getInitialFrameSize(), maxReads);
- }
-
- public ResultState getResultState() {
- return resultState;
- }
-
- @Override
- public void open() {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("open(" + partition + ")");
- }
- partitionRegistered = false;
- resultState.open();
- }
-
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- registerResultPartitionLocation(false);
- if (datasetMemoryManager == null) {
- resultState.write(buffer);
- } else {
- resultState.write(datasetMemoryManager, buffer);
- }
- }
-
- @Override
- public void fail() throws HyracksDataException {
- failed = true;
- resultState.closeAndDelete();
- resultState.abort();
- }
-
- @Override
- public void close() throws HyracksDataException {
- if (LOGGER.isInfoEnabled()) {
- LOGGER.info("close(" + partition + ")");
- }
- try {
- if (!failed) {
- registerResultPartitionLocation(true);
- }
- } finally {
- resultState.close();
- }
- try {
- if (partitionRegistered) {
- manager.reportPartitionWriteCompletion(jobId, resultSetId, partition);
- }
- } catch (HyracksException e) {
- throw HyracksDataException.create(e);
- }
- }
-
- void registerResultPartitionLocation(boolean empty) throws HyracksDataException {
- try {
- if (!partitionRegistered) {
- manager.registerResultPartitionLocation(jobId, resultSetId, partition, nPartitions, orderedResult,
- empty);
- partitionRegistered = true;
- }
- } catch (HyracksException e) {
- if (e instanceof HyracksDataException) {
- throw (HyracksDataException) e;
- } else {
- throw HyracksDataException.create(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/Page.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/Page.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/Page.java
deleted file mode 100644
index 2eb33fd..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/Page.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.dataset;
-
-import java.nio.ByteBuffer;
-
-public class Page {
- private final ByteBuffer buffer;
-
- public Page(ByteBuffer buffer) {
- this.buffer = buffer;
- }
-
- public ByteBuffer getBuffer() {
- return buffer;
- }
-
- public ByteBuffer clear() {
- return (ByteBuffer) buffer.clear();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultSetMap.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultSetMap.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultSetMap.java
deleted file mode 100644
index 1a64a5a..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultSetMap.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.dataset;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hyracks.api.dataset.IDatasetStateRecord;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-class ResultSetMap implements IDatasetStateRecord, Serializable {
- private static final long serialVersionUID = 1L;
-
- private static final Logger LOGGER = LogManager.getLogger();
-
- private final long timestamp;
- private final HashMap<ResultSetId, ResultState[]> resultStateMap;
-
- ResultSetMap() {
- timestamp = System.nanoTime();
- resultStateMap = new HashMap<>();
- }
-
- @Override
- public long getTimestamp() {
- return timestamp;
- }
-
- ResultState[] getResultStates(ResultSetId rsId) {
- return resultStateMap.get(rsId);
- }
-
- ResultState[] createOrGetResultStates(ResultSetId rsId, int nPartitions) {
- return resultStateMap.computeIfAbsent(rsId, (k) -> new ResultState[nPartitions]);
- }
-
- /**
- * removes a result partition for a result set
- *
- * @param jobId
- * the id of the job that produced the result set
- * @param resultSetId
- * the id of the result set
- * @param partition
- * the partition number
- * @return true, if all partitions for the resultSetId have been removed
- */
- boolean removePartition(JobId jobId, ResultSetId resultSetId, int partition) {
- final ResultState[] resultStates = resultStateMap.get(resultSetId);
- if (resultStates != null) {
- final ResultState state = resultStates[partition];
- if (state != null) {
- state.closeAndDelete();
- LOGGER.debug("Removing partition: " + partition + " for JobId: " + jobId);
- }
- resultStates[partition] = null;
- boolean stateEmpty = true;
- for (ResultState resState : resultStates) {
- if (resState != null) {
- stateEmpty = false;
- break;
- }
- }
- if (stateEmpty) {
- resultStateMap.remove(resultSetId);
- }
- return resultStateMap.isEmpty();
- }
- return true;
- }
-
- void abortAll() {
- applyToAllStates((rsId, state, i) -> state.abort());
- }
-
- void closeAndDeleteAll() {
- applyToAllStates((rsId, state, i) -> {
- state.closeAndDelete();
- LOGGER.debug("Removing partition: " + i + " for result set " + rsId);
- });
- }
-
- @FunctionalInterface
- private interface StateModifier {
- void modify(ResultSetId rsId, ResultState entry, int partition);
- }
-
- private void applyToAllStates(StateModifier modifier) {
- for (Map.Entry<ResultSetId, ResultState[]> entry : resultStateMap.entrySet()) {
- final ResultSetId rsId = entry.getKey();
- final ResultState[] resultStates = entry.getValue();
- if (resultStates == null) {
- continue;
- }
- for (int i = 0; i < resultStates.length; i++) {
- final ResultState state = resultStates[i];
- if (state != null) {
- modifier.modify(rsId, state, i);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultState.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultState.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultState.java
deleted file mode 100644
index 6b35912..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/ResultState.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.dataset;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hyracks.api.dataflow.state.IStateObject;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IFileHandle;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.api.io.IWorkspaceFileFactory;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.api.partitions.ResultSetPartitionId;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-public class ResultState implements IStateObject {
- private static final String FILE_PREFIX = "result_";
-
- private final ResultSetPartitionId resultSetPartitionId;
-
- private final boolean asyncMode;
-
- private final int frameSize;
-
- private final IIOManager ioManager;
-
- private final IWorkspaceFileFactory fileFactory;
-
- private final AtomicBoolean eos;
-
- private final AtomicBoolean failed;
-
- private final List<Page> localPageList;
-
- private FileReference fileRef;
-
- private IFileHandle fileHandle;
-
- private volatile int referenceCount = 0;
-
- private long size;
-
- private long persistentSize;
- private long remainingReads;
-
- ResultState(ResultSetPartitionId resultSetPartitionId, boolean asyncMode, IIOManager ioManager,
- IWorkspaceFileFactory fileFactory, int frameSize, long maxReads) {
- if (maxReads <= 0) {
- throw new IllegalArgumentException("maxReads must be > 0");
- }
- this.resultSetPartitionId = resultSetPartitionId;
- this.asyncMode = asyncMode;
- this.ioManager = ioManager;
- this.fileFactory = fileFactory;
- this.frameSize = frameSize;
- remainingReads = maxReads;
- eos = new AtomicBoolean(false);
- failed = new AtomicBoolean(false);
- localPageList = new ArrayList<>();
-
- fileRef = null;
- fileHandle = null;
- }
-
- public synchronized void open() {
- size = 0;
- persistentSize = 0;
- referenceCount = 0;
- }
-
- public synchronized void close() {
- eos.set(true);
- closeWriteFileHandle();
- notifyAll();
- }
-
- public synchronized void closeAndDelete() {
- // Deleting a job is equivalent to aborting the job for all practical purposes, so the same action, needs
- // to be taken when there are more requests to these result states.
- failed.set(true);
- closeWriteFileHandle();
- if (fileRef != null) {
- fileRef.delete();
- fileRef = null;
- }
- }
-
- private void closeWriteFileHandle() {
- if (fileHandle != null) {
- doCloseFileHandle();
- }
- }
-
- private void doCloseFileHandle() {
- if (--referenceCount == 0) {
- // close the file if there is no more reference
- try {
- ioManager.close(fileHandle);
- } catch (IOException e) {
- // Since file handle could not be closed, just ignore.
- }
- fileHandle = null;
- }
- }
-
- public synchronized void write(ByteBuffer buffer) throws HyracksDataException {
- if (fileRef == null) {
- initWriteFileHandle();
- }
-
- size += ioManager.syncWrite(fileHandle, size, buffer);
- notifyAll();
- }
-
- public synchronized void write(DatasetMemoryManager datasetMemoryManager, ByteBuffer buffer)
- throws HyracksDataException {
- int srcOffset = 0;
- Page destPage = null;
-
- if (!localPageList.isEmpty()) {
- destPage = localPageList.get(localPageList.size() - 1);
- }
-
- while (srcOffset < buffer.limit()) {
- if ((destPage == null) || (destPage.getBuffer().remaining() <= 0)) {
- destPage = datasetMemoryManager.requestPage(resultSetPartitionId, this);
- localPageList.add(destPage);
- }
- int srcLength = Math.min(buffer.limit() - srcOffset, destPage.getBuffer().remaining());
- destPage.getBuffer().put(buffer.array(), srcOffset, srcLength);
- srcOffset += srcLength;
- size += srcLength;
- }
-
- notifyAll();
- }
-
- public synchronized void readOpen() {
- if (isExhausted()) {
- throw new IllegalStateException("Result reads exhausted");
- }
- remainingReads--;
- }
-
- public synchronized void readClose() throws HyracksDataException {
- if (fileHandle != null) {
- doCloseFileHandle();
- }
- }
-
- public synchronized long read(long offset, ByteBuffer buffer) throws HyracksDataException {
- long readSize = 0;
-
- while (offset >= size && !eos.get() && !failed.get()) {
- try {
- wait();
- } catch (InterruptedException e) {
- throw HyracksDataException.create(e);
- }
- }
- if ((offset >= size && eos.get()) || failed.get()) {
- return readSize;
- }
-
- if (fileHandle == null) {
- initReadFileHandle();
- }
- readSize = ioManager.syncRead(fileHandle, offset, buffer);
-
- return readSize;
- }
-
- public synchronized long read(DatasetMemoryManager datasetMemoryManager, long offset, ByteBuffer buffer)
- throws HyracksDataException {
- long readSize = 0;
- while (offset >= size && !eos.get() && !failed.get()) {
- try {
- wait();
- } catch (InterruptedException e) {
- throw HyracksDataException.create(e);
- }
- }
-
- if ((offset >= size && eos.get()) || failed.get()) {
- return readSize;
- }
-
- if (offset < persistentSize) {
- if (fileHandle == null) {
- initReadFileHandle();
- }
- readSize = ioManager.syncRead(fileHandle, offset, buffer);
- if (readSize < 0) {
- throw new HyracksDataException("Premature end of file");
- }
- }
-
- if (readSize < buffer.capacity()) {
- long localPageOffset = offset - persistentSize;
- int localPageIndex = (int) (localPageOffset / DatasetMemoryManager.getPageSize());
- int pageOffset = (int) (localPageOffset % DatasetMemoryManager.getPageSize());
- Page page = getPage(localPageIndex);
- if (page == null) {
- return readSize;
- }
- readSize += buffer.remaining();
- buffer.put(page.getBuffer().array(), pageOffset, buffer.remaining());
- }
- datasetMemoryManager.pageReferenced(resultSetPartitionId);
- return readSize;
- }
-
- public synchronized void abort() {
- failed.set(true);
- notifyAll();
- }
-
- public synchronized Page returnPage() throws HyracksDataException {
- Page page = removePage();
-
- // If we do not have any pages to be given back close the write channel since we don't write any more, return null.
- if (page == null) {
- ioManager.close(fileHandle);
- return null;
- }
-
- page.getBuffer().flip();
-
- if (fileRef == null) {
- initWriteFileHandle();
- }
-
- long delta = ioManager.syncWrite(fileHandle, persistentSize, page.getBuffer());
- persistentSize += delta;
- return page;
- }
-
- public synchronized void setEOS(boolean eos) {
- this.eos.set(eos);
- }
-
- public ResultSetPartitionId getResultSetPartitionId() {
- return resultSetPartitionId;
- }
-
- public int getFrameSize() {
- return frameSize;
- }
-
- public IIOManager getIOManager() {
- return ioManager;
- }
-
- public boolean getAsyncMode() {
- return asyncMode;
- }
-
- @Override
- public JobId getJobId() {
- return resultSetPartitionId.getJobId();
- }
-
- @Override
- public Object getId() {
- return resultSetPartitionId;
- }
-
- @Override
- public long getMemoryOccupancy() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void toBytes(DataOutput out) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void fromBytes(DataInput in) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- private Page getPage(int index) {
- Page page = null;
- if (!localPageList.isEmpty()) {
- page = localPageList.get(index);
- }
- return page;
- }
-
- private Page removePage() {
- Page page = null;
- if (!localPageList.isEmpty()) {
- page = localPageList.remove(localPageList.size() - 1);
- }
- return page;
- }
-
- private void initWriteFileHandle() throws HyracksDataException {
- if (fileHandle == null) {
- String fName = FILE_PREFIX + String.valueOf(resultSetPartitionId.getPartition());
- fileRef = fileFactory.createUnmanagedWorkspaceFile(fName);
- fileHandle = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE,
- IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
- if (referenceCount != 0) {
- throw new IllegalStateException("Illegal reference count " + referenceCount);
- }
- referenceCount = 1;
- notifyAll(); // NOSONAR: always called from a synchronized block
- }
- }
-
- private void initReadFileHandle() throws HyracksDataException {
- while (fileRef == null && !failed.get()) {
- // wait for writer to create the file
- try {
- wait();
- } catch (InterruptedException e) {
- throw HyracksDataException.create(e);
- }
- }
- if (failed.get()) {
- return;
- }
- if (fileHandle == null) {
- // fileHandle has been closed by the writer, create it again
- fileHandle = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_ONLY,
- IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
- }
- referenceCount++;
- }
-
- @Override
- public String toString() {
- try {
- ObjectMapper om = new ObjectMapper();
- ObjectNode on = om.createObjectNode();
- on.put("rspid", resultSetPartitionId.toString());
- on.put("async", asyncMode);
- on.put("remainingReads", remainingReads);
- on.put("eos", eos.get());
- on.put("failed", failed.get());
- on.put("fileRef", String.valueOf(fileRef));
- return om.writer(new MinimalPrettyPrinter()).writeValueAsString(on);
- } catch (JsonProcessingException e) { // NOSONAR
- return e.getMessage();
- }
- }
-
- public synchronized boolean isExhausted() {
- return remainingReads == 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
index dfa5ff3..9bfd061 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
@@ -85,11 +85,11 @@ public class HeartbeatComputeTask extends TimerTask {
hbData.netSignalingBytesRead = netPC.getSignalingBytesRead();
hbData.netSignalingBytesWritten = netPC.getSignalingBytesWritten();
- MuxDemuxPerformanceCounters datasetNetPC = ncs.getDatasetNetworkManager().getPerformanceCounters();
- hbData.datasetNetPayloadBytesRead = datasetNetPC.getPayloadBytesRead();
- hbData.datasetNetPayloadBytesWritten = datasetNetPC.getPayloadBytesWritten();
- hbData.datasetNetSignalingBytesRead = datasetNetPC.getSignalingBytesRead();
- hbData.datasetNetSignalingBytesWritten = datasetNetPC.getSignalingBytesWritten();
+ MuxDemuxPerformanceCounters resultNetPC = ncs.getResultNetworkManager().getPerformanceCounters();
+ hbData.resultNetPayloadBytesRead = resultNetPC.getPayloadBytesRead();
+ hbData.resultNetPayloadBytesWritten = resultNetPC.getPayloadBytesWritten();
+ hbData.resultNetSignalingBytesRead = resultNetPC.getSignalingBytesRead();
+ hbData.resultNetSignalingBytesWritten = resultNetPC.getSignalingBytesWritten();
IPCPerformanceCounters ipcPC = ncs.getIpcSystem().getPerformanceCounters();
hbData.ipcMessagesSent = ipcPC.getMessageSentCount();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/DatasetNetworkManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/DatasetNetworkManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/DatasetNetworkManager.java
deleted file mode 100644
index 5eba281..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/DatasetNetworkManager.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.nc.net;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.SocketAddress;
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.api.comm.IChannelInterfaceFactory;
-import org.apache.hyracks.api.comm.ICloseableBufferAcceptor;
-import org.apache.hyracks.api.comm.NetworkAddress;
-import org.apache.hyracks.api.dataset.IDatasetPartitionManager;
-import org.apache.hyracks.api.dataset.ResultSetId;
-import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.exceptions.NetException;
-import org.apache.hyracks.api.job.JobId;
-import org.apache.hyracks.comm.channels.IChannelConnectionFactory;
-import org.apache.hyracks.comm.channels.NetworkOutputChannel;
-import org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock;
-import org.apache.hyracks.net.protocols.muxdemux.IChannelOpenListener;
-import org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection;
-import org.apache.hyracks.net.protocols.muxdemux.MuxDemux;
-import org.apache.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class DatasetNetworkManager implements IChannelConnectionFactory {
- private static final Logger LOGGER = LogManager.getLogger();
-
- private static final int MAX_CONNECTION_ATTEMPTS = 5;
-
- static final int INITIAL_MESSAGE_SIZE = 20;
-
- private final IDatasetPartitionManager partitionManager;
-
- private final MuxDemux md;
-
- private final int nBuffers;
-
- private NetworkAddress localNetworkAddress;
-
- private NetworkAddress publicNetworkAddress;
-
- /**
- * @param inetAddress
- * - Internet address to bind the listen port to
- * @param inetPort
- * - Port to bind on inetAddress
- * @param publicInetAddress
- * - Internet address to report to consumers;
- * useful when behind NAT. null = same as inetAddress
- * @param publicInetPort
- * - Port to report to consumers; useful when
- * behind NAT. Ignored if publicInetAddress is null. 0 = same as inetPort
- */
- public DatasetNetworkManager(String inetAddress, int inetPort, IDatasetPartitionManager partitionManager,
- int nThreads, int nBuffers, String publicInetAddress, int publicInetPort,
- IChannelInterfaceFactory channelInterfaceFactory) {
- this.partitionManager = partitionManager;
- this.nBuffers = nBuffers;
- md = new MuxDemux(new InetSocketAddress(inetAddress, inetPort), new ChannelOpenListener(), nThreads,
- MAX_CONNECTION_ATTEMPTS, channelInterfaceFactory);
- // Just save these values for the moment; may be reset in start()
- publicNetworkAddress = new NetworkAddress(publicInetAddress, publicInetPort);
- }
-
- public void start() throws IOException {
- md.start();
- InetSocketAddress sockAddr = md.getLocalAddress();
- localNetworkAddress = new NetworkAddress(sockAddr.getHostString(), sockAddr.getPort());
-
- // See if the public address was explicitly specified, and if not,
- // make it a copy of localNetworkAddress
- if (publicNetworkAddress.getAddress() == null) {
- publicNetworkAddress = localNetworkAddress;
- } else {
- // Likewise for public port
- if (publicNetworkAddress.getPort() == 0) {
- publicNetworkAddress = new NetworkAddress(publicNetworkAddress.getAddress(), sockAddr.getPort());
- }
- }
- }
-
- public NetworkAddress getLocalNetworkAddress() {
- return localNetworkAddress;
- }
-
- public NetworkAddress getPublicNetworkAddress() {
- return publicNetworkAddress;
- }
-
- public void stop() {
-
- }
-
- public ChannelControlBlock connect(SocketAddress remoteAddress) throws InterruptedException, NetException {
- MultiplexedConnection mConn = md.connect((InetSocketAddress) remoteAddress);
- return mConn.openChannel();
- }
-
- private class ChannelOpenListener implements IChannelOpenListener {
- @Override
- public void channelOpened(ChannelControlBlock channel) {
- channel.getReadInterface().setFullBufferAcceptor(new InitialBufferAcceptor(channel));
- channel.getReadInterface().getEmptyBufferAcceptor().accept(ByteBuffer.allocate(INITIAL_MESSAGE_SIZE));
- }
- }
-
- private class InitialBufferAcceptor implements ICloseableBufferAcceptor {
- private final ChannelControlBlock ccb;
-
- private NetworkOutputChannel noc;
-
- public InitialBufferAcceptor(ChannelControlBlock ccb) {
- this.ccb = ccb;
- }
-
- @Override
- public void accept(ByteBuffer buffer) {
- JobId jobId = new JobId(buffer.getLong());
- ResultSetId rsId = new ResultSetId(buffer.getLong());
- int partition = buffer.getInt();
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("Received initial dataset partition read request for JobId: " + jobId + " partition: "
- + partition + " on channel: " + ccb);
- }
- noc = new NetworkOutputChannel(ccb, nBuffers);
- try {
- partitionManager.initializeDatasetPartitionReader(jobId, rsId, partition, noc);
- } catch (HyracksException e) {
- noc.abort();
- }
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public void error(int ecode) {
- if (noc != null) {
- noc.abort();
- }
- }
- }
-
- public MuxDemuxPerformanceCounters getPerformanceCounters() {
- return md.getPerformanceCounters();
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/ResultNetworkManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/ResultNetworkManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/ResultNetworkManager.java
new file mode 100644
index 0000000..e56bfe6
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/net/ResultNetworkManager.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.net;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.comm.IChannelInterfaceFactory;
+import org.apache.hyracks.api.comm.ICloseableBufferAcceptor;
+import org.apache.hyracks.api.comm.NetworkAddress;
+import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.exceptions.NetException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.IResultPartitionManager;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.comm.channels.IChannelConnectionFactory;
+import org.apache.hyracks.comm.channels.NetworkOutputChannel;
+import org.apache.hyracks.net.protocols.muxdemux.ChannelControlBlock;
+import org.apache.hyracks.net.protocols.muxdemux.IChannelOpenListener;
+import org.apache.hyracks.net.protocols.muxdemux.MultiplexedConnection;
+import org.apache.hyracks.net.protocols.muxdemux.MuxDemux;
+import org.apache.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ResultNetworkManager implements IChannelConnectionFactory {
+ private static final Logger LOGGER = LogManager.getLogger();
+
+ private static final int MAX_CONNECTION_ATTEMPTS = 5;
+
+ static final int INITIAL_MESSAGE_SIZE = 20;
+
+ private final IResultPartitionManager partitionManager;
+
+ private final MuxDemux md;
+
+ private final int nBuffers;
+
+ private NetworkAddress localNetworkAddress;
+
+ private NetworkAddress publicNetworkAddress;
+
+ /**
+ * @param inetAddress
+ * - Internet address to bind the listen port to
+ * @param inetPort
+ * - Port to bind on inetAddress
+ * @param publicInetAddress
+ * - Internet address to report to consumers;
+ * useful when behind NAT. null = same as inetAddress
+ * @param publicInetPort
+ * - Port to report to consumers; useful when
+ * behind NAT. Ignored if publicInetAddress is null. 0 = same as inetPort
+ */
+ public ResultNetworkManager(String inetAddress, int inetPort, IResultPartitionManager partitionManager,
+ int nThreads, int nBuffers, String publicInetAddress, int publicInetPort,
+ IChannelInterfaceFactory channelInterfaceFactory) {
+ this.partitionManager = partitionManager;
+ this.nBuffers = nBuffers;
+ md = new MuxDemux(new InetSocketAddress(inetAddress, inetPort), new ChannelOpenListener(), nThreads,
+ MAX_CONNECTION_ATTEMPTS, channelInterfaceFactory);
+ // Just save these values for the moment; may be reset in start()
+ publicNetworkAddress = new NetworkAddress(publicInetAddress, publicInetPort);
+ }
+
+ public void start() throws IOException {
+ md.start();
+ InetSocketAddress sockAddr = md.getLocalAddress();
+ localNetworkAddress = new NetworkAddress(sockAddr.getHostString(), sockAddr.getPort());
+
+ // See if the public address was explicitly specified, and if not,
+ // make it a copy of localNetworkAddress
+ if (publicNetworkAddress.getAddress() == null) {
+ publicNetworkAddress = localNetworkAddress;
+ } else {
+ // Likewise for public port
+ if (publicNetworkAddress.getPort() == 0) {
+ publicNetworkAddress = new NetworkAddress(publicNetworkAddress.getAddress(), sockAddr.getPort());
+ }
+ }
+ }
+
+ public NetworkAddress getLocalNetworkAddress() {
+ return localNetworkAddress;
+ }
+
+ public NetworkAddress getPublicNetworkAddress() {
+ return publicNetworkAddress;
+ }
+
+ public void stop() {
+
+ }
+
+ public ChannelControlBlock connect(SocketAddress remoteAddress) throws InterruptedException, NetException {
+ MultiplexedConnection mConn = md.connect((InetSocketAddress) remoteAddress);
+ return mConn.openChannel();
+ }
+
+ private class ChannelOpenListener implements IChannelOpenListener {
+ @Override
+ public void channelOpened(ChannelControlBlock channel) {
+ channel.getReadInterface().setFullBufferAcceptor(new InitialBufferAcceptor(channel));
+ channel.getReadInterface().getEmptyBufferAcceptor().accept(ByteBuffer.allocate(INITIAL_MESSAGE_SIZE));
+ }
+ }
+
+ private class InitialBufferAcceptor implements ICloseableBufferAcceptor {
+ private final ChannelControlBlock ccb;
+
+ private NetworkOutputChannel noc;
+
+ public InitialBufferAcceptor(ChannelControlBlock ccb) {
+ this.ccb = ccb;
+ }
+
+ @Override
+ public void accept(ByteBuffer buffer) {
+ JobId jobId = new JobId(buffer.getLong());
+ ResultSetId rsId = new ResultSetId(buffer.getLong());
+ int partition = buffer.getInt();
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Received initial result partition read request for JobId: " + jobId + " partition: "
+ + partition + " on channel: " + ccb);
+ }
+ noc = new NetworkOutputChannel(ccb, nBuffers);
+ try {
+ partitionManager.initializeResultPartitionReader(jobId, rsId, partition, noc);
+ } catch (HyracksException e) {
+ LOGGER.warn("Failed to initialize result partition reader", e);
+ noc.abort();
+ }
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public void error(int ecode) {
+ if (noc != null) {
+ noc.abort();
+ }
+ }
+ }
+
+ public MuxDemuxPerformanceCounters getPerformanceCounters() {
+ return md.getPerformanceCounters();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/Page.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/Page.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/Page.java
new file mode 100644
index 0000000..47230a6
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/Page.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.result;
+
+import java.nio.ByteBuffer;
+
+public class Page {
+ private final ByteBuffer buffer;
+
+ public Page(ByteBuffer buffer) {
+ this.buffer = buffer;
+ }
+
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+
+ public ByteBuffer clear() {
+ return (ByteBuffer) buffer.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/1d2f00d8/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultMemoryManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultMemoryManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultMemoryManager.java
new file mode 100644
index 0000000..34c9eb5
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultMemoryManager.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.control.nc.result;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.partitions.ResultSetPartitionId;
+
+public class ResultMemoryManager {
+ private int availableMemory;
+
+ private final Set<Page> availPages;
+
+ private final LeastRecentlyUsedList leastRecentlyUsedList;
+
+ private final Map<ResultSetPartitionId, PartitionNode> resultPartitionNodesMap;
+
+ private final static int FRAME_SIZE = 32768;
+
+ public ResultMemoryManager(int availableMemory) {
+ this.availableMemory = availableMemory;
+
+ availPages = new HashSet<Page>();
+
+ // Atleast have one page for temporarily storing the results.
+ if (this.availableMemory <= FRAME_SIZE) {
+ this.availableMemory = FRAME_SIZE;
+ }
+
+ leastRecentlyUsedList = new LeastRecentlyUsedList();
+ resultPartitionNodesMap = new HashMap<ResultSetPartitionId, PartitionNode>();
+ }
+
+ public synchronized Page requestPage(ResultSetPartitionId resultSetPartitionId, ResultState resultState)
+ throws HyracksDataException {
+ Page page;
+ if (availPages.isEmpty()) {
+ if (availableMemory >= FRAME_SIZE) {
+ /* TODO(madhusudancs): Should we have some way of accounting this memory usage by using Hyrack's
+ * allocateFrame() instead of direct ByteBuffer.allocate()?
+ */
+ availPages.add(new Page(ByteBuffer.allocate(FRAME_SIZE)));
+ availableMemory -= FRAME_SIZE;
+ page = getAvailablePage();
+ } else {
+ page = evictPage();
+ }
+ } else {
+ page = getAvailablePage();
+ }
+
+ page.clear();
+
+ /*
+ * It is extremely important to update the reference after obtaining the page because, in the cases where
+ * memory manager is allocated only one page of memory, the front of the LRU list should not be created by the
+ * update reference call before a page is pushed on to the element of the LRU list. So we first obtain the
+ * page, then make a updateReference call which in turn creates a new node in the LRU list and then add the
+ * page to it.
+ */
+ PartitionNode pn = updateReference(resultSetPartitionId, resultState);
+ pn.add(page);
+ return page;
+ }
+
+ public void pageReferenced(ResultSetPartitionId resultSetPartitionId) {
+ // When a page is referenced the result partition writer should already be known, so we pass null.
+ updateReference(resultSetPartitionId, null);
+ }
+
+ public static int getPageSize() {
+ return FRAME_SIZE;
+ }
+
+ protected void insertPartitionNode(ResultSetPartitionId resultSetPartitionId, PartitionNode pn) {
+ leastRecentlyUsedList.add(pn);
+ resultPartitionNodesMap.put(resultSetPartitionId, pn);
+ }
+
+ protected PartitionNode updateReference(ResultSetPartitionId resultSetPartitionId, ResultState resultState) {
+ PartitionNode pn = null;
+
+ if (!resultPartitionNodesMap.containsKey(resultSetPartitionId)) {
+ if (resultState != null) {
+ pn = new PartitionNode(resultSetPartitionId, resultState);
+ insertPartitionNode(resultSetPartitionId, pn);
+ }
+ return pn;
+ }
+ synchronized (this) {
+ pn = resultPartitionNodesMap.get(resultSetPartitionId);
+ leastRecentlyUsedList.remove(pn);
+ insertPartitionNode(resultSetPartitionId, pn);
+ }
+
+ return pn;
+ }
+
+ protected Page evictPage() throws HyracksDataException {
+ PartitionNode pn = leastRecentlyUsedList.getFirst();
+ ResultState resultState = pn.getResultState();
+ Page page = resultState.returnPage();
+
+ /* If the partition holding the pages breaks the contract by not returning the page or it has no page, just
+ * take away all the pages allocated to it and add to the available pages set.
+ */
+ if (page == null) {
+ availPages.addAll(pn);
+ pn.clear();
+ resultPartitionNodesMap.remove(pn.getResultSetPartitionId());
+ leastRecentlyUsedList.remove(pn);
+
+ /* Based on the assumption that if the result partition writer returned a null page, it should be lying
+ * about the number of pages it holds in which case we just evict all the pages it holds and should thus be
+ * able to add all those pages to available set and we have at least one page to allocate back.
+ */
+ page = getAvailablePage();
+ } else {
+ pn.remove(page);
+
+ // If the partition no more holds any pages, remove it from the linked list and the hash map.
+ if (pn.isEmpty()) {
+ resultPartitionNodesMap.remove(pn.getResultSetPartitionId());
+ leastRecentlyUsedList.remove(pn);
+ }
+ }
+
+ return page;
+ }
+
+ protected Page getAvailablePage() {
+ Iterator<Page> iter = availPages.iterator();
+ Page page = iter.next();
+ iter.remove();
+ return page;
+ }
+
+ private class LeastRecentlyUsedList {
+ private PartitionNode head;
+
+ private PartitionNode tail;
+
+ public LeastRecentlyUsedList() {
+ head = null;
+ tail = null;
+ }
+
+ public void add(PartitionNode node) {
+ if (head == null) {
+ head = tail = node;
+ return;
+ }
+ tail.setNext(node);
+ node.setPrev(tail);
+ tail = node;
+ }
+
+ public void remove(PartitionNode node) {
+ if ((node == head) && (node == tail)) {
+ head = tail = null;
+ return;
+ } else if (node == head) {
+ head = head.getNext();
+ head.setPrev(null);
+ return;
+ } else if (node == tail) {
+ tail = tail.getPrev();
+ tail.setNext(null);
+ return;
+ } else {
+ PartitionNode prev = node.getPrev();
+ PartitionNode next = node.getNext();
+ prev.setNext(next);
+ next.setPrev(prev);
+ }
+ }
+
+ public PartitionNode getFirst() {
+ return head;
+ }
+ }
+
+ private class PartitionNode extends HashSet<Page> {
+ private static final long serialVersionUID = 1L;
+
+ private final ResultSetPartitionId resultSetPartitionId;
+
+ private final ResultState resultState;
+
+ private PartitionNode prev;
+
+ private PartitionNode next;
+
+ public PartitionNode(ResultSetPartitionId resultSetPartitionId, ResultState resultState) {
+ this.resultSetPartitionId = resultSetPartitionId;
+ this.resultState = resultState;
+ prev = null;
+ next = null;
+ }
+
+ public ResultSetPartitionId getResultSetPartitionId() {
+ return resultSetPartitionId;
+ }
+
+ public ResultState getResultState() {
+ return resultState;
+ }
+
+ public void setPrev(PartitionNode node) {
+ prev = node;
+ }
+
+ public PartitionNode getPrev() {
+ return prev;
+ }
+
+ public void setNext(PartitionNode node) {
+ next = node;
+ }
+
+ public PartitionNode getNext() {
+ return next;
+ }
+ }
+}