You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2019/12/30 04:50:56 UTC
[asterixdb] 01/06: [ASTERIXDB-2686][RT] Delete run files eagerly
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 66aaf4151cba49a6fdb2c535911f6b75c7083ce3
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Tue Dec 10 00:39:06 2019 -0800
[ASTERIXDB-2686][RT] Delete run files eagerly
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Delete run files eagerly.
Change-Id: Idc780b44b0554bf36e2378700aae1b38b8e57353
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4403
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../join-ASTERIXDB-2686.1.ddl.sqlpp | 16 ++++++++++------
.../join-ASTERIXDB-2686.2.update.sqlpp | 11 +++++------
.../join-ASTERIXDB-2686.3.query.sqlpp | 10 ++++------
.../join-ASTERIXDB-2686.4.ddl.sqlpp | 9 +--------
.../misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm | 1 +
.../src/test/resources/runtimets/testsuite_sqlpp.xml | 5 +++++
.../apache/hyracks/api/io/IWorkspaceFileFactory.java | 16 ++++++++++++++++
.../ExternalGroupWriteOperatorNodePushable.java | 3 +--
.../OptimizedHybridHashJoinOperatorDescriptor.java | 18 ++++++++++++++++++
.../std/sort/AbstractExternalSortRunMerger.java | 2 +-
10 files changed, 62 insertions(+), 29 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.ddl.sqlpp
similarity index 68%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.ddl.sqlpp
index 9a35bb8..bafc59a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.ddl.sqlpp
@@ -16,12 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.api.io;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+/*
+ * Description : Test that temporary run files generated are deleted as soon as possible and query runs fine
+ * Expected Result : Success
+ */
-public interface IWorkspaceFileFactory {
- public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException;
+drop dataverse test if exists;
+create dataverse test;
+use test;
- public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException;
-}
+create type openType as open {id: uuid};
+create dataset ds1(openType) primary key id autogenerated;
+create dataset ds2(openType) primary key id autogenerated;
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.2.update.sqlpp
similarity index 70%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.2.update.sqlpp
index 9a35bb8..9a695ff 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.2.update.sqlpp
@@ -16,12 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.api.io;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+use test;
-public interface IWorkspaceFileFactory {
- public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException;
+insert into ds1(from range(0,1000000) as data select value {"f1": 1});
+insert into ds2(from range(0,1000000) as data select value {"f1": 2});
- public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException;
-}
+insert into ds1 {"f1": 3};
+insert into ds2 {"f1": 3};
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.3.query.sqlpp
similarity index 70%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.3.query.sqlpp
index 9a35bb8..f668266 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.3.query.sqlpp
@@ -16,12 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.api.io;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
+use test;
-public interface IWorkspaceFileFactory {
- public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException;
+set `compiler.joinmemory` "180KB";
- public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException;
-}
+from ds1 join ds2 on ds1.f1 = ds2.f1
+select ds1.f1;
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.4.ddl.sqlpp
similarity index 70%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.4.ddl.sqlpp
index 9a35bb8..548e632 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.4.ddl.sqlpp
@@ -16,12 +16,5 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.hyracks.api.io;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public interface IWorkspaceFileFactory {
- public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException;
-
- public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException;
-}
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm
new file mode 100644
index 0000000..fca24df
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/join-ASTERIXDB-2686/join-ASTERIXDB-2686.1.adm
@@ -0,0 +1 @@
+{ "f1": 3 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 929ecd5..817bab6 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3801,6 +3801,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="join-ASTERIXDB-2686">
+ <output-dir compare="Text">join-ASTERIXDB-2686</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="poll-dynamic">
<output-dir compare="Text">poll-dynamic</output-dir>
</compilation-unit>
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
index 9a35bb8..6859ec7 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IWorkspaceFileFactory.java
@@ -18,10 +18,26 @@
*/
package org.apache.hyracks.api.io;
+import org.apache.hyracks.api.context.IHyracksJobletContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IWorkspaceFileFactory {
+
+ /**
+ * Creates an unmanaged file. Unmanaged files are not automatically deleted. The caller has to delete them.
+ *
+ * @param prefix a meaningful string to start the file name with.
+ */
public FileReference createUnmanagedWorkspaceFile(String prefix) throws HyracksDataException;
+ /**
+ * Creates a managed file. Managed files are automatically deleted at a certain point. This depends on which
+ * context the file is created in. Files created in the {@link IHyracksJobletContext job context} will be deleted
+ * as soon as the job is done. Files created in the {@link IHyracksTaskContext task context} will be deleted as
+ * soon as the task is done.
+ *
+ * @param prefix a meaningful string to start the file name with.
+ */
public FileReference createManagedWorkspaceFile(String prefix) throws HyracksDataException;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java
index 999dbac..e8a1b76 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/group/external/ExternalGroupWriteOperatorNodePushable.java
@@ -171,8 +171,7 @@ public class ExternalGroupWriteOperatorNodePushable extends AbstractUnaryOutputS
@Override
public RunFileWriter getRunFileWriter() throws HyracksDataException {
- FileReference newRun = ctx.getJobletContext()
- .createManagedWorkspaceFile(ExternalGroupOperatorDescriptor.class.getSimpleName());
+ FileReference newRun = ctx.createManagedWorkspaceFile(ExternalGroupOperatorDescriptor.class.getSimpleName());
return new RunFileWriter(newRun, ctx.getIoManager());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index e7984d6..6ba03c5 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -442,6 +442,12 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD
if (isLeftOuter && pReader != null) {
appendNullToProbeTuples(pReader);
}
+ if (bReader != null) {
+ bReader.close();
+ }
+ if (pReader != null) {
+ pReader.close();
+ }
continue;
}
int bSize = state.hybridHJ.getBuildPartitionSizeInTup(pid);
@@ -637,6 +643,12 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD
// For the outer join, we don't reverse the role.
appendNullToProbeTuples(rprfw);
}
+ if (rbrfw != null) {
+ rbrfw.close();
+ }
+ if (rprfw != null) {
+ rprfw.close();
+ }
continue;
}
@@ -661,6 +673,12 @@ public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorD
// For the outer join, we don't reverse the role.
appendNullToProbeTuples(rprfw);
}
+ if (rbrfw != null) {
+ rbrfw.close();
+ }
+ if (rprfw != null) {
+ rprfw.close();
+ }
continue;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java
index 4a77b3c..f3e9320 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/sort/AbstractExternalSortRunMerger.java
@@ -142,7 +142,7 @@ public abstract class AbstractExternalSortRunMerger {
} finally {
mergeResultWriter.close();
}
- reader = mergeFileWriter.createReader();
+ reader = mergeFileWriter.createDeleteOnCloseReader();
}
runs.add(reader);