You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by bu...@apache.org on 2017/02/02 18:24:27 UTC
[18/28] asterixdb git commit: Introduce IStorageComponentProvider
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
index 49ffa28..3c9acf4 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/DatasetConfig.java
@@ -42,16 +42,38 @@ public class DatasetConfig {
LENGTH_PARTITIONED_NGRAM_INVIX
}
- public enum ExternalDatasetTransactionState {
- COMMIT, // The committed state <- nothing is required->
- BEGIN, // The state after starting the refresh transaction <- will either abort moving to committed state or move to ready to commit->
- READY_TO_COMMIT // The transaction is ready to commit <- can only move forward to committed state->
- };
+ public enum TransactionState {
+ /**
+ * The committed state <- nothing is required->
+ */
+ COMMIT,
+ /**
+ * The state after starting the refresh transaction
+ * <- will either abort moving to committed state or move to ready to commit->
+ */
+ BEGIN,
+ /**
+ * The transaction is ready to commit <- can only move forward to committed state->
+ */
+ READY_TO_COMMIT
+ }
public enum ExternalFilePendingOp {
- PENDING_NO_OP, // the stored file is part of a committed transaction nothing is required
- PENDING_ADD_OP, // the stored file is part of an ongoing transaction (will be added if transaction succeed)
- PENDING_DROP_OP, // the stored file is part of an ongoing transaction (will be dropped if transaction succeed)
- PENDING_APPEND_OP // the stored file is part of an ongoing transaction (will be updated if transaction succeed)
- };
+ /**
+ * the stored file is part of a committed transaction nothing is required
+ */
+ NO_OP,
+ /**
+ * the stored file is part of an ongoing transaction (will be added if transaction succeed)
+ */
+ ADD_OP,
+ /**
+ * the stored file is part of an ongoing transaction (will be dropped if transaction succeed)
+ */
+ DROP_OP,
+ /**
+ * the stored file is part of an ongoing transaction (will be updated if transaction succeed)
+ */
+ APPEND_OP
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java
new file mode 100644
index 0000000..d454349
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/IStorageComponentProvider.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.context;
+
+import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
+import org.apache.hyracks.algebricks.data.ITypeTraitProvider;
+import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider;
+import org.apache.hyracks.storage.common.IStorageManager;
+
+/**
+ * Responsible for storage components
+ */
+public interface IStorageComponentProvider {
+
+ /**
+ * @return {@link org.apache.asterix.common.context.ITransactionSubsystemProvider} instance
+ */
+ ITransactionSubsystemProvider getTransactionSubsystemProvider();
+
+ /**
+ * @return {@link org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider} instance
+ */
+ ILSMIOOperationSchedulerProvider getIoOperationSchedulerProvider();
+
+ /**
+ * @return the application's root
+ * {@link org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider} instance
+ */
+ IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider();
+
+ /**
+ * @return {@link org.apache.hyracks.storage.common.IStorageManager} instance
+ */
+ IStorageManager getStorageManager();
+
+ /**
+ * @return {@link org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory} instance
+ */
+ IMetadataPageManagerFactory getMetadataPageManagerFactory();
+
+ /**
+ * @return {@link org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory} instance
+ */
+ IPrimitiveValueProviderFactory getPrimitiveValueProviderFactory();
+
+ /**
+ * @return {@link org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider} instance
+ */
+ IBinaryComparatorFactoryProvider getComparatorFactoryProvider();
+
+ /**
+ * @return {@link org.apache.hyracks.algebricks.data.ITypeTraitProvider} instance
+ */
+ ITypeTraitProvider getTypeTraitProvider();
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java
index 586fb4d..1dd1845 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/ITransactionSubsystemProvider.java
@@ -24,6 +24,10 @@ import java.io.Serializable;
import org.apache.asterix.common.transactions.ITransactionSubsystem;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+/**
+ * Provides the system's implementation of {@link ITransactionSubsystem}
+ */
+@FunctionalInterface
public interface ITransactionSubsystemProvider extends Serializable {
- public ITransactionSubsystem getTransactionSubsystem(IHyracksTaskContext ctx);
+ ITransactionSubsystem getTransactionSubsystem(IHyracksTaskContext ctx);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java
index 706c303..cfbad43 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/TransactionSubsystemProvider.java
@@ -30,6 +30,10 @@ import org.apache.hyracks.api.context.IHyracksTaskContext;
*/
public class TransactionSubsystemProvider implements ITransactionSubsystemProvider {
private static final long serialVersionUID = 1L;
+ public static final TransactionSubsystemProvider INSTANCE = new TransactionSubsystemProvider();
+
+ private TransactionSubsystemProvider() {
+ }
@Override
public ITransactionSubsystem getTransactionSubsystem(IHyracksTaskContext ctx) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java
index b9d547f..eb944a2 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/IApplicationContextInfo.java
@@ -18,15 +18,20 @@
*/
package org.apache.asterix.common.dataflow;
-import org.apache.asterix.common.cluster.IGlobalRecoveryMaanger;
+import org.apache.asterix.common.cluster.IGlobalRecoveryManager;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.hyracks.api.application.ICCApplicationContext;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
-import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.apache.hyracks.storage.common.IStorageManager;
/**
- * Provides methods for obtaining the IIndexLifecycleManagerProvider, IStorageManagerInterface and
- * ICCApplicationContext implementation.
+ * Provides methods for obtaining
+ * {@link org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider},
+ * {@link org.apache.hyracks.storage.common.IStorageManager},
+ * {@link org.apache.hyracks.api.application.ICCApplicationContext},
+ * {@link org.apache.asterix.common.cluster.IGlobalRecoveryManager},
+ * and {@link org.apache.asterix.common.library.ILibraryManager}
+ * at the cluster controller side.
*/
public interface IApplicationContextInfo {
@@ -38,26 +43,23 @@ public interface IApplicationContextInfo {
public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider();
/**
- * Returns an instance of the implementation for IStorageManagerInterface.
- *
- * @return IStorageManagerInterface implementation instance
+ * @return an instance which implements {@link org.apache.hyracks.storage.common.IStorageManager}
*/
- public IStorageManagerInterface getStorageManagerInterface();
+ public IStorageManager getStorageManager();
/**
- * Returns an instance of the implementation for ICCApplicationContext.
- *
- * @return ICCApplicationContext implementation instance
+ * @return an instance which implements {@link org.apache.hyracks.api.application.ICCApplicationContext}
*/
public ICCApplicationContext getCCApplicationContext();
/**
- * @return the global recovery manager.
+ * @return the global recovery manager which implements
+ * {@link org.apache.asterix.common.cluster.IGlobalRecoveryManager}
*/
- public IGlobalRecoveryMaanger getGlobalRecoveryManager();
+ public IGlobalRecoveryManager getGlobalRecoveryManager();
/**
- * @return the library manager (at CC side).
+ * @return the library manager which implements {@link org.apache.asterix.common.library.ILibraryManager}
*/
public ILibraryManager getLibraryManager();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java
index 27bb14f..cd40179 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMIndexUtil.java
@@ -21,8 +21,6 @@ package org.apache.asterix.common.dataflow;
import org.apache.asterix.common.ioopcallbacks.AbstractLSMIOOperationCallback;
import org.apache.asterix.common.transactions.ILogManager;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
-import org.apache.hyracks.storage.am.common.freepage.AppendOnlyLinkedMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
@@ -49,8 +47,4 @@ public class LSMIndexUtil {
.getIOOperationCallback();
return ioOpCallback.getComponentFileLSNOffset(lsmComponent, componentFilePath);
}
-
- public static IMetadataPageManagerFactory getMetadataPageManagerFactory() {
- return new AppendOnlyLinkedMetadataPageManagerFactory();
- }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java
index 20912ef..ac878ec 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInsertDeleteOperatorNodePushable.java
@@ -33,7 +33,7 @@ import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import org.apache.hyracks.dataflow.common.util.TaskUtils;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
@@ -82,7 +82,7 @@ public class LSMInsertDeleteOperatorNodePushable extends LSMIndexInsertUpdateDel
try {
if (isPrimary && ctx.getSharedObject() != null) {
PrimaryIndexLogMarkerCallback callback = new PrimaryIndexLogMarkerCallback(lsmIndex);
- TaskUtils.putInSharedMap(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx);
+ TaskUtil.putInSharedMap(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx);
}
writer.open();
modCallback = opDesc.getModificationOpCallbackFactory().createModificationOperationCallback(
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java
index 4a4502a..2ff0617 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMInvertedIndexInsertDeleteOperatorDescriptor.java
@@ -28,14 +28,14 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
-import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexInsertUpdateDeleteOperator;
import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory;
-import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.apache.hyracks.storage.common.IStorageManager;
public class LSMInvertedIndexInsertDeleteOperatorDescriptor extends LSMInvertedIndexInsertUpdateDeleteOperator {
@@ -43,27 +43,27 @@ public class LSMInvertedIndexInsertDeleteOperatorDescriptor extends LSMInvertedI
private final String indexName;
- public LSMInvertedIndexInsertDeleteOperatorDescriptor(IOperatorDescriptorRegistry spec,
- RecordDescriptor recDesc, IStorageManagerInterface storageManager, IFileSplitProvider fileSplitProvider,
+ public LSMInvertedIndexInsertDeleteOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
+ IStorageManager storageManager, IFileSplitProvider fileSplitProvider,
IIndexLifecycleManagerProvider lifecycleManagerProvider, ITypeTraits[] tokenTypeTraits,
IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits,
IBinaryComparatorFactory[] invListComparatorFactories, IBinaryTokenizerFactory tokenizerFactory,
int[] fieldPermutation, IndexOperation op, IIndexDataflowHelperFactory dataflowHelperFactory,
ITupleFilterFactory tupleFilterFactory,
IModificationOperationCallbackFactory modificationOpCallbackFactory, String indexName,
- IMetadataPageManagerFactory metadataPageManagerFactory) {
+ IPageManagerFactory pageManagerFactory) {
super(spec, recDesc, storageManager, fileSplitProvider, lifecycleManagerProvider, tokenTypeTraits,
tokenComparatorFactories, invListsTypeTraits, invListComparatorFactories, tokenizerFactory,
fieldPermutation, op, dataflowHelperFactory, tupleFilterFactory, modificationOpCallbackFactory,
- metadataPageManagerFactory);
+ pageManagerFactory);
this.indexName = indexName;
}
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
- return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation,
- recordDescProvider, op, false);
+ return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation, recordDescProvider, op,
+ false);
}
public String getIndexName() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java
index 0f83dd3..b9e7c23 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/LSMTreeInsertDeleteOperatorDescriptor.java
@@ -29,14 +29,14 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
-import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IModificationOperationCallbackFactory;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMTreeIndexInsertUpdateDeleteOperatorDescriptor;
-import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.apache.hyracks.storage.common.IStorageManager;
public class LSMTreeInsertDeleteOperatorDescriptor extends LSMTreeIndexInsertUpdateDeleteOperatorDescriptor {
@@ -48,19 +48,18 @@ public class LSMTreeInsertDeleteOperatorDescriptor extends LSMTreeIndexInsertUpd
private final String indexName;
public LSMTreeInsertDeleteOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
- IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider,
+ IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider,
IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories, int[] bloomFilterKeyFields, int[] fieldPermutation,
IndexOperation op, IIndexDataflowHelperFactory dataflowHelperFactory,
ITupleFilterFactory tupleFilterFactory, boolean isPrimary, String indexName,
IMissingWriterFactory nullWriterFactory,
IModificationOperationCallbackFactory modificationOpCallbackProvider,
- ISearchOperationCallbackFactory searchOpCallbackProvider,
- IMetadataPageManagerFactory metadataPageManagerFactory) {
+ ISearchOperationCallbackFactory searchOpCallbackProvider, IPageManagerFactory pageManagerFactory) {
super(spec, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits,
comparatorFactories, bloomFilterKeyFields, fieldPermutation, op, dataflowHelperFactory,
tupleFilterFactory, nullWriterFactory, modificationOpCallbackProvider, searchOpCallbackProvider,
- metadataPageManagerFactory);
+ pageManagerFactory);
this.isPrimary = isPrimary;
this.indexName = indexName;
}
@@ -68,8 +67,8 @@ public class LSMTreeInsertDeleteOperatorDescriptor extends LSMTreeIndexInsertUpd
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
- return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation,
- recordDescProvider, op, isPrimary);
+ return new LSMInsertDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation, recordDescProvider, op,
+ isPrimary);
}
public boolean isPrimary() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index fad00a7..22c080a 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -64,6 +64,19 @@ public class ErrorCode {
public static final int COMPILATION_INVALID_PARAMETER_NUMBER = 1008;
public static final int COMPILATION_INVALID_RETURNING_EXPRESSION = 1009;
public static final int COMPILATION_FULLTEXT_PHRASE_FOUND = 1010;
+ public static final int COMPILATION_UNKNOWN_DATASET_TYPE = 1011;
+ public static final int COMPILATION_UNKNOWN_INDEX_TYPE = 1012;
+ public static final int COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD = 1013;
+ public static final int COMPILATION_FIELD_NOT_FOUND = 1014;
+ public static final int COMPILATION_ILLEGAL_INDEX_FOR_DATASET_WITH_COMPOSITE_PRIMARY_INDEX = 1015;
+ public static final int COMPILATION_INDEX_TYPE_NOT_SUPPORTED_FOR_DATASET_TYPE = 1016;
+ public static final int COMPILATION_FILTER_CANNOT_BE_NULLABLE = 1017;
+ public static final int COMPILATION_ILLEGAL_FILTER_TYPE = 1018;
+ public static final int COMPILATION_CANNOT_AUTOGENERATE_COMPOSITE_PRIMARY_KEY = 1019;
+ public static final int COMPILATION_ILLEGAL_AUTOGENERATED_TYPE = 1020;
+ public static final int COMPILATION_PRIMARY_KEY_CANNOT_BE_NULLABLE = 1021;
+ public static final int COMPILATION_ILLEGAL_PRIMARY_KEY_TYPE = 1022;
+ public static final int COMPILATION_CANT_DROP_ACTIVE_DATASET = 1023;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java
index 7d33340..848de29 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMBTreeWithBuddyIOOperationCallbackFactory.java
@@ -25,7 +25,8 @@ public class LSMBTreeWithBuddyIOOperationCallbackFactory implements ILSMIOOperat
private static final long serialVersionUID = 1L;
- public static LSMBTreeWithBuddyIOOperationCallbackFactory INSTANCE = new LSMBTreeWithBuddyIOOperationCallbackFactory();
+ public static final LSMBTreeWithBuddyIOOperationCallbackFactory INSTANCE =
+ new LSMBTreeWithBuddyIOOperationCallbackFactory();
private LSMBTreeWithBuddyIOOperationCallbackFactory() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java
index 3b71027..8951cb4 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMInvertedIndexIOOperationCallbackFactory.java
@@ -26,7 +26,8 @@ public class LSMInvertedIndexIOOperationCallbackFactory implements ILSMIOOperati
private static final long serialVersionUID = 1L;
- public static LSMInvertedIndexIOOperationCallbackFactory INSTANCE = new LSMInvertedIndexIOOperationCallbackFactory();
+ public static final LSMInvertedIndexIOOperationCallbackFactory INSTANCE =
+ new LSMInvertedIndexIOOperationCallbackFactory();
private LSMInvertedIndexIOOperationCallbackFactory() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java
index 6ae833e..954c6e1 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/ioopcallbacks/LSMRTreeIOOperationCallbackFactory.java
@@ -26,7 +26,7 @@ public class LSMRTreeIOOperationCallbackFactory implements ILSMIOOperationCallba
private static final long serialVersionUID = 1L;
- public static LSMRTreeIOOperationCallbackFactory INSTANCE = new LSMRTreeIOOperationCallbackFactory();
+ public static final LSMRTreeIOOperationCallbackFactory INSTANCE = new LSMRTreeIOOperationCallbackFactory();
private LSMRTreeIOOperationCallbackFactory() {
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java
index af056ae..ebad94d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionManager.java
@@ -122,7 +122,7 @@ public interface ITransactionManager {
* @see ITransactionSubsystem
* @return TransactionProvider
*/
- public ITransactionSubsystem getTransactionProvider();
+ public ITransactionSubsystem getTransactionSubsystem();
/**
* @return The current max job id.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java
index b3a3eba..7bd55e8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ITransactionSubsystem.java
@@ -18,19 +18,29 @@
*/
package org.apache.asterix.common.transactions;
+import org.apache.asterix.common.config.TransactionProperties;
+
+/**
+ * A transaction subsystem is responsible of maintaining Atomicity, Consistency, Isolation, and Durability
+ * on a record level.
+ */
public interface ITransactionSubsystem {
- public ILogManager getLogManager();
+ ILogManager getLogManager();
+
+ ILockManager getLockManager();
+
+ ITransactionManager getTransactionManager();
- public ILockManager getLockManager();
+ IRecoveryManager getRecoveryManager();
- public ITransactionManager getTransactionManager();
+ IAppRuntimeContextProvider getAsterixAppRuntimeContextProvider();
- public IRecoveryManager getRecoveryManager();
+ String getId();
- public IAppRuntimeContextProvider getAsterixAppRuntimeContextProvider();
+ ICheckpointManager getCheckpointManager();
- public String getId();
+ TransactionProperties getTransactionProperties();
- public ICheckpointManager getCheckpointManager();
+ void incrementEntityCommitCount();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java
index 0032a70..e53ca7f 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/Resource.java
@@ -21,15 +21,20 @@ package org.apache.asterix.common.transactions;
import java.io.Serializable;
import java.util.List;
+import org.apache.hyracks.api.application.INCApplicationContext;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
import org.apache.hyracks.storage.common.file.LocalResource;
/**
+ * TODO(amoudi): Change this class and its subclasses to use json serialization instead of Java serialization
* The base resource that will be written to disk. it will go in the serializable resource
* member in {@link LocalResource}
*/
@@ -41,14 +46,22 @@ public abstract class Resource implements Serializable {
protected final ITypeTraits[] filterTypeTraits;
protected final IBinaryComparatorFactory[] filterCmpFactories;
protected final int[] filterFields;
+ protected final ILSMOperationTrackerFactory opTrackerProvider;
+ protected final ILSMIOOperationCallbackFactory ioOpCallbackFactory;
+ protected final IMetadataPageManagerFactory metadataPageManagerFactory;
public Resource(int datasetId, int partition, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory) {
this.datasetId = datasetId;
this.partition = partition;
this.filterTypeTraits = filterTypeTraits;
this.filterCmpFactories = filterCmpFactories;
this.filterFields = filterFields;
+ this.opTrackerProvider = opTrackerProvider;
+ this.ioOpCallbackFactory = ioOpCallbackFactory;
+ this.metadataPageManagerFactory = metadataPageManagerFactory;
}
public int partition() {
@@ -59,8 +72,8 @@ public abstract class Resource implements Serializable {
return datasetId;
}
- public abstract ILSMIndex createIndexInstance(IAppRuntimeContextProvider runtimeContextProvider,
- LocalResource resource) throws HyracksDataException;
+ public abstract ILSMIndex createIndexInstance(INCApplicationContext appCtx, LocalResource resource)
+ throws HyracksDataException;
public static int getIoDeviceNum(IIOManager ioManager, IODeviceHandle deviceHandle) {
List<IODeviceHandle> ioDevices = ioManager.getIODevices();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java
index f7f7a71..891edd8 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/ResourceFactory.java
@@ -20,6 +20,9 @@ package org.apache.asterix.common.transactions;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
public abstract class ResourceFactory implements IResourceFactory {
@@ -28,12 +31,20 @@ public abstract class ResourceFactory implements IResourceFactory {
protected final ITypeTraits[] filterTypeTraits;
protected final IBinaryComparatorFactory[] filterCmpFactories;
protected final int[] filterFields;
+ protected final ILSMOperationTrackerFactory opTrackerProvider;
+ protected final ILSMIOOperationCallbackFactory ioOpCallbackFactory;
+ protected final IMetadataPageManagerFactory metadataPageManagerFactory;
public ResourceFactory(int datasetId, ITypeTraits[] filterTypeTraits,
- IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields) {
+ IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields,
+ ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory) {
this.datasetId = datasetId;
this.filterTypeTraits = filterTypeTraits;
this.filterCmpFactories = filterCmpFactories;
this.filterFields = filterFields;
+ this.opTrackerProvider = opTrackerProvider;
+ this.ioOpCallbackFactory = ioOpCallbackFactory;
+ this.metadataPageManagerFactory = metadataPageManagerFactory;
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java
deleted file mode 100644
index 2ce3cca..0000000
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ANameSchema.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.utils;
-
-/*
- * Author: Guangqiang Li
- * Created on Nov 30, 2009
- */
-public class ANameSchema {
- public static final int LOCALNAME_PROPERTY = 0;
- public static final int URI = 1;
- public static final int NODE_TYPE = 2;
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java
new file mode 100644
index 0000000..88de295
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Job.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import org.apache.hyracks.api.job.JobSpecification;
+
+public class Job {
+
+ public enum SubmissionMode {
+ SYNCHRONOUS,
+ ASYNCHRONOUS
+ }
+
+ private final JobSpecification jobSpec;
+ private final SubmissionMode submissionMode;
+
+ public Job(JobSpecification jobSpecification, SubmissionMode submissionMode) {
+ this.jobSpec = jobSpecification;
+ this.submissionMode = submissionMode;
+ }
+
+ public Job(JobSpecification jobSpec) {
+ this.jobSpec = jobSpec;
+ this.submissionMode = SubmissionMode.SYNCHRONOUS;
+ }
+
+ public JobSpecification getJobSpec() {
+ return jobSpec;
+ }
+
+ public SubmissionMode getSubmissionMode() {
+ return submissionMode;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
new file mode 100644
index 0000000..9fb6a31
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/JobUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.common.utils;
+
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.job.JobSpecification;
+
+public class JobUtils {
+
+ public enum ProgressState {
+ NO_PROGRESS,
+ ADDED_PENDINGOP_RECORD_TO_METADATA
+ }
+
+ public static JobId runJob(IHyracksClientConnection hcc, JobSpecification spec, boolean waitForCompletion)
+ throws Exception {
+ JobId[] jobIds = executeJobArray(hcc, new Job[] { new Job(spec) }, waitForCompletion);
+ return jobIds[0];
+ }
+
+ public static JobId[] executeJobArray(IHyracksClientConnection hcc, Job[] jobs, boolean waitForCompletion)
+ throws Exception {
+ JobId[] startedJobIds = new JobId[jobs.length];
+ for (int i = 0; i < jobs.length; i++) {
+ JobSpecification spec = jobs[i].getJobSpec();
+ spec.setMaxReattempts(0);
+ JobId jobId = hcc.startJob(spec);
+ startedJobIds[i] = jobId;
+ if (waitForCompletion) {
+ hcc.waitForCompletion(jobId);
+ }
+ }
+ return startedJobIds;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
index 34af5c3..9f2e3e7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StoragePathUtil.java
@@ -39,6 +39,9 @@ public class StoragePathUtil {
public static final String DATASET_INDEX_NAME_SEPARATOR = "_idx_";
public static final String ADAPTER_INSTANCE_PREFIX = "adapter_";
+ private StoragePathUtil() {
+ }
+
public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraints(
FileSplit[] splits) {
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
@@ -67,7 +70,7 @@ public class StoragePathUtil {
}
private static String prepareFullIndexName(String datasetName, String idxName) {
- return (datasetName + DATASET_INDEX_NAME_SEPARATOR + idxName);
+ return datasetName + DATASET_INDEX_NAME_SEPARATOR + idxName;
}
public static int getPartitionNumFromName(String name) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java
index 2878d5a..4ba35ae 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/TransactionUtil.java
@@ -29,6 +29,11 @@ import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
public class TransactionUtil {
+ public static final boolean PROFILE_MODE = false;
+
+ private TransactionUtil() {
+ }
+
public static void formJobTerminateLogRecord(ITransactionContext txnCtx, LogRecord logRecord, boolean isCommit) {
logRecord.setTxnCtx(txnCtx);
TransactionUtil.formJobTerminateLogRecord(logRecord, txnCtx.getJobId().getId(), isCommit);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index f0e98c9..f1e9836 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -49,6 +49,19 @@
1007 = Invalid expression: function %1$s expects its %2$s input parameter to be a %3$s expression, but the actual expression is %4$s
1008 = Invalid parameter number: function %1$s cannot take %2$s parameters
1010 = Phrase search in Full-text is not yet supported. Only one keyword per expression is permitted
+1011 = Unknown dataset type %1$s
+1012 = Unknown index type %1$s
+1013 = Cannot use %1$s fields as a key for the %2$s index. The index can only support keys of size %3$s
+1014 = Field \"%1$s\" is not found
+1015 = Index of type %1$s is not supported for dataset \"%2$s\" since it has composite primary keys
+1016 = Index of type %1$s is not supported for dataset of type %2$s
+1017 = The filter field \"%1$s\" cannot be an optional field
+1018 = Field of type %1$s cannot be used as a filter field
+1019 = Cannot autogenerate a composite primary key
+1020 = Cannot autogenerate a primary key for primary key of type %1$s. Autogenerated primary keys must be of type %2$s
+1021 = The primary key field \"%1$s\" cannot be nullable
+1022 = Field of type %1$s cannot be used as a primary key field
+1023 = Can't drop dataset %1$s since it is connected to active entity: %2$s
# Feed Errors
3001 = Illegal state.
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
index b35d9c6..e2274b9 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IExternalDataSourceFactory.java
@@ -25,8 +25,8 @@ import java.util.Map;
import java.util.Set;
import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.runtime.util.AppContextInfo;
-import org.apache.asterix.runtime.util.ClusterStateManager;
+import org.apache.asterix.runtime.utils.AppContextInfo;
+import org.apache.asterix.runtime.utils.ClusterStateManager;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 859a011..2e687ba 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -39,7 +39,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.HyracksConstants;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender;
-import org.apache.hyracks.dataflow.common.util.TaskUtils;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import org.apache.log4j.Logger;
public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowController {
@@ -144,7 +144,7 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
ExecutorService executorService = sendMarker ? Executors.newSingleThreadExecutor() : null;
if (sendMarker && dataflowMarker == null) {
dataflowMarker = new DataflowMarker(recordReader.getProgressReporter(),
- TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx));
+ TaskUtil.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx));
dataflowMarkerResult = executorService.submit(dataflowMarker);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
index b8d0532..d31e074 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedTupleForwarder.java
@@ -33,7 +33,7 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
import org.apache.hyracks.dataflow.common.io.MessagingFrameTupleAppender;
-import org.apache.hyracks.dataflow.common.util.TaskUtils;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
public class FeedTupleForwarder implements ITupleForwarder {
@@ -59,7 +59,7 @@ public class FeedTupleForwarder implements ITupleForwarder {
this.writer = writer;
this.appender = new FrameTupleAppender(frame);
// Set null feed message
- VSizeFrame message = TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx);
+ VSizeFrame message = TaskUtil.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx);
// a null message
message.getBuffer().put(MessagingFrameTupleAppender.NULL_FEED_MESSAGE);
message.getBuffer().flip();
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
index 64f0b8d..fa48b58 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/management/FeedEventsListener.java
@@ -45,7 +45,7 @@ import org.apache.asterix.external.operators.FeedCollectOperatorDescriptor;
import org.apache.asterix.external.operators.FeedIntakeOperatorDescriptor;
import org.apache.asterix.external.operators.FeedMetaOperatorDescriptor;
import org.apache.asterix.external.util.FeedUtils.JobType;
-import org.apache.asterix.runtime.util.AppContextInfo;
+import org.apache.asterix.runtime.utils.AppContextInfo;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
index 34237c4..d32a604 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/feed/runtime/IngestionRuntime.java
@@ -29,7 +29,7 @@ import org.apache.hyracks.api.comm.VSizeFrame;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.util.HyracksConstants;
-import org.apache.hyracks.dataflow.common.util.TaskUtils;
+import org.apache.hyracks.dataflow.common.utils.TaskUtil;
public class IngestionRuntime extends SubscribableRuntime {
@@ -50,9 +50,9 @@ public class IngestionRuntime extends SubscribableRuntime {
dWriter.subscribe(collector);
subscribers.add(collectionRuntime);
if (numSubscribers == 0) {
- TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx);
- TaskUtils.putInSharedMap(HyracksConstants.KEY_MESSAGE,
- TaskUtils.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx), collectionRuntime.getCtx());
+ TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, new VSizeFrame(ctx), ctx);
+ TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE,
+ TaskUtil.<VSizeFrame> get(HyracksConstants.KEY_MESSAGE, ctx), collectionRuntime.getCtx());
start();
}
numSubscribers++;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
index 7acb1f8..2aac497 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/ExternalFile.java
@@ -48,7 +48,7 @@ public class ExternalFile implements Serializable, Comparable<ExternalFile> {
this.fileName = "";
this.lastModefiedTime = new Date();
this.size = 0;
- this.pendingOp = ExternalFilePendingOp.PENDING_NO_OP;
+ this.pendingOp = ExternalFilePendingOp.NO_OP;
}
public ExternalFile(String dataverseName, String datasetName, int fileNumber, String fileName,
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java
index 3ac8da8..152fa8b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/FilesIndexDescription.java
@@ -37,12 +37,13 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
@SuppressWarnings("rawtypes")
public class FilesIndexDescription {
- public final static int FILE_INDEX_TUPLE_SIZE = 2;
- public final static int FILE_KEY_INDEX = 0;
- public final static int FILE_KEY_SIZE = 1;
- public final static int FILE_PAYLOAD_INDEX = 1;
- public final static String[] payloadFieldNames = { "FileName", "FileSize", "FileModDate" };
- public final static IAType[] payloadFieldTypes = { BuiltinType.ASTRING, BuiltinType.AINT64, BuiltinType.ADATETIME };
+ public static final int FILE_INDEX_TUPLE_SIZE = 2;
+ public static final int FILE_KEY_INDEX = 0;
+ public static final int FILE_KEY_SIZE = 1;
+ public static final int FILE_PAYLOAD_INDEX = 1;
+ private static final String[] payloadFieldNames = { "FileName", "FileSize", "FileModDate" };
+ private static final IAType[] payloadFieldTypes = { BuiltinType.ASTRING, BuiltinType.AINT64,
+ BuiltinType.ADATETIME };
public static final int[] BLOOM_FILTER_FIELDS = { 0 };
public static final int EXTERNAL_FILE_NAME_FIELD_INDEX = 0;
@@ -58,7 +59,7 @@ public class FilesIndexDescription {
public final RecordDescriptor FILE_BUDDY_BTREE_RECORD_DESCRIPTOR;
public final ISerializerDeserializer[] EXTERNAL_FILE_BUDDY_BTREE_FIELDS = new ISerializerDeserializer[1];
public final ISerializerDeserializer[] EXTERNAL_FILE_TUPLE_FIELDS = new ISerializerDeserializer[FILE_INDEX_TUPLE_SIZE];
- public final IBinaryComparatorFactory[] FILES_INDEX_COMP_FACTORIES = new IBinaryComparatorFactory[] {
+ public static final IBinaryComparatorFactory[] FILES_INDEX_COMP_FACTORIES = new IBinaryComparatorFactory[] {
BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(BuiltinType.AINT32, true) };
public FilesIndexDescription() {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java
index a269144..86f5aa5 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/indexing/IndexingConstants.java
@@ -20,6 +20,7 @@ package org.apache.asterix.external.indexing;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -81,7 +82,7 @@ public class IndexingConstants {
public static final int RECORD_OFFSET_FIELD_INDEX = 1;
public static final int ROW_NUMBER_FIELD_INDEX = 2;
- public static final ArrayList<List<String>> RecordIDFields = new ArrayList<List<String>>();
+ public static final List<List<String>> RECORD_ID_FIELDS;
static {
@@ -101,25 +102,32 @@ public class IndexingConstants {
rowNumberEvalFactory = new TupleFieldEvaluatorFactory(3);
// Add field names
- RecordIDFields.add(new ArrayList<String>(Arrays.asList("FileNumber")));
- RecordIDFields.add(new ArrayList<String>(Arrays.asList("RecordOffset")));
- RecordIDFields.add(new ArrayList<String>(Arrays.asList("RowNumber")));
+ List<List<String>> ridFields = new ArrayList<>();
+ ridFields.add(new ArrayList<>(Arrays.asList("FileNumber")));
+ ridFields.add(new ArrayList<>(Arrays.asList("RecordOffset")));
+ ridFields.add(new ArrayList<>(Arrays.asList("RowNumber")));
+ RECORD_ID_FIELDS = Collections.unmodifiableList(ridFields);
+ }
+
+ private IndexingConstants() {
}
// This function returns the size of the RID for the passed file input format
public static int getRIDSize(String fileInputFormat) {
- if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED))
+ if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) {
return 3;
- else
+ } else {
return 2;
+ }
}
// This function returns the size of the RID for the passed file input format
public static IBinaryComparatorFactory[] getComparatorFactories(String fileInputFormat) {
- if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED))
+ if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) {
return rCFileRIDComparatorFactories;
- else
+ } else {
return txtSeqFileRIDComparatorFactories;
+ }
}
public static IAType getFieldType(int fieldNumber) throws AsterixException {
@@ -197,9 +205,14 @@ public class IndexingConstants {
public static List<List<String>> getRIDKeys(Map<String, String> properties) {
String fileInputFormat = properties.get(KEY_INPUT_FORMAT);
- if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED))
- return RecordIDFields;
- else
- return RecordIDFields.subList(0, ROW_NUMBER_FIELD_INDEX);
+ if (fileInputFormat.equals(INPUT_FORMAT_RC) || fileInputFormat.equals(INPUT_FORMAT_RC_FULLY_QUALIFIED)) {
+ return RECORD_ID_FIELDS;
+ } else {
+ return RECORD_ID_FIELDS.subList(0, ROW_NUMBER_FIELD_INDEX);
+ }
+ }
+
+ public static String getFilesIndexName(String datasetName) {
+ return datasetName.concat(IndexingConstants.EXTERNAL_FILE_INDEX_NAME_SUFFIX);
}
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java
index c2f74e4..ebf1d17 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/RecordWithMetadataAndPK.java
@@ -40,7 +40,7 @@ import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.util.NonTaggedFormatUtil;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java
index 38e5fa3..f948c1c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/converter/DCPMessageToRecordConverter.java
@@ -30,9 +30,9 @@ import org.apache.asterix.external.api.IRawRecord;
import org.apache.asterix.external.input.record.CharArrayRecord;
import org.apache.asterix.external.input.record.RecordWithMetadataAndPK;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.RecordUtil;
import com.couchbase.client.core.message.dcp.DCPRequest;
import com.couchbase.client.core.message.dcp.MutationMessage;
@@ -57,7 +57,7 @@ public class DCPMessageToRecordConverter implements IRecordToRecordWithMetadataA
public DCPMessageToRecordConverter() {
this.value = new CharArrayRecord();
this.recordWithMetadata = new RecordWithMetadataAndPK<>(value, CB_META_TYPES,
- ARecordType.FULLY_OPEN_RECORD_TYPE, PK_INDICATOR, PK_INDEXES, PK_TYPES);
+ RecordUtil.FULLY_OPEN_RECORD_TYPE, PK_INDICATOR, PK_INDEXES, PK_TYPES);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
index aafee8e..5337be1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/stream/factory/SocketServerInputStreamFactory.java
@@ -32,7 +32,7 @@ import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.api.IInputStreamFactory;
import org.apache.asterix.external.input.stream.SocketServerInputStream;
import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.runtime.util.RuntimeUtils;
+import org.apache.asterix.runtime.utils.RuntimeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.utils.Pair;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
index 1b89b0c..c80aebc 100755
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
@@ -32,7 +32,7 @@ import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.EnumDeserializer;
import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.asterix.runtime.util.AppContextInfo;
+import org.apache.asterix.runtime.utils.AppContextInfo;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java
index 95e566b..a107421 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/java/JObjectUtil.java
@@ -56,8 +56,8 @@ import org.apache.asterix.om.types.AUnorderedListType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.EnumDeserializer;
import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.util.NonTaggedFormatUtil;
import org.apache.asterix.om.util.container.IObjectPool;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
public class JObjectUtil {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java
index 8080f81..d9471d1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/AbstractExternalDatasetIndexesOperatorDescriptor.java
@@ -32,9 +32,6 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
-import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory;
-import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
-import org.apache.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory;
// This is an operator that takes a single file index and an array of secondary indexes
// it is intended to be used for
@@ -45,27 +42,20 @@ public abstract class AbstractExternalDatasetIndexesOperatorDescriptor
extends AbstractSingleActivityOperatorDescriptor {
private static final long serialVersionUID = 1L;
- private ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory;
+ private IIndexDataflowHelperFactory filesIndexDataflowHelperFactory;
private IndexInfoOperatorDescriptor fileIndexInfo;
- private List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories;
- private List<IndexInfoOperatorDescriptor> bTreeIndexesInfos;
- private List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories;
- private List<IndexInfoOperatorDescriptor> rTreeIndexesInfos;
+ private List<IIndexDataflowHelperFactory> treeIndexesDataflowHelperFactories;
+ private List<IndexInfoOperatorDescriptor> treeIndexesInfos;
public AbstractExternalDatasetIndexesOperatorDescriptor(IOperatorDescriptorRegistry spec,
- ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory,
- IndexInfoOperatorDescriptor fileIndexesInfo,
- List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories,
- List<IndexInfoOperatorDescriptor> bTreeIndexesInfos,
- List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories,
- List<IndexInfoOperatorDescriptor> rTreeIndexesInfos) {
+ IIndexDataflowHelperFactory filesIndexDataflowHelperFactory, IndexInfoOperatorDescriptor fileIndexesInfo,
+ List<IIndexDataflowHelperFactory> treeIndexesDataflowHelperFactories,
+ List<IndexInfoOperatorDescriptor> indexesInfos) {
super(spec, 0, 0);
this.filesIndexDataflowHelperFactory = filesIndexDataflowHelperFactory;
this.fileIndexInfo = fileIndexesInfo;
- this.bTreeIndexesDataflowHelperFactories = bTreeIndexesDataflowHelperFactories;
- this.bTreeIndexesInfos = bTreeIndexesInfos;
- this.rTreeIndexesDataflowHelperFactories = rTreeIndexesDataflowHelperFactories;
- this.rTreeIndexesInfos = rTreeIndexesInfos;
+ this.treeIndexesDataflowHelperFactories = treeIndexesDataflowHelperFactories;
+ this.treeIndexesInfos = indexesInfos;
}
// opening and closing the index is done inside these methods since we don't always need open indexes
@@ -87,13 +77,8 @@ public abstract class AbstractExternalDatasetIndexesOperatorDescriptor
performOpOnIndex(filesIndexDataflowHelperFactory, ctx, fileIndexInfo, partition);
}
// perform operation on btrees
- for (int i = 0; i < bTreeIndexesDataflowHelperFactories.size(); i++) {
- performOpOnIndex(bTreeIndexesDataflowHelperFactories.get(i), ctx, bTreeIndexesInfos.get(i),
- partition);
- }
- // perform operation on rtrees
- for (int i = 0; i < rTreeIndexesDataflowHelperFactories.size(); i++) {
- performOpOnIndex(rTreeIndexesDataflowHelperFactories.get(i), ctx, rTreeIndexesInfos.get(i),
+ for (int i = 0; i < treeIndexesDataflowHelperFactories.size(); i++) {
+ performOpOnIndex(treeIndexesDataflowHelperFactories.get(i), ctx, treeIndexesInfos.get(i),
partition);
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java
index 5f4e159..ac715a0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalBTreeSearchOperatorDescriptor.java
@@ -30,27 +30,27 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.dataflow.std.file.IFileSplitProvider;
import org.apache.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
-import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.IPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.ISearchOperationCallbackFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
-import org.apache.hyracks.storage.common.IStorageManagerInterface;
+import org.apache.hyracks.storage.common.IStorageManager;
public class ExternalBTreeSearchOperatorDescriptor extends BTreeSearchOperatorDescriptor {
private static final long serialVersionUID = 1L;
public ExternalBTreeSearchOperatorDescriptor(IOperatorDescriptorRegistry spec, RecordDescriptor recDesc,
- IStorageManagerInterface storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider,
+ IStorageManager storageManager, IIndexLifecycleManagerProvider lifecycleManagerProvider,
IFileSplitProvider fileSplitProvider, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories, int[] bloomFilterKeyFields, int[] lowKeyFields,
int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive,
IIndexDataflowHelperFactory dataflowHelperFactory, boolean retainInput, boolean retainMissing,
IMissingWriterFactory missingWriterFactory, ISearchOperationCallbackFactory searchOpCallbackProvider,
- IMetadataPageManagerFactory metadataPageManagerFactory) {
+ IPageManagerFactory pageManagerFactory) {
super(spec, recDesc, storageManager, lifecycleManagerProvider, fileSplitProvider, typeTraits,
comparatorFactories, bloomFilterKeyFields, lowKeyFields, highKeyFields, lowKeyInclusive,
highKeyInclusive, dataflowHelperFactory, retainInput, retainMissing, missingWriterFactory,
- searchOpCallbackProvider, null, null, metadataPageManagerFactory);
+ searchOpCallbackProvider, null, null, pageManagerFactory);
}
@Override
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java
deleted file mode 100644
index aed8bb9..0000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDataScanOperatorDescriptor.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.operators;
-
-import org.apache.asterix.external.api.IAdapterFactory;
-import org.apache.asterix.external.api.IDataSourceAdapter;
-import org.apache.hyracks.api.context.IHyracksTaskContext;
-import org.apache.hyracks.api.dataflow.IOperatorNodePushable;
-import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobSpecification;
-import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
-import org.apache.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
-
-/*
- * A single activity operator that provides the functionality of scanning data using an
- * instance of the configured adapter.
- */
-public class ExternalDataScanOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
-
- private static final long serialVersionUID = 1L;
-
- private IAdapterFactory adapterFactory;
-
- public ExternalDataScanOperatorDescriptor(JobSpecification spec, RecordDescriptor rDesc,
- IAdapterFactory dataSourceAdapterFactory) {
- super(spec, 0, 1);
- recordDescriptors[0] = rDesc;
- this.adapterFactory = dataSourceAdapterFactory;
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
- IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
- throws HyracksDataException {
-
- return new AbstractUnaryOutputSourceOperatorNodePushable() {
-
- @Override
- public void initialize() throws HyracksDataException {
- IDataSourceAdapter adapter = null;
- try {
- writer.open();
- adapter = adapterFactory.createAdapter(ctx, partition);
- adapter.start(partition, writer);
- } catch (Throwable th) {
- writer.fail();
- throw new HyracksDataException(th);
- } finally {
- writer.close();
- }
- }
- };
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ef9be0f9/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java
index 8321074..3ce0da8 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/ExternalDatasetIndexesAbortOperatorDescriptor.java
@@ -25,24 +25,18 @@ import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil;
-import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeDataflowHelperFactory;
-import org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyDataflowHelperFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.AbortRecoverLSMIndexFileManager;
-import org.apache.hyracks.storage.am.lsm.rtree.dataflow.ExternalRTreeDataflowHelperFactory;
public class ExternalDatasetIndexesAbortOperatorDescriptor extends AbstractExternalDatasetIndexesOperatorDescriptor {
private static final long serialVersionUID = 1L;
public ExternalDatasetIndexesAbortOperatorDescriptor(IOperatorDescriptorRegistry spec,
- ExternalBTreeDataflowHelperFactory filesIndexDataflowHelperFactory,
- IndexInfoOperatorDescriptor fileIndexesInfo,
- List<ExternalBTreeWithBuddyDataflowHelperFactory> bTreeIndexesDataflowHelperFactories,
- List<IndexInfoOperatorDescriptor> bTreeIndexesInfos,
- List<ExternalRTreeDataflowHelperFactory> rTreeIndexesDataflowHelperFactories,
- List<IndexInfoOperatorDescriptor> rTreeIndexesInfos) {
- super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, bTreeIndexesDataflowHelperFactories,
- bTreeIndexesInfos, rTreeIndexesDataflowHelperFactories, rTreeIndexesInfos);
+ IIndexDataflowHelperFactory filesIndexDataflowHelperFactory, IndexInfoOperatorDescriptor fileIndexesInfo,
+ List<IIndexDataflowHelperFactory> indexesDataflowHelperFactories,
+ List<IndexInfoOperatorDescriptor> indexesInfos) {
+ super(spec, filesIndexDataflowHelperFactory, fileIndexesInfo, indexesDataflowHelperFactories,
+ indexesInfos);
}
@Override