You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2026/09/23 16:17:45 UTC
(atlas) 01/01: ATLAS-5205 : Improve bootstrap performance
This is an automated email from the ASF dual-hosted git repository.
mandarambawane pushed a commit to branch ATLAS-5205-0923
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 359d6256e1ce05f5db750a42635fab5d86041a64
Author: Mandar Ambawane <ma...@cloudera.com>
AuthorDate: Wed Sep 23 21:45:09 2026 +0530
ATLAS-5205 : Improve bootstrap performance
---
.../bootstrap/AtlasTypeDefStoreInitializer.java | 111 ++++++++++++++++++---
.../store/bootstrap/ModelsFolderLoader.java | 27 +++++
.../store/bootstrap/TypeDefBootstrapTask.java | 98 ++++++++++++++++++
.../bootstrap/TypeDefBootstrapTaskFactory.java | 57 +++++++++++
4 files changed, 281 insertions(+), 12 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index 31e748096..cdeabd3d5 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
@@ -31,6 +32,7 @@ import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.ActiveStateChangeHandler;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.patches.AtlasPatch.PatchStatus;
+import org.apache.atlas.model.tasks.AtlasTask;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
@@ -52,6 +54,7 @@ import org.apache.atlas.repository.patches.AtlasPatchManager;
import org.apache.atlas.repository.patches.AtlasPatchRegistry;
import org.apache.atlas.repository.patches.SuperTypesUpdatePatch;
import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.tasks.AbstractTask;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
@@ -84,6 +87,10 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
@@ -91,6 +98,9 @@ import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED;
import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.FAILED;
import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.SKIPPED;
import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN;
+import static org.apache.atlas.model.tasks.AtlasTask.Status.COMPLETE;
+import static org.apache.atlas.repository.store.bootstrap.TypeDefBootstrapTaskFactory.MODELS_FOLDER_PATH;
+import static org.apache.atlas.repository.store.bootstrap.TypeDefBootstrapTaskFactory.TYPEDEF_BOOTSTRAP_LOAD;
/**
* Class that handles initial loading of models and patches into typedef store
@@ -99,7 +109,8 @@ import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.UNKNOWN;
@Order(2)
public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
public static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
- public static final String PATCHES_FOLDER_NAME = "patches";
+ public static final String PATCHES_FOLDER_NAME = "patches";
+ public static final String BASE_MODELS_MODULE_FOLDER = "0000-Area0";
public static final String RELATIONSHIP_LABEL = "relationshipLabel";
public static final String RELATIONSHIP_CATEGORY = "relationshipCategory";
public static final String RELATIONSHIP_SWAP_ENDS = "swapEnds";
@@ -308,13 +319,11 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
}
/**
- * This method is looking for folders in alphabetical order in the models directory. It loads each of these folders and their associated patches in order.
- * It then loads any models in the top level folder and its patches.
- *
- * This allows models to be grouped into folders to help managability.
- *
+ * Loads typedef models from the models directory. The {@value #BASE_MODELS_MODULE_FOLDER} folder is always loaded
+ * first (parent/base typedefs), then all other module subdirectories are loaded in parallel. Top-level model files
+ * and patches are loaded last.
*/
- private void loadBootstrapTypeDefs() {
+ private void loadBootstrapTypeDefs() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
String atlasHomeDir = System.getProperty("atlas.home");
@@ -331,13 +340,38 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
if (modelsDirContents != null && modelsDirContents.length > 0) {
Arrays.sort(modelsDirContents);
+ List<File> moduleFolders = new ArrayList<>();
+
for (File folder : modelsDirContents) {
if (folder.isFile()) {
- // ignore files
continue;
- } else if (!folder.getName().equals(PATCHES_FOLDER_NAME)) {
- // load the models alphabetically in the subfolders apart from patches
- loadModelsInFolder(folder, patchRegistry);
+ }
+
+ if (!folder.getName().equals(PATCHES_FOLDER_NAME)) {
+ moduleFolders.add(folder);
+ }
+ }
+
+ if (!moduleFolders.isEmpty()) {
+ File baseModelsFolder = null;
+ List<File> parallelModuleFolders = new ArrayList<>();
+
+ for (File folder : moduleFolders) {
+ if (BASE_MODELS_MODULE_FOLDER.equals(folder.getName())) {
+ baseModelsFolder = folder;
+ } else {
+ parallelModuleFolders.add(folder);
+ }
+ }
+
+ if (baseModelsFolder != null) {
+ LOG.info("Loading parent typedefs from {} before addon module folders", BASE_MODELS_MODULE_FOLDER);
+
+ loadModelsInFolder(baseModelsFolder, patchRegistry);
+ }
+
+ if (!parallelModuleFolders.isEmpty()) {
+ loadModuleFoldersInParallel(parallelModuleFolders, patchRegistry);
}
}
}
@@ -349,11 +383,58 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
LOG.info("<== AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
}
+ private void loadModuleFoldersInParallel(List<File> moduleFolders, AtlasPatchRegistry patchRegistry) throws AtlasBaseException {
+ LOG.info("Loading {} module folders in parallel", moduleFolders.size());
+
+ ExecutorService executor = Executors.newFixedThreadPool(moduleFolders.size(),
+ new ThreadFactoryBuilder().setDaemon(true).setNameFormat("typedef-bootstrap-%d").build());
+
+ List<Future<AtlasTask.Status>> futures = new ArrayList<>(moduleFolders.size());
+
+ try {
+ for (File folder : moduleFolders) {
+ futures.add(executor.submit(() -> runBootstrapTaskForFolder(folder, patchRegistry)));
+ }
+
+ for (int i = 0; i < futures.size(); i++) {
+ File folder = moduleFolders.get(i);
+ AtlasTask.Status status = futures.get(i).get();
+
+ if (status != COMPLETE) {
+ throw new AtlasBaseException("Failed to load typedef models from module folder " + folder.getAbsolutePath());
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new AtlasBaseException(e);
+ } catch (ExecutionException e) {
+ throw new AtlasBaseException(e.getCause());
+ } finally {
+ executor.shutdown();
+ }
+ }
+
+ private AtlasTask.Status runBootstrapTaskForFolder(File folder, AtlasPatchRegistry patchRegistry) {
+ Map<String, Object> params = Collections.singletonMap(MODELS_FOLDER_PATH, folder.getAbsolutePath());
+ AtlasTask task = new AtlasTask(TYPEDEF_BOOTSTRAP_LOAD, "system", params);
+
+ AbstractTask bootstrapTask = TypeDefBootstrapTaskFactory.createBootstrapTask(task, patchRegistry, this::loadModelsInFolder);
+
+ try {
+ return bootstrapTask.perform();
+ } catch (Exception e) {
+ LOG.error("Error loading typedef models from folder {}", folder.getAbsolutePath(), e);
+
+ return AtlasTask.Status.FAILED;
+ }
+ }
+
/**
* Load all the model files in the supplied folder followed by the contents of the patches folder.
* @param typesDir
*/
- private void loadModelsInFolder(File typesDir, AtlasPatchRegistry patchRegistry) {
+ void loadModelsInFolder(File typesDir, AtlasPatchRegistry patchRegistry) {
LOG.info("==> AtlasTypeDefStoreInitializer({})", typesDir);
String typesDirName = typesDir.getName();
@@ -446,6 +527,12 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
}
private void applyTypePatches(String typesDirName, AtlasPatchRegistry patchRegistry) {
+ synchronized (patchRegistry) {
+ applyTypePatchesInternal(typesDirName, patchRegistry);
+ }
+ }
+
+ private void applyTypePatchesInternal(String typesDirName, AtlasPatchRegistry patchRegistry) {
String typePatchesDirName = typesDirName + File.separator + PATCHES_FOLDER_NAME;
File typePatchesDir = new File(typePatchesDirName);
File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/ModelsFolderLoader.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/ModelsFolderLoader.java
new file mode 100644
index 000000000..3c5cc7856
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/ModelsFolderLoader.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.bootstrap;
+
+import org.apache.atlas.repository.patches.AtlasPatchRegistry;
+
+import java.io.File;
+
+@FunctionalInterface
+public interface ModelsFolderLoader {
+ void load(File modelsFolder, AtlasPatchRegistry patchRegistry);
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/TypeDefBootstrapTask.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/TypeDefBootstrapTask.java
new file mode 100644
index 000000000..685154b2a
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/TypeDefBootstrapTask.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.bootstrap;
+
+import org.apache.atlas.RequestContext;
+import org.apache.atlas.model.tasks.AtlasTask;
+import org.apache.atlas.repository.patches.AtlasPatchRegistry;
+import org.apache.atlas.tasks.AbstractTask;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Map;
+
+import static org.apache.atlas.model.tasks.AtlasTask.Status.COMPLETE;
+import static org.apache.atlas.model.tasks.AtlasTask.Status.FAILED;
+import static org.apache.atlas.repository.store.bootstrap.TypeDefBootstrapTaskFactory.MODELS_FOLDER_PATH;
+
+public class TypeDefBootstrapTask extends AbstractTask {
+ private static final Logger LOG = LoggerFactory.getLogger(TypeDefBootstrapTask.class);
+
+ private final AtlasPatchRegistry patchRegistry;
+ private final ModelsFolderLoader modelsFolderLoader;
+
+ public TypeDefBootstrapTask(AtlasTask task, AtlasPatchRegistry patchRegistry, ModelsFolderLoader modelsFolderLoader) {
+ super(task);
+
+ this.patchRegistry = patchRegistry;
+ this.modelsFolderLoader = modelsFolderLoader;
+ }
+
+ @Override
+ public AtlasTask.Status perform() throws Exception {
+ RequestContext.clear();
+
+ Map<String, Object> params = getTaskDef().getParameters();
+
+ if (params == null || !params.containsKey(MODELS_FOLDER_PATH)) {
+ LOG.warn("Task: {}: Missing models folder path parameter", getTaskGuid());
+
+ return FAILED;
+ }
+
+ Object folderPathObj = params.get(MODELS_FOLDER_PATH);
+
+ if (!(folderPathObj instanceof String) || StringUtils.isEmpty((String) folderPathObj)) {
+ LOG.warn("Task: {}: Invalid models folder path parameter", getTaskGuid());
+
+ return FAILED;
+ }
+
+ File modelsFolder = new File((String) folderPathObj);
+
+ if (!modelsFolder.isDirectory()) {
+ LOG.warn("Task: {}: Models folder does not exist or is not a directory: {}", getTaskGuid(), modelsFolder.getAbsolutePath());
+
+ return FAILED;
+ }
+
+ if (patchRegistry == null || modelsFolderLoader == null) {
+ LOG.warn("Task: {}: Bootstrap loader context is not available", getTaskGuid());
+
+ return FAILED;
+ }
+
+ try {
+ modelsFolderLoader.load(modelsFolder, patchRegistry);
+
+ setStatus(COMPLETE);
+ } catch (Exception e) {
+ LOG.error("Task: {}: Failed to load typedef models from folder {}", getTaskGuid(), modelsFolder.getAbsolutePath(), e);
+
+ setStatus(FAILED);
+
+ throw e;
+ } finally {
+ RequestContext.clear();
+ }
+
+ return getStatus();
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/TypeDefBootstrapTaskFactory.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/TypeDefBootstrapTaskFactory.java
new file mode 100644
index 000000000..17592aa1d
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/TypeDefBootstrapTaskFactory.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.bootstrap;
+
+import org.apache.atlas.model.tasks.AtlasTask;
+import org.apache.atlas.repository.patches.AtlasPatchRegistry;
+import org.apache.atlas.tasks.AbstractTask;
+import org.apache.atlas.tasks.TaskFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+@Component
+public class TypeDefBootstrapTaskFactory implements TaskFactory {
+ private static final Logger LOG = LoggerFactory.getLogger(TypeDefBootstrapTaskFactory.class);
+
+ public static final String TYPEDEF_BOOTSTRAP_LOAD = "TYPEDEF_BOOTSTRAP_LOAD";
+ public static final String MODELS_FOLDER_PATH = "modelsFolderPath";
+
+ private static final List<String> SUPPORTED_TYPES = new ArrayList<>(Collections.singletonList(TYPEDEF_BOOTSTRAP_LOAD));
+
+ public static AbstractTask createBootstrapTask(AtlasTask task, AtlasPatchRegistry patchRegistry, ModelsFolderLoader modelsFolderLoader) {
+ return new TypeDefBootstrapTask(task, patchRegistry, modelsFolderLoader);
+ }
+
+ @Override
+ public AbstractTask create(AtlasTask task) {
+ LOG.warn("Type: {} - {} cannot be created without bootstrap context; use createBootstrapTask() during typedef bootstrap",
+ task.getType(), task.getGuid());
+
+ return null;
+ }
+
+ @Override
+ public List<String> getSupportedTypes() {
+ return SUPPORTED_TYPES;
+ }
+}