You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2022/07/11 02:18:18 UTC
[incubator-linkis] branch dev-1.2.0 updated: [Feature-2450] The new storage path is optimized when the engine material is updated. (#2451)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.2.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.2.0 by this push:
new d0355fdbf [Feature-2450] The new storage path is optimized when the engine material is updated. (#2451)
d0355fdbf is described below
commit d0355fdbf7902996de285fa6020710bfe30c38a3
Author: weixiao <le...@gmail.com>
AuthorDate: Mon Jul 11 10:18:14 2022 +0800
[Feature-2450] The new storage path is optimized when the engine material is updated. (#2451)
* [Feature-2450] The new storage path is optimized when the engine material is updated
* [Feature-2450] Add more test cases.
---
.../linkis/bml/common/HdfsResourceHelper.java | 6 +++
.../linkis/bml/common/LocalResourceHelper.java | 6 +++
.../apache/linkis/bml/common/ResourceHelper.java | 8 ++++
.../bml/service/impl/VersionServiceImpl.java | 8 +++-
.../linkis/bml/common/HdfsResourceHelperTest.java} | 37 +++++++-----------
.../bml/common/LocalResourceHelperTest.java} | 38 ++++++++-----------
.../linkis/bml/common/VersionServiceImplTest.java | 44 ++++++++++++++++++++++
7 files changed, 100 insertions(+), 47 deletions(-)
diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/HdfsResourceHelper.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/HdfsResourceHelper.java
index e2e51f118..e2589429a 100644
--- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/HdfsResourceHelper.java
+++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/HdfsResourceHelper.java
@@ -176,4 +176,10 @@ public class HdfsResourceHelper implements ResourceHelper {
fileSystem.close();
}
}
+
+ @Override
+ public boolean checkBmlResourceStoragePrefixPathIfChanged(String path) {
+ String prefixPath = getSchema() + BmlServerConfiguration.BML_HDFS_PREFIX().getValue();
+ return !path.startsWith(prefixPath);
+ }
}
diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/LocalResourceHelper.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/LocalResourceHelper.java
index db5e30839..cd7c9621c 100644
--- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/LocalResourceHelper.java
+++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/LocalResourceHelper.java
@@ -170,4 +170,10 @@ public class LocalResourceHelper implements ResourceHelper {
fileSystem.close();
}
}
+
+ @Override
+ public boolean checkBmlResourceStoragePrefixPathIfChanged(String path) {
+ String prefixPath = getSchema() + BmlServerConfiguration.BML_LOCAL_PREFIX().getValue();
+ return !path.startsWith(prefixPath);
+ }
}
diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java
index 7f2a261cb..d57f5dbac 100644
--- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java
+++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java
@@ -40,4 +40,12 @@ public interface ResourceHelper {
public String getSchema();
boolean checkIfExists(String path, String user) throws IOException;
+
+ /**
+ * check wds.linkis.bml.hdfs.prefix or wds.linkis.bml.local.prefix has changed or not
+ *
+ * @param path resource path
+ * @return result
+ */
+ boolean checkBmlResourceStoragePrefixPathIfChanged(String path);
}
diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java
index 4a613f70b..feef9932f 100644
--- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java
+++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/service/impl/VersionServiceImpl.java
@@ -91,7 +91,13 @@ public class VersionServiceImpl implements VersionService {
String fileName = new String(file.getOriginalFilename().getBytes("ISO8859-1"), "UTF-8");
// 获取资源的path
String newVersion = params.get("newVersion").toString();
- String path = versionDao.getResourcePath(resourceId) + "_" + newVersion;
+ String path = versionDao.getResourcePath(resourceId);
+ // if the bml resource storage prefix has changed，then regenerate the path.
+ if (resourceHelper.checkBmlResourceStoragePrefixPathIfChanged(path)) {
+ path =
+ resourceHelper.generatePath(
+ user, path.substring(path.lastIndexOf("/") + 1), new HashMap<>());
+ }
// 上传资源前，需要对resourceId这个字符串的intern进行加锁，这样所有需要更新该资源的用户都会同步
// synchronized (resourceIdLock.intern()){
// 资源上传到hdfs
diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/HdfsResourceHelperTest.java
similarity index 56%
copy from linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java
copy to linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/HdfsResourceHelperTest.java
index 7f2a261cb..1bd810401 100644
--- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java
+++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/HdfsResourceHelperTest.java
@@ -17,27 +17,18 @@
package org.apache.linkis.bml.common;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-
-public interface ResourceHelper {
-
- public long upload(
- String path,
- String user,
- InputStream inputStream,
- StringBuilder stringBuilder,
- boolean overwrite)
- throws UploadResourceException;
-
- public void update(String path);
-
- public void getResource(String path, int start, int end);
-
- public String generatePath(String user, String fileName, Map<String, Object> properties);
-
- public String getSchema();
-
- boolean checkIfExists(String path, String user) throws IOException;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class HdfsResourceHelperTest {
+ @Test
+ @DisplayName("testCheckBmlResourceStoragePrefixPathIfChanged")
+ public void testCheckBmlResourceStoragePrefixPathIfChanged() {
+ String path = "hdfs:///data/linkis/linkis/20220609/b4fd8f59-9492-4a0f-a074-9ac573a69b60";
+ ResourceHelper hdfsResourceHelper = new HdfsResourceHelper();
+ boolean hasChanged = hdfsResourceHelper.checkBmlResourceStoragePrefixPathIfChanged(path);
+ assertTrue(hasChanged);
+ }
}
diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/LocalResourceHelperTest.java
similarity index 56%
copy from linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java
copy to linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/LocalResourceHelperTest.java
index 7f2a261cb..4fe0c48b6 100644
--- a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/main/java/org/apache/linkis/bml/common/ResourceHelper.java
+++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/LocalResourceHelperTest.java
@@ -17,27 +17,19 @@
package org.apache.linkis.bml.common;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Map;
-
-public interface ResourceHelper {
-
- public long upload(
- String path,
- String user,
- InputStream inputStream,
- StringBuilder stringBuilder,
- boolean overwrite)
- throws UploadResourceException;
-
- public void update(String path);
-
- public void getResource(String path, int start, int end);
-
- public String generatePath(String user, String fileName, Map<String, Object> properties);
-
- public String getSchema();
-
- boolean checkIfExists(String path, String user) throws IOException;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class LocalResourceHelperTest {
+ @Test
+ @DisplayName("testCheckBmlResourceStoragePrefixPathIfChanged")
+ public void testCheckBmlResourceStoragePrefixPathIfChanged() {
+ String path =
+ "file:///data/dss/bml/linkis/linkis/20220609/b4fd8f59-9492-4a0f-a074-9ac573a69b60";
+ LocalResourceHelper localResourceHelper = new LocalResourceHelper();
+ boolean hasChanged = localResourceHelper.checkBmlResourceStoragePrefixPathIfChanged(path);
+ assertFalse(hasChanged);
+ }
}
diff --git a/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/VersionServiceImplTest.java b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/VersionServiceImplTest.java
new file mode 100644
index 000000000..ce8355a54
--- /dev/null
+++ b/linkis-public-enhancements/linkis-bml/linkis-bml-server/src/test/java/org/apache/linkis/bml/common/VersionServiceImplTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.bml.common;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.HashMap;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class VersionServiceImplTest {
+
+ @Test
+ @DisplayName("testCheckBmlResourceStoragePrefixPathIfChanged")
+ public void testUpdateVersion() {
+ String path = "hdfs:///data/linkis/linkis/20220609/b4fd8f59-9492-4a0f-a074-9ac573a69b60";
+ ResourceHelper hdfsResourceHelper = new HdfsResourceHelper();
+ String newPath =
+ hdfsResourceHelper.generatePath(
+ "linkis",
+ path.substring(path.lastIndexOf(File.separator) + 1),
+ new HashMap<>());
+ assertEquals(
+ newPath,
+ "hdfs:///apps-data/linkis/bml/20220710/b4fd8f59-9492-4a0f-a074-9ac573a69b60");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org