You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by zh...@apache.org on 2022/11/24 10:52:24 UTC
[incubator-hugegraph-computer] branch improve_package created (now b0349985)
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a change to branch improve_package
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
at b0349985 improve package
This branch includes the following new commits:
new b0349985 improve package
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-hugegraph-computer] 01/01: improve package
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a commit to branch improve_package
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
commit b034998535998f0110cc26b3a348888a23e88fd6
Author: coderzc <zh...@apache.org>
AuthorDate: Thu Nov 24 18:52:11 2022 +0800
improve package
---
.../hugegraph/computer/core/worker/WorkerService.java | 4 ++--
computer-dist/Dockerfile | 3 ++-
computer-dist/pom.xml | 6 ++++++
.../src/assembly/static/conf/computer.properties | 14 ++++++++++++++
.../baidu/hugegraph/computer/dist/HugeGraphComputer.java | 16 +++++++++++-----
computer-dist/src/main/resources/computer.properties | 0
.../hugegraph/computer/dist/HugeGraphComputerTest.java | 6 +++---
.../src/main/resources/computer-worker1.properties | 15 ---------------
.../{computer-master.properties => computer.properties} | 2 +-
9 files changed, 39 insertions(+), 27 deletions(-)
diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
index 515b34df..aba86e31 100644
--- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
+++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
@@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.baidu.hugegraph.computer.core.aggregator.Aggregator;
@@ -56,8 +58,6 @@ import com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager;
import com.baidu.hugegraph.computer.core.sort.sorting.SortManager;
import com.baidu.hugegraph.computer.core.store.FileManager;
import com.baidu.hugegraph.computer.core.util.ShutdownHook;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
public class WorkerService implements Closeable {
diff --git a/computer-dist/Dockerfile b/computer-dist/Dockerfile
index 6ab4d462..9c6b8cec 100644
--- a/computer-dist/Dockerfile
+++ b/computer-dist/Dockerfile
@@ -18,6 +18,7 @@ FROM openjdk:11-jre
LABEL maintainer="HugeGraph Docker Maintainers <hu...@googlegroups.com>"
# use ParallelGC which is more friendly to olap system
ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
-COPY . /etc/local/hugegraph-computer
+COPY ./bin /etc/local/hugegraph-computer/bin
+COPY ./lib /etc/local/hugegraph-computer/lib
WORKDIR /etc/local/hugegraph-computer
RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget
diff --git a/computer-dist/pom.xml b/computer-dist/pom.xml
index e570fd9a..160530a1 100644
--- a/computer-dist/pom.xml
+++ b/computer-dist/pom.xml
@@ -116,6 +116,12 @@
<configuration>
<tasks>
<echo file="${top.level.dir}/dist.sh">
+ mkdir -p ${top.level.dir}/${final.name}/algorithm/
+ mv ${top.level.dir}/computer-algorithm/target/computer-algorithm-${project.version}.jar ${top.level.dir}/${final.name}/algorithm/
+ mkdir -p ${top.level.dir}/${final.name}/k8s-operator/
+ mv ${top.level.dir}/computer-k8s-operator/target/hugegraph-computer-operator-${project.version}.jar ${top.level.dir}/${final.name}/k8s-operator/
+ rm -rf ${top.level.dir}/computer-algorithm/target/computer-dist-${project.version}.jar
+
tar -zcvf \
${top.level.dir}/${final.name}.tar.gz \
${final.name} || exit 1
diff --git a/computer-dist/src/assembly/static/conf/computer.properties b/computer-dist/src/assembly/static/conf/computer.properties
new file mode 100644
index 00000000..302abfcc
--- /dev/null
+++ b/computer-dist/src/assembly/static/conf/computer.properties
@@ -0,0 +1,14 @@
+rpc.server_host=localhost
+rpc.server_port=8190
+
+job.id=local_001
+job.workers_count=1
+
+bsp.register_timeout=100000
+bsp.log_interval=30000
+bsp.max_super_step=2
+
+hugegraph.url=http://127.0.0.1:8080
+hugegraph.name=hugegraph
+
+algorithm.params_class=com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams
diff --git a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
index 9b2a60f2..16a9e28b 100644
--- a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
+++ b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
@@ -25,6 +25,10 @@ import java.util.Properties;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hugegraph.config.OptionSpace;
+import org.apache.hugegraph.config.RpcOptions;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.baidu.hugegraph.computer.core.common.ComputerContext;
@@ -34,9 +38,6 @@ import com.baidu.hugegraph.computer.core.master.MasterService;
import com.baidu.hugegraph.computer.core.network.message.MessageType;
import com.baidu.hugegraph.computer.core.util.ComputerContextUtil;
import com.baidu.hugegraph.computer.core.worker.WorkerService;
-import org.apache.hugegraph.config.OptionSpace;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
public class HugeGraphComputer {
@@ -69,7 +70,7 @@ public class HugeGraphComputer {
setUncaughtExceptionHandler();
loadClass();
registerOptions();
- ComputerContext context = parseContext(args[0]);
+ ComputerContext context = parseContext(args[0], role);
switch (role) {
case ROLE_MASTER:
executeMasterService(context);
@@ -124,12 +125,17 @@ public class HugeGraphComputer {
}
}
- private static ComputerContext parseContext(String conf)
+ private static ComputerContext parseContext(String conf, String role)
throws IOException {
Properties properties = new Properties();
BufferedReader bufferedReader = new BufferedReader(
new FileReader(conf));
properties.load(bufferedReader);
+ properties.remove(RpcOptions.RPC_REMOTE_URL.name());
+ if (ROLE_WORKER.equals(role)) {
+ properties.remove(RpcOptions.RPC_SERVER_HOST.name());
+ properties.remove(RpcOptions.RPC_SERVER_PORT.name());
+ }
ComputerContextUtil.initContext(properties);
return ComputerContext.instance();
}
diff --git a/computer-dist/src/main/resources/computer.properties b/computer-dist/src/main/resources/computer.properties
deleted file mode 100644
index e69de29b..00000000
diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
index a54d54f9..52e31fba 100644
--- a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
+++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
@@ -41,9 +41,9 @@ public class HugeGraphComputerTest extends UnitTestBase {
CountDownLatch countDownLatch = new CountDownLatch(2);
Throwable[] exceptions = new Throwable[2];
String masterConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-master.properties").getPath();
+ "/computer.properties").getPath();
String work1ConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-worker1.properties").getPath();
+ "/computer.properties").getPath();
pool.submit(() -> {
try {
Thread.sleep(2000L);
@@ -79,7 +79,7 @@ public class HugeGraphComputerTest extends UnitTestBase {
@Test
public void testServiceWithError() {
String work1ConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-worker1.properties").getPath();
+ "/computer.properties").getPath();
Assert.assertThrows(IllegalArgumentException.class,
() -> {
String[] args1 = {work1ConfPath, "worker111",
diff --git a/computer-test/src/main/resources/computer-worker1.properties b/computer-test/src/main/resources/computer-worker1.properties
deleted file mode 100644
index 8852515b..00000000
--- a/computer-test/src/main/resources/computer-worker1.properties
+++ /dev/null
@@ -1,15 +0,0 @@
-job.id=local_002
-job.workers_count=1
-
-transport.server_port=8086
-
-bsp.register_timeout=100000
-bsp.log_interval=30000
-bsp.max_super_step=2
-
-worker.computation_class=com.baidu.hugegraph.computer.core.worker.MockComputation
-
-algorithm.result_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue
-algorithm.message_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue
-algorithm.params_class=com.baidu.hugegraph.computer.core.worker.MockComputationParams
-
diff --git a/computer-test/src/main/resources/computer-master.properties b/computer-test/src/main/resources/computer.properties
similarity index 95%
rename from computer-test/src/main/resources/computer-master.properties
rename to computer-test/src/main/resources/computer.properties
index 37968263..dd52c3b6 100644
--- a/computer-test/src/main/resources/computer-master.properties
+++ b/computer-test/src/main/resources/computer.properties
@@ -1,5 +1,5 @@
rpc.server_host=localhost
-rpc.server_port=8090
+rpc.server_port=8190
job.id=local_002
job.workers_count=1