You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hugegraph.apache.org by zh...@apache.org on 2022/11/24 10:52:25 UTC
[incubator-hugegraph-computer] 01/01: improve package
This is an automated email from the ASF dual-hosted git repository.
zhaocong pushed a commit to branch improve_package
in repository https://gitbox.apache.org/repos/asf/incubator-hugegraph-computer.git
commit b034998535998f0110cc26b3a348888a23e88fd6
Author: coderzc <zh...@apache.org>
AuthorDate: Thu Nov 24 18:52:11 2022 +0800
improve package
---
.../hugegraph/computer/core/worker/WorkerService.java | 4 ++--
computer-dist/Dockerfile | 3 ++-
computer-dist/pom.xml | 6 ++++++
.../src/assembly/static/conf/computer.properties | 14 ++++++++++++++
.../baidu/hugegraph/computer/dist/HugeGraphComputer.java | 16 +++++++++++-----
computer-dist/src/main/resources/computer.properties | 0
.../hugegraph/computer/dist/HugeGraphComputerTest.java | 6 +++---
.../src/main/resources/computer-worker1.properties | 15 ---------------
.../{computer-master.properties => computer.properties} | 2 +-
9 files changed, 39 insertions(+), 27 deletions(-)
diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
index 515b34df..aba86e31 100644
--- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
+++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
@@ -25,6 +25,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.baidu.hugegraph.computer.core.aggregator.Aggregator;
@@ -56,8 +58,6 @@ import com.baidu.hugegraph.computer.core.sort.sorting.SendSortManager;
import com.baidu.hugegraph.computer.core.sort.sorting.SortManager;
import com.baidu.hugegraph.computer.core.store.FileManager;
import com.baidu.hugegraph.computer.core.util.ShutdownHook;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
public class WorkerService implements Closeable {
diff --git a/computer-dist/Dockerfile b/computer-dist/Dockerfile
index 6ab4d462..9c6b8cec 100644
--- a/computer-dist/Dockerfile
+++ b/computer-dist/Dockerfile
@@ -18,6 +18,7 @@ FROM openjdk:11-jre
LABEL maintainer="HugeGraph Docker Maintainers <hu...@googlegroups.com>"
# use ParallelGC which is more friendly to olap system
ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
-COPY . /etc/local/hugegraph-computer
+COPY ./bin /etc/local/hugegraph-computer/bin
+COPY ./lib /etc/local/hugegraph-computer/lib
WORKDIR /etc/local/hugegraph-computer
RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget
diff --git a/computer-dist/pom.xml b/computer-dist/pom.xml
index e570fd9a..160530a1 100644
--- a/computer-dist/pom.xml
+++ b/computer-dist/pom.xml
@@ -116,6 +116,12 @@
<configuration>
<tasks>
<echo file="${top.level.dir}/dist.sh">
+ mkdir -p ${top.level.dir}/${final.name}/algorithm/
+ mv ${top.level.dir}/computer-algorithm/target/computer-algorithm-${project.version}.jar ${top.level.dir}/${final.name}/algorithm/
+ mkdir -p ${top.level.dir}/${final.name}/k8s-operator/
+ mv ${top.level.dir}/computer-k8s-operator/target/hugegraph-computer-operator-${project.version}.jar ${top.level.dir}/${final.name}/k8s-operator/
+ rm -rf ${top.level.dir}/computer-algorithm/target/computer-dist-${project.version}.jar
+
tar -zcvf \
${top.level.dir}/${final.name}.tar.gz \
${final.name} || exit 1
diff --git a/computer-dist/src/assembly/static/conf/computer.properties b/computer-dist/src/assembly/static/conf/computer.properties
new file mode 100644
index 00000000..302abfcc
--- /dev/null
+++ b/computer-dist/src/assembly/static/conf/computer.properties
@@ -0,0 +1,14 @@
+rpc.server_host=localhost
+rpc.server_port=8190
+
+job.id=local_001
+job.workers_count=1
+
+bsp.register_timeout=100000
+bsp.log_interval=30000
+bsp.max_super_step=2
+
+hugegraph.url=http://127.0.0.1:8080
+hugegraph.name=hugegraph
+
+algorithm.params_class=com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams
diff --git a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
index 9b2a60f2..16a9e28b 100644
--- a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
+++ b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java
@@ -25,6 +25,10 @@ import java.util.Properties;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hugegraph.config.OptionSpace;
+import org.apache.hugegraph.config.RpcOptions;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
import org.slf4j.Logger;
import com.baidu.hugegraph.computer.core.common.ComputerContext;
@@ -34,9 +38,6 @@ import com.baidu.hugegraph.computer.core.master.MasterService;
import com.baidu.hugegraph.computer.core.network.message.MessageType;
import com.baidu.hugegraph.computer.core.util.ComputerContextUtil;
import com.baidu.hugegraph.computer.core.worker.WorkerService;
-import org.apache.hugegraph.config.OptionSpace;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
public class HugeGraphComputer {
@@ -69,7 +70,7 @@ public class HugeGraphComputer {
setUncaughtExceptionHandler();
loadClass();
registerOptions();
- ComputerContext context = parseContext(args[0]);
+ ComputerContext context = parseContext(args[0], role);
switch (role) {
case ROLE_MASTER:
executeMasterService(context);
@@ -124,12 +125,17 @@ public class HugeGraphComputer {
}
}
- private static ComputerContext parseContext(String conf)
+ private static ComputerContext parseContext(String conf, String role)
throws IOException {
Properties properties = new Properties();
BufferedReader bufferedReader = new BufferedReader(
new FileReader(conf));
properties.load(bufferedReader);
+ properties.remove(RpcOptions.RPC_REMOTE_URL.name());
+ if (ROLE_WORKER.equals(role)) {
+ properties.remove(RpcOptions.RPC_SERVER_HOST.name());
+ properties.remove(RpcOptions.RPC_SERVER_PORT.name());
+ }
ComputerContextUtil.initContext(properties);
return ComputerContext.instance();
}
diff --git a/computer-dist/src/main/resources/computer.properties b/computer-dist/src/main/resources/computer.properties
deleted file mode 100644
index e69de29b..00000000
diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
index a54d54f9..52e31fba 100644
--- a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
+++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java
@@ -41,9 +41,9 @@ public class HugeGraphComputerTest extends UnitTestBase {
CountDownLatch countDownLatch = new CountDownLatch(2);
Throwable[] exceptions = new Throwable[2];
String masterConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-master.properties").getPath();
+ "/computer.properties").getPath();
String work1ConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-worker1.properties").getPath();
+ "/computer.properties").getPath();
pool.submit(() -> {
try {
Thread.sleep(2000L);
@@ -79,7 +79,7 @@ public class HugeGraphComputerTest extends UnitTestBase {
@Test
public void testServiceWithError() {
String work1ConfPath = HugeGraphComputerTest.class.getResource(
- "/computer-worker1.properties").getPath();
+ "/computer.properties").getPath();
Assert.assertThrows(IllegalArgumentException.class,
() -> {
String[] args1 = {work1ConfPath, "worker111",
diff --git a/computer-test/src/main/resources/computer-worker1.properties b/computer-test/src/main/resources/computer-worker1.properties
deleted file mode 100644
index 8852515b..00000000
--- a/computer-test/src/main/resources/computer-worker1.properties
+++ /dev/null
@@ -1,15 +0,0 @@
-job.id=local_002
-job.workers_count=1
-
-transport.server_port=8086
-
-bsp.register_timeout=100000
-bsp.log_interval=30000
-bsp.max_super_step=2
-
-worker.computation_class=com.baidu.hugegraph.computer.core.worker.MockComputation
-
-algorithm.result_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue
-algorithm.message_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue
-algorithm.params_class=com.baidu.hugegraph.computer.core.worker.MockComputationParams
-
diff --git a/computer-test/src/main/resources/computer-master.properties b/computer-test/src/main/resources/computer.properties
similarity index 95%
rename from computer-test/src/main/resources/computer-master.properties
rename to computer-test/src/main/resources/computer.properties
index 37968263..dd52c3b6 100644
--- a/computer-test/src/main/resources/computer-master.properties
+++ b/computer-test/src/main/resources/computer.properties
@@ -1,5 +1,5 @@
rpc.server_host=localhost
-rpc.server_port=8090
+rpc.server_port=8190
job.id=local_002
job.workers_count=1