You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by sz...@apache.org on 2022/02/01 07:19:17 UTC
[ratis] branch master updated: RATIS-1512. Remove hadoop related components from ratis.git. (#595)
This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 6fb2e54 RATIS-1512. Remove hadoop related components from ratis.git. (#595)
6fb2e54 is described below
commit 6fb2e547ff0d08be41de49a269ae3dd4c47c8149
Author: Tsz-Wo Nicholas Sze <sz...@apache.org>
AuthorDate: Tue Feb 1 15:19:13 2022 +0800
RATIS-1512. Remove hadoop related components from ratis.git. (#595)
---
BUILDING.md | 5 +-
pom.xml | 46 +-
ratis-assembly/pom.xml | 19 -
ratis-assembly/src/main/assembly/bin.xml | 2 -
ratis-assembly/src/main/assembly/src.xml | 2 -
ratis-examples/pom.xml | 12 -
.../ratis/examples/ParameterizedBaseTest.java | 4 -
ratis-hadoop/dev-support/findbugsExcludeFile.xml | 21 -
ratis-hadoop/pom.xml | 138 -
.../apache/ratis/hadooprpc/HadoopConfigKeys.java | 81 -
.../apache/ratis/hadooprpc/HadoopConstants.java | 28 -
.../org/apache/ratis/hadooprpc/HadoopFactory.java | 69 -
.../java/org/apache/ratis/hadooprpc/Proxy.java | 55 -
.../hadooprpc/client/CombinedClientProtocol.java | 25 -
...mbinedClientProtocolClientSideTranslatorPB.java | 154 -
.../hadooprpc/client/CombinedClientProtocolPB.java | 37 -
...mbinedClientProtocolServerSideTranslatorPB.java | 151 -
.../ratis/hadooprpc/client/HadoopClientRpc.java | 72 -
.../ratis/hadooprpc/server/HadoopRpcService.java | 214 --
.../hadooprpc/server/RaftServerProtocolPB.java | 37 -
.../RaftServerProtocolServerSideTranslatorPB.java | 98 -
.../src/main/proto/HadoopCompatability.proto | 76 -
.../hadooprpc/MiniRaftClusterWithHadoopRpc.java | 97 -
.../hadooprpc/TestGroupInfoWithHadoopRpc.java | 25 -
.../TestGroupManagementWithHadoopRpc.java | 28 -
.../hadooprpc/TestLeaderElectionWithHadoopRpc.java | 54 -
.../hadooprpc/TestLogAppenderWithHadoopRpc.java | 25 -
.../TestPreAppendLeaderStepDownWithHadoopRpc.java | 25 -
.../hadooprpc/TestRaftExceptionWithHadoopRpc.java | 37 -
.../TestRaftReconfigurationWithHadoopRpc.java | 37 -
.../hadooprpc/TestRaftSnapshotWithHadoopRpc.java | 28 -
...TestRaftStateMachineExceptionWithHadoopRpc.java | 25 -
.../ratis/hadooprpc/TestRaftWithHadoopRpc.java | 62 -
.../hadooprpc/TestRetryCacheWithHadoopRpc.java | 25 -
.../TestSnapshotManagementWithHadoopRpc.java | 25 -
ratis-hadoop/src/test/resources/log4j.properties | 18 -
ratis-logservice/Dockerfile | 35 -
ratis-logservice/README.md | 99 -
ratis-logservice/TUNING.md | 58 -
ratis-logservice/build-docker.sh | 31 -
ratis-logservice/client-env.sh | 19 -
.../dev-support/findbugsExcludeFile.xml | 21 -
ratis-logservice/docker-compose.yml | 87 -
ratis-logservice/logservice-example.xml | 88 -
ratis-logservice/pom.xml | 330 ---
ratis-logservice/src/assembly/assembly.xml | 54 -
ratis-logservice/src/assembly/bin/load-test | 23 -
ratis-logservice/src/assembly/bin/shell | 23 -
.../ratis/logservice/api/ArchiveLogReader.java | 27 -
.../ratis/logservice/api/ArchiveLogWriter.java | 45 -
.../org/apache/ratis/logservice/api/LogInfo.java | 49 -
.../apache/ratis/logservice/api/LogMessage.java | 39 -
.../org/apache/ratis/logservice/api/LogName.java | 89 -
.../org/apache/ratis/logservice/api/LogReader.java | 88 -
.../ratis/logservice/api/LogServiceClient.java | 275 --
.../logservice/api/LogServiceConfiguration.java | 196 --
.../org/apache/ratis/logservice/api/LogStream.java | 134 -
.../org/apache/ratis/logservice/api/LogWriter.java | 59 -
.../ratis/logservice/api/RecordListener.java | 33 -
.../apache/ratis/logservice/common/Constants.java | 71 -
.../common/LogAlreadyExistException.java | 27 -
.../logservice/common/LogNotFoundException.java | 28 -
.../common/NoEnoughWorkersException.java | 32 -
.../logservice/impl/ArchiveHdfsLogReader.java | 212 --
.../logservice/impl/ArchiveHdfsLogWriter.java | 111 -
.../logservice/impl/ArchivedLogStreamImpl.java | 140 -
.../logservice/impl/ExportedLogStreamImpl.java | 35 -
.../ratis/logservice/impl/LogReaderImpl.java | 202 --
.../ratis/logservice/impl/LogStreamImpl.java | 234 --
.../ratis/logservice/impl/LogWriterImpl.java | 113 -
.../metrics/LogServiceMetaDataMetrics.java | 47 -
.../logservice/metrics/LogServiceMetrics.java | 45 -
.../ratis/logservice/server/ArchivalInfo.java | 95 -
.../apache/ratis/logservice/server/BaseServer.java | 147 -
.../apache/ratis/logservice/server/LogServer.java | 216 --
.../logservice/server/LogServiceRaftLogReader.java | 170 --
.../ratis/logservice/server/LogStateMachine.java | 787 ------
.../logservice/server/ManagementStateMachine.java | 29 -
.../ratis/logservice/server/MetaStateMachine.java | 526 ----
.../ratis/logservice/server/MetadataServer.java | 173 --
.../ratis/logservice/server/RaftLogReader.java | 47 -
.../apache/ratis/logservice/server/ServerOpts.java | 144 -
.../org/apache/ratis/logservice/shell/Command.java | 32 -
.../ratis/logservice/shell/CommandFactory.java | 71 -
.../ratis/logservice/shell/LogServiceShell.java | 150 -
.../logservice/shell/LogServiceShellOpts.java | 29 -
.../shell/commands/ArchiveLogCommand.java | 47 -
.../shell/commands/CreateLogCommand.java | 47 -
.../shell/commands/DeleteLogCommand.java | 49 -
.../logservice/shell/commands/ExitCommand.java | 37 -
.../shell/commands/ExportLogCommand.java | 49 -
.../logservice/shell/commands/HelpCommand.java | 40 -
.../logservice/shell/commands/ListLogsCommand.java | 59 -
.../logservice/shell/commands/PutToLogCommand.java | 53 -
.../logservice/shell/commands/ReadLogCommand.java | 72 -
.../ratis/logservice/tool/VerificationTool.java | 420 ---
.../ratis/logservice/util/LogServiceProtoUtil.java | 311 --
.../ratis/logservice/util/LogServiceUtils.java | 75 -
.../logservice/util/MetaServiceProtoUtil.java | 210 --
ratis-logservice/src/main/proto/LogService.proto | 198 --
ratis-logservice/src/main/proto/MetaService.proto | 142 -
.../src/main/resources/log4j.properties | 23 -
.../ratis/logservice/LogServiceReadWriteBase.java | 332 ---
.../ratis/logservice/TestLogServiceWithGrpc.java | 26 -
.../ratis/logservice/TestLogServiceWithNetty.java | 25 -
.../impl/TestArchiveHdfsLogReaderAndWriter.java | 170 --
.../ratis/logservice/server/TestBaseServer.java | 49 -
.../ratis/logservice/server/TestMetaServer.java | 444 ---
.../logservice/tool/TestVerificationTool.java | 59 -
.../ratis/logservice/util/LogServiceCluster.java | 169 --
.../logservice/util/TestLogServiceProtoUtil.java | 232 --
.../apache/ratis/logservice/util/TestUtils.java | 55 -
.../src/test/resources/hadoop-metrics2.properties | 56 -
.../src/test/resources/log4j.properties | 19 -
ratis-logservice/src/test/resources/logservice.xml | 28 -
ratis-proto/src/main/proto/Hadoop.proto | 60 -
.../src/main/resources/supplemental-models.xml | 2959 +-------------------
117 files changed, 96 insertions(+), 14091 deletions(-)
diff --git a/BUILDING.md b/BUILDING.md
index 137cb4d..b91ffba 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -31,11 +31,11 @@ See the next section for more info.
# Thirdparty
All bundled thirdparty dependencies are centralized in the *ratis-thirdparty* module
-and the *ratis-thirdparty-hadoop* module.
+and the *ratis-thirdparty-misc* module.
These modules are located in a separated repository
but not attached to the core Apache Ratis repository
as they only need to change when one of these dependencies are changed.
-All dependencies included in ratis-thirdparty/ratis-thirdparty-hadoop
+All dependencies included in ratis-thirdparty/ratis-thirdparty-misc
must be relocated to a different package to ensure no downstream classpath pollution.
Ratis developers should rely on these relocated thirdparty classes.
@@ -53,7 +53,6 @@ build are limited only to the code in Ratis itself.
| `com.google.thirdparty.publicsuffix`| `org.apache.ratis.thirdparty.com.google.thirdparty.publicsuffix` |
| `io.grpc` | `org.apache.ratis.thirdparty.io.grpc` |
| `io.netty` | `org.apache.ratis.thirdparty.io.netty` |
-| `org.apache.hadoop.ipc.protobuf` | `org.apache.ratis.thirdparty.org.apache.hadoop.ipc.protobuf` |
All compiled protocol-buffer definitions in `ratis-proto` are stored in the
`org.apache.ratis.proto` Java package.
diff --git a/pom.xml b/pom.xml
index 8c4bada..0638992 100644
--- a/pom.xml
+++ b/pom.xml
@@ -78,13 +78,11 @@
<module>ratis-resource-bundle</module>
<module>ratis-grpc</module>
<module>ratis-netty</module>
- <module>ratis-hadoop</module>
<module>ratis-test</module>
<module>ratis-examples</module>
<module>ratis-replicated-map</module>
- <module>ratis-logservice</module>
<module>ratis-metrics</module>
<module>ratis-tools</module>
<module>ratis-shell</module>
@@ -198,9 +196,6 @@
<shell-executable>bash</shell-executable>
- <hadoop.version>3.3.1</hadoop.version>
- <hadoop-maven-plugins.version>${hadoop.version}</hadoop-maven-plugins.version>
-
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
<java.min.version>${javac.version}</java.min.version>
@@ -213,8 +208,6 @@
<shaded.protobuf.version>3.12.0</shaded.protobuf.version>
<shaded.grpc.version>1.33.0</shaded.grpc.version>
- <hadoop.protobuf.version>2.5.0</hadoop.protobuf.version>
-
<!-- Test properties -->
<maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
<test.exclude.pattern>_</test.exclude.pattern>
@@ -320,18 +313,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
<dependency>
<artifactId>ratis-netty</artifactId>
@@ -399,19 +380,6 @@
<dependency>
<groupId>org.apache.ratis</groupId>
- <artifactId>ratis-logservice</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <artifactId>ratis-logservice</artifactId>
- <groupId>org.apache.ratis</groupId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ratis</groupId>
<artifactId>ratis-thirdparty-misc</artifactId>
<version>${ratis.thirdparty.version}</version>
</dependency>
@@ -461,14 +429,8 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
- <version>3.5.15</version>
+ <version>4.3.1</version>
</dependency>
- <dependency>
- <groupId>org.jline</groupId>
- <artifactId>jline</artifactId>
- <version>3.9.0</version>
- </dependency>
-
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>annotations-api</artifactId>
@@ -562,12 +524,6 @@
</plugin>
<plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <version>${hadoop-maven-plugins.version}</version>
- </plugin>
-
- <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
<configuration>
diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml
index 2bfb5c3..bc4ae92 100644
--- a/ratis-assembly/pom.xml
+++ b/ratis-assembly/pom.xml
@@ -231,16 +231,6 @@
</dependency>
<dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
<artifactId>ratis-test</artifactId>
<groupId>org.apache.ratis</groupId>
</dependency>
@@ -271,15 +261,6 @@
</dependency>
<dependency>
- <artifactId>ratis-logservice</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-logservice</artifactId>
- <groupId>org.apache.ratis</groupId>
- <type>test-jar</type>
- </dependency>
- <dependency>
<groupId>org.apache.ratis</groupId>
<artifactId>ratis-metrics</artifactId>
</dependency>
diff --git a/ratis-assembly/src/main/assembly/bin.xml b/ratis-assembly/src/main/assembly/bin.xml
index 261cf7c..0464264 100644
--- a/ratis-assembly/src/main/assembly/bin.xml
+++ b/ratis-assembly/src/main/assembly/bin.xml
@@ -33,8 +33,6 @@
<include>org.apache.ratis:ratis-client</include>
<include>org.apache.ratis:ratis-common</include>
<include>org.apache.ratis:ratis-grpc</include>
- <include>org.apache.ratis:ratis-hadoop</include>
- <include>org.apache.ratis:ratis-logservice</include>
<include>org.apache.ratis:ratis-netty</include>
<include>org.apache.ratis:ratis-proto</include>
<include>org.apache.ratis:ratis-replicated-map</include>
diff --git a/ratis-assembly/src/main/assembly/src.xml b/ratis-assembly/src/main/assembly/src.xml
index c20441b..2e8570e 100644
--- a/ratis-assembly/src/main/assembly/src.xml
+++ b/ratis-assembly/src/main/assembly/src.xml
@@ -34,8 +34,6 @@
<include>org.apache.ratis:ratis-common</include>
<include>org.apache.ratis:ratis-examples</include>
<include>org.apache.ratis:ratis-grpc</include>
- <include>org.apache.ratis:ratis-hadoop</include>
- <include>org.apache.ratis:ratis-logservice</include>
<include>org.apache.ratis:ratis-netty</include>
<include>org.apache.ratis:ratis-proto</include>
<include>org.apache.ratis:ratis-docs</include>
diff --git a/ratis-examples/pom.xml b/ratis-examples/pom.xml
index bb9c1a4..c8a37e8 100644
--- a/ratis-examples/pom.xml
+++ b/ratis-examples/pom.xml
@@ -65,18 +65,6 @@
</dependency>
<dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
<artifactId>ratis-grpc</artifactId>
<groupId>org.apache.ratis</groupId>
</dependency>
diff --git a/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java b/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java
index ba4cdc2..72f4ee4 100644
--- a/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java
+++ b/ratis-examples/src/test/java/org/apache/ratis/examples/ParameterizedBaseTest.java
@@ -22,7 +22,6 @@ import org.apache.ratis.server.impl.MiniRaftCluster;
import org.apache.ratis.RaftTestUtil;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.MiniRaftClusterWithGrpc;
-import org.apache.ratis.hadooprpc.MiniRaftClusterWithHadoopRpc;
import org.apache.ratis.netty.MiniRaftClusterWithNetty;
import org.apache.ratis.server.RaftServerConfigKeys;
import org.apache.ratis.server.simulation.MiniRaftClusterWithSimulatedRpc;
@@ -109,9 +108,6 @@ public abstract class ParameterizedBaseTest extends BaseTest {
if (isAll || classes.contains(MiniRaftClusterWithNetty.class)) {
add(clusters, MiniRaftClusterWithNetty.FACTORY, ids.next(), prop);
}
- if (isAll || classes.contains(MiniRaftClusterWithHadoopRpc.class)) {
-// add(clusters, MiniRaftClusterWithHadoopRpc.FACTORY, ids.next(), prop);
- }
for(int i = 0; i < clusters.size(); i++) {
LOG.info(i + ": " + JavaUtils.getClassSimpleName(clusters.get(i)[0].getClass()));
}
diff --git a/ratis-hadoop/dev-support/findbugsExcludeFile.xml b/ratis-hadoop/dev-support/findbugsExcludeFile.xml
deleted file mode 100644
index c76f7d5..0000000
--- a/ratis-hadoop/dev-support/findbugsExcludeFile.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
- <Match>
- <Package name="org.apache.ratis.proto.hadoop"/>
- </Match>
-</FindBugsFilter>
diff --git a/ratis-hadoop/pom.xml b/ratis-hadoop/pom.xml
deleted file mode 100644
index f1e5776..0000000
--- a/ratis-hadoop/pom.xml
+++ /dev/null
@@ -1,138 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>ratis</artifactId>
- <groupId>org.apache.ratis</groupId>
- <version>2.3.0-SNAPSHOT</version>
- </parent>
-
- <artifactId>ratis-hadoop</artifactId>
- <name>Apache Ratis Hadoop Support</name>
-
- <dependencies>
- <dependency>
- <artifactId>ratis-proto</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
-
- <dependency>
- <artifactId>ratis-common</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-common</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <artifactId>ratis-client</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <artifactId>ratis-client</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <artifactId>ratis-server</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <artifactId>ratis-server</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-annotations</artifactId>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <configuration>
- <protocArtifact>
- com.google.protobuf:protoc:${hadoop.protobuf.version}:exe:${os.detected.classifier}
- </protocArtifact>
- <!-- Place these in a location that compiler-plugin is already looking -->
- <outputDirectory>${project.build.directory}/generated-sources</outputDirectory>
- <!-- With multiple executions, this must be `false` otherwise we wipe out the previous execution -->
- <clearOutputDirectory>false</clearOutputDirectory>
- </configuration>
- <executions>
- <execution>
- <id>compile-protobuf</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <skipTests>true</skipTests>
- </configuration>
- </plugin>
- <plugin>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-maven-plugin</artifactId>
- <configuration>
- <excludeFilterFile>${project.basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopConfigKeys.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopConfigKeys.java
deleted file mode 100644
index b399ec4..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopConfigKeys.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.ratis.conf.Parameters;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-import java.util.function.Consumer;
-
-import static org.apache.ratis.conf.ConfUtils.*;
-
-/** Hadoop Rpc specific configuration properties. */
-public interface HadoopConfigKeys {
- String PREFIX = "raft.hadoop";
-
- String CONF_PARAMETER = PREFIX + ".conf";
- Class<Configuration> CONF_CLASS = Configuration.class;
-
- static Configuration getConf(Parameters parameters) {
- return parameters.get(CONF_PARAMETER, CONF_CLASS);
- }
-
- static void setConf(Parameters parameters, Configuration conf) {
- parameters.put(CONF_PARAMETER, conf, Configuration.class);
- }
-
- /** IPC server configurations */
- interface Ipc {
- Logger LOG = LoggerFactory.getLogger(Ipc.class);
- static Consumer<String> getDefaultLog() {
- return LOG::info;
- }
-
- String PREFIX = HadoopConfigKeys.PREFIX + ".ipc";
-
- String ADDRESS_KEY = PREFIX + ".address";
- int PORT_DEFAULT = 10718;
- String ADDRESS_DEFAULT = "0.0.0.0:" + PORT_DEFAULT;
-
- String HANDLERS_KEY = PREFIX + ".handlers";
- int HANDLERS_DEFAULT = 10;
-
- static int handlers(Configuration conf) {
- return getInt(conf::getInt, HANDLERS_KEY, HANDLERS_DEFAULT, getDefaultLog(), requireMin(1));
- }
-
- static void setHandlers(Configuration conf, int handers) {
- set(conf::setInt, HANDLERS_KEY, handers);
- }
-
- static InetSocketAddress address(Configuration conf) {
- return getInetSocketAddress(conf::getTrimmed, ADDRESS_KEY, ADDRESS_DEFAULT, getDefaultLog());
- }
-
- static void setAddress(Configuration conf, String address) {
- set(conf::set, ADDRESS_KEY, address);
- }
- }
-
- static void main(String[] args) {
- printAll(HadoopConfigKeys.class);
- }
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopConstants.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopConstants.java
deleted file mode 100644
index c1672bd..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopConstants.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-public final class HadoopConstants {
- private HadoopConstants() {
- //Never constructed
- }
- public static final String RAFT_SERVER_KERBEROS_PRINCIPAL_KEY = "raft.server.kerberos.principal";
- public static final String RAFT_CLIENT_KERBEROS_PRINCIPAL_KEY = "raft.client.kerberos.principal";
- public static final String RAFT_SERVER_PROTOCOL_NAME = "org.apache.hadoop.raft.server.protocol.RaftServerProtocol";
- public static final String COMBINED_CLIENT_PROTOCOL_NAME = "org.apache.ratis.hadooprpc.client.CombinedClientProtocol";
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopFactory.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopFactory.java
deleted file mode 100644
index 6cdfd36..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/HadoopFactory.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.ratis.client.ClientFactory;
-import org.apache.ratis.conf.Parameters;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.hadooprpc.client.HadoopClientRpc;
-import org.apache.ratis.hadooprpc.server.HadoopRpcService;
-import org.apache.ratis.protocol.ClientId;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.ServerFactory;
-
-public class HadoopFactory implements ServerFactory, ClientFactory {
- public static Parameters newRaftParameters(Configuration conf) {
- final Parameters p = new Parameters();
- HadoopConfigKeys.setConf(p, conf);
- return p;
- }
-
- private final Configuration conf;
-
- public HadoopFactory(Parameters parameters) {
- this(HadoopConfigKeys.getConf(parameters));
- }
-
- public HadoopFactory(Configuration conf) {
- this.conf = conf != null? conf: new Configuration();
- }
-
- public Configuration getConf() {
- return conf;
- }
-
- @Override
- public SupportedRpcType getRpcType() {
- return SupportedRpcType.HADOOP;
- }
-
- @Override
- public HadoopRpcService newRaftServerRpc(RaftServer server) {
- return HadoopRpcService.newBuilder()
- .setServer(server)
- .setConf(getConf())
- .build();
- }
-
- @Override
- public HadoopClientRpc newRaftClientRpc(ClientId clientId, RaftProperties properties) {
- return new HadoopClientRpc(clientId, getConf());
- }
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/Proxy.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/Proxy.java
deleted file mode 100644
index b9c32e7..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/Proxy.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-public class Proxy<PROTOCOL> implements Closeable {
- public static <PROTOCOL> PROTOCOL getProxy(
- Class<PROTOCOL> clazz, String addressStr, Configuration conf)
- throws IOException {
- RPC.setProtocolEngine(conf, clazz, ProtobufRpcEngine.class);
- return RPC.getProxy(clazz, RPC.getProtocolVersion(clazz),
- org.apache.ratis.util.NetUtils.createSocketAddr(addressStr),
- UserGroupInformation.getCurrentUser(),
- conf, NetUtils.getSocketFactory(conf, clazz));
- }
-
- private final PROTOCOL protocol;
-
- public Proxy(Class<PROTOCOL> clazz, String addressStr, Configuration conf)
- throws IOException {
- this.protocol = getProxy(clazz, addressStr, conf);
- }
-
- public PROTOCOL getProtocol() {
- return protocol;
- }
-
- @Override
- public void close() {
- RPC.stopProxy(protocol);
- }
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocol.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocol.java
deleted file mode 100644
index 6281987..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocol.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.client;
-
-import org.apache.ratis.protocol.AdminProtocol;
-import org.apache.ratis.protocol.RaftClientProtocol;
-
-public interface CombinedClientProtocol
- extends RaftClientProtocol, AdminProtocol {
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolClientSideTranslatorPB.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolClientSideTranslatorPB.java
deleted file mode 100644
index ac0f30d..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.client;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.ratis.client.impl.ClientProtoUtils;
-import org.apache.ratis.hadooprpc.Proxy;
-import org.apache.ratis.proto.RaftProtos;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ClientRequestProto;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ClientOps;
-import org.apache.ratis.protocol.GroupInfoReply;
-import org.apache.ratis.protocol.GroupInfoRequest;
-import org.apache.ratis.protocol.GroupListReply;
-import org.apache.ratis.protocol.GroupListRequest;
-import org.apache.ratis.protocol.GroupManagementRequest;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.apache.ratis.protocol.RaftClientRequest;
-import org.apache.ratis.protocol.SetConfigurationRequest;
-import org.apache.ratis.protocol.SnapshotManagementRequest;
-import org.apache.ratis.protocol.TransferLeadershipRequest;
-import org.apache.ratis.thirdparty.com.google.protobuf
- .GeneratedMessageV3;
-import org.apache.ratis.thirdparty.com.google.protobuf
- .InvalidProtocolBufferException;
-import org.apache.ratis.util.function.CheckedFunction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.function.Function;
-
-
-@InterfaceAudience.Private
-public class CombinedClientProtocolClientSideTranslatorPB
- extends Proxy<CombinedClientProtocolPB>
- implements CombinedClientProtocol {
- private static final Logger LOG = LoggerFactory.getLogger(CombinedClientProtocolClientSideTranslatorPB.class);
-
- public CombinedClientProtocolClientSideTranslatorPB(
- String addressStr, Configuration conf) throws IOException {
- super(CombinedClientProtocolPB.class, addressStr, conf);
- }
-
- @Override
- public RaftClientReply submitClientRequest(RaftClientRequest request)
- throws IOException {
- return handleRequest(request,
- ClientProtoUtils::toRaftClientRequestProto,
- ClientProtoUtils::toRaftClientReply,
- ClientOps.submitClientRequest,
- RaftProtos.RaftClientReplyProto::parseFrom);
- }
-
- @Override
- public RaftClientReply setConfiguration(SetConfigurationRequest request)
- throws IOException {
- return handleRequest(request,
- ClientProtoUtils::toSetConfigurationRequestProto,
- ClientProtoUtils::toRaftClientReply,
- ClientOps.setConfiguration,
- RaftProtos.RaftClientReplyProto::parseFrom);
- }
-
- @Override
- public RaftClientReply transferLeadership(TransferLeadershipRequest request)
- throws IOException {
- return handleRequest(request,
- ClientProtoUtils::toTransferLeadershipRequestProto,
- ClientProtoUtils::toRaftClientReply,
- ClientOps.transferLeadership,
- RaftProtos.RaftClientReplyProto::parseFrom);
- }
-
- @Override
- public RaftClientReply groupManagement(GroupManagementRequest request) throws IOException {
- return handleRequest(request,
- ClientProtoUtils::toGroupManagementRequestProto,
- ClientProtoUtils::toRaftClientReply,
- ClientOps.groupManagement,
- RaftProtos.RaftClientReplyProto::parseFrom);
- }
-
- @Override
- public RaftClientReply snapshotManagement(SnapshotManagementRequest request) throws IOException {
- return handleRequest(request,
- ClientProtoUtils::toSnapshotManagementRequestProto,
- ClientProtoUtils::toRaftClientReply,
- ClientOps.snapshotManagement,
- RaftProtos.RaftClientReplyProto::parseFrom);
- }
-
- @Override
- public GroupListReply getGroupList(GroupListRequest request) throws IOException {
- return handleRequest(request,
- ClientProtoUtils::toGroupListRequestProto,
- ClientProtoUtils::toGroupListReply,
- ClientOps.groupList,
- RaftProtos.GroupListReplyProto::parseFrom);
- }
-
- @Override
- public GroupInfoReply getGroupInfo(GroupInfoRequest request) throws IOException {
- return handleRequest(request,
- ClientProtoUtils::toGroupInfoRequestProto,
- ClientProtoUtils::toGroupInfoReply,
- ClientOps.groupInfo,
- RaftProtos.GroupInfoReplyProto::parseFrom);
- }
-
- <REQUEST extends RaftClientRequest,
- REPLY extends RaftClientReply,
- PROTO_REQ extends GeneratedMessageV3,
- PROTO_REP extends GeneratedMessageV3> REPLY handleRequest(
- REQUEST request,
- Function<REQUEST, PROTO_REQ> reqToProto,
- Function<PROTO_REP, REPLY> repToProto,
- ClientOps type,
- CheckedFunction<byte[], PROTO_REP, InvalidProtocolBufferException> byteToProto)
- throws IOException {
- final PROTO_REQ proto = reqToProto.apply(request);
- try {
- ClientRequestProto req = ClientRequestProto.newBuilder()
- .setType(type)
- .setRequest(ByteString.copyFrom(proto.toByteArray()))
- .build();
- byte[] reply = getProtocol().sendClient(null, req)
- .getResponse().toByteArray();
-
- PROTO_REP replyProto = byteToProto.apply(reply);
- return repToProto.apply(replyProto);
- } catch (ServiceException se) {
- LOG.trace("Failed to handle " + request, se);
- throw new IOException(se);
- }
- }
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolPB.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolPB.java
deleted file mode 100644
index a6b0573..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolPB.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.client;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.ratis.hadooprpc.HadoopConstants;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.HadoopClientProtocolService;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-@KerberosInfo(
- serverPrincipal = HadoopConstants.RAFT_SERVER_KERBEROS_PRINCIPAL_KEY,
- clientPrincipal = HadoopConstants.RAFT_CLIENT_KERBEROS_PRINCIPAL_KEY)
-@ProtocolInfo(
- protocolName = HadoopConstants.COMBINED_CLIENT_PROTOCOL_NAME,
- protocolVersion = 1)
-public interface CombinedClientProtocolPB extends
- HadoopClientProtocolService.BlockingInterface {
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolServerSideTranslatorPB.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 47d85f8..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/CombinedClientProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.client;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ServiceException;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.ratis.client.impl.ClientProtoUtils;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ClientReplyProto;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ClientRequestProto;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ClientOps;
-import org.apache.ratis.protocol.*;
-import org.apache.ratis.server.RaftServer;
-import com.google.protobuf.RpcController;
-import org.apache.ratis.proto.RaftProtos.RaftClientReplyProto;
-import org.apache.ratis.proto.RaftProtos.RaftClientRequestProto;
-import org.apache.ratis.proto.RaftProtos.SetConfigurationRequestProto;
-import org.apache.ratis.proto.RaftProtos.GroupManagementRequestProto;
-import org.apache.ratis.proto.RaftProtos.GroupListRequestProto;
-import org.apache.ratis.proto.RaftProtos.GroupListReplyProto;
-import org.apache.ratis.proto.RaftProtos.GroupInfoRequestProto;
-import org.apache.ratis.proto.RaftProtos.GroupInfoReplyProto;
-import org.apache.ratis.proto.RaftProtos.TransferLeadershipRequestProto;
-import org.apache.ratis.proto.RaftProtos.SnapshotManagementRequestProto;
-import org.apache.ratis.thirdparty.com.google.protobuf.GeneratedMessageV3;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-@InterfaceAudience.Private
-public class CombinedClientProtocolServerSideTranslatorPB
- implements CombinedClientProtocolPB {
- private Logger log = LoggerFactory.getLogger(CombinedClientProtocolServerSideTranslatorPB.class);
- private final RaftServer impl;
-
- public CombinedClientProtocolServerSideTranslatorPB(RaftServer impl) {
- this.impl = impl;
- }
-
- @Override
- @SuppressFBWarnings("NP_NULL_ON_SOME_PATH")
- public ClientReplyProto sendClient(RpcController unused, ClientRequestProto req) throws ServiceException {
- ByteBuffer buf = req.getRequest().asReadOnlyByteBuffer();
- GeneratedMessageV3 response = null;
- ClientOps type = req.getType();
- try {
- switch (type) {
- case groupInfo:
- response = groupInfo(GroupInfoRequestProto.parseFrom(buf));
- break;
- case groupList:
- response = groupList(GroupListRequestProto.parseFrom(buf));
- break;
- case groupManagement:
- response = groupManagement(GroupManagementRequestProto.parseFrom(buf));
- break;
- case setConfiguration:
- response = setConfiguration(SetConfigurationRequestProto.parseFrom(buf));
- break;
- case submitClientRequest:
- response = submitClientRequest(RaftClientRequestProto.parseFrom(buf));
- break;
- case transferLeadership:
- response = transferLeadership(TransferLeadershipRequestProto.parseFrom(buf));
- break;
- case snapshotManagement:
- response = snapshotManagement(SnapshotManagementRequestProto.parseFrom(buf));
- break;
- default:
- String message = "Internal error, all response types are not being handled as expected. " +
- "Developer: check that all response types have appropriate handlers.";
- log.error(message);
- throw new ServiceException(message);
- }
- } catch(IOException ioe) {
- throw new ServiceException(ioe);
- }
- return ClientReplyProto.newBuilder()
- .setType(type)
- .setResponse(ByteString.copyFrom(response.toByteArray()))
- .build();
- }
-
- public RaftClientReplyProto submitClientRequest(RaftClientRequestProto proto)
- throws IOException {
- final RaftClientRequest request = ClientProtoUtils.toRaftClientRequest(proto);
- final RaftClientReply reply = impl.submitClientRequest(request);
- return ClientProtoUtils.toRaftClientReplyProto(reply);
- }
-
- public RaftClientReplyProto setConfiguration(SetConfigurationRequestProto proto)
- throws IOException {
- final SetConfigurationRequest request = ClientProtoUtils.toSetConfigurationRequest(proto);
- final RaftClientReply reply = impl.setConfiguration(request);
- return ClientProtoUtils.toRaftClientReplyProto(reply);
- }
-
- public RaftClientReplyProto groupManagement(GroupManagementRequestProto proto)
- throws IOException {
- final GroupManagementRequest request = ClientProtoUtils.toGroupManagementRequest(proto);
- final RaftClientReply reply = impl.groupManagement(request);
- return ClientProtoUtils.toRaftClientReplyProto(reply);
- }
-
- public GroupListReplyProto groupList(GroupListRequestProto proto)
- throws IOException {
- final GroupListRequest request = ClientProtoUtils.toGroupListRequest(proto);
- final GroupListReply reply = impl.getGroupList(request);
- return ClientProtoUtils.toGroupListReplyProto(reply);
- }
-
- public GroupInfoReplyProto groupInfo(GroupInfoRequestProto proto)
- throws IOException {
- final GroupInfoRequest request = ClientProtoUtils.toGroupInfoRequest(proto);
- final GroupInfoReply reply = impl.getGroupInfo(request);
- return ClientProtoUtils.toGroupInfoReplyProto(reply);
- }
-
- public RaftClientReplyProto transferLeadership(TransferLeadershipRequestProto proto)
- throws IOException {
- final TransferLeadershipRequest request = ClientProtoUtils.toTransferLeadershipRequest(proto);
- final RaftClientReply reply = impl.transferLeadership(request);
- return ClientProtoUtils.toRaftClientReplyProto(reply);
- }
-
- public RaftClientReplyProto snapshotManagement(SnapshotManagementRequestProto proto)
- throws IOException {
- final SnapshotManagementRequest request = ClientProtoUtils.toSnapshotManagementRequest(proto);
- final RaftClientReply reply = impl.snapshotManagement(request);
- return ClientProtoUtils.toRaftClientReplyProto(reply);
- }
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/HadoopClientRpc.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/HadoopClientRpc.java
deleted file mode 100644
index 36a5adc..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/client/HadoopClientRpc.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.ratis.client.impl.RaftClientRpcWithProxy;
-import org.apache.ratis.protocol.*;
-import org.apache.ratis.protocol.exceptions.GroupMismatchException;
-import org.apache.ratis.protocol.exceptions.LeaderNotReadyException;
-import org.apache.ratis.protocol.exceptions.RaftException;
-import org.apache.ratis.protocol.exceptions.ReconfigurationInProgressException;
-import org.apache.ratis.protocol.exceptions.ReconfigurationTimeoutException;
-import org.apache.ratis.protocol.exceptions.StateMachineException;
-import org.apache.ratis.util.PeerProxyMap;
-
-import java.io.IOException;
-
-public class HadoopClientRpc extends RaftClientRpcWithProxy<CombinedClientProtocolClientSideTranslatorPB> {
- public HadoopClientRpc(ClientId clientId, Configuration conf) {
- super(new PeerProxyMap<>(clientId.toString(),
- p -> new CombinedClientProtocolClientSideTranslatorPB(p.getAddress(), conf)));
- }
-
- @Override
- public RaftClientReply sendRequest(RaftClientRequest request)
- throws IOException {
- final RaftPeerId serverId = request.getServerId();
- final CombinedClientProtocolClientSideTranslatorPB proxy =
- getProxies().getProxy(serverId);
- try {
- if (request instanceof GroupManagementRequest) {
- return proxy.groupManagement((GroupManagementRequest) request);
- } else if (request instanceof SetConfigurationRequest) {
- return proxy.setConfiguration((SetConfigurationRequest) request);
- } else if (request instanceof GroupListRequest) {
- return proxy.getGroupList((GroupListRequest) request);
- } else if (request instanceof GroupInfoRequest) {
- return proxy.getGroupInfo((GroupInfoRequest) request);
- } else if (request instanceof TransferLeadershipRequest) {
- return proxy.transferLeadership((TransferLeadershipRequest) request);
- } else if (request instanceof SnapshotManagementRequest) {
- return proxy.snapshotManagement((SnapshotManagementRequest) request);
- } else {
- return proxy.submitClientRequest(request);
- }
- } catch (RemoteException e) {
- throw e.unwrapRemoteException(
- StateMachineException.class,
- ReconfigurationTimeoutException.class,
- ReconfigurationInProgressException.class,
- RaftException.class,
- LeaderNotReadyException.class,
- GroupMismatchException.class);
- }
- }
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/HadoopRpcService.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/HadoopRpcService.java
deleted file mode 100644
index 52c2c31..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/HadoopRpcService.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.server;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.ratis.hadooprpc.HadoopConfigKeys;
-import org.apache.ratis.hadooprpc.Proxy;
-import org.apache.ratis.hadooprpc.client.CombinedClientProtocolPB;
-import org.apache.ratis.hadooprpc.client.CombinedClientProtocolServerSideTranslatorPB;
-import org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto;
-import org.apache.ratis.proto.RaftProtos.AppendEntriesRequestProto;
-import org.apache.ratis.proto.RaftProtos.InstallSnapshotReplyProto;
-import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto;
-import org.apache.ratis.proto.RaftProtos.RequestVoteReplyProto;
-import org.apache.ratis.proto.RaftProtos.RequestVoteRequestProto;
-import org.apache.ratis.proto.RaftProtos.StartLeaderElectionReplyProto;
-import org.apache.ratis.proto.RaftProtos.StartLeaderElectionRequestProto;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.HadoopServerProtocolService;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.HadoopClientProtocolService;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ServerOps;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ServerRequestProto;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.RaftServerRpcWithProxy;
-import org.apache.ratis.server.protocol.RaftServerProtocol;
-import com.google.protobuf.BlockingService;
-import com.google.protobuf.ByteString;
-import org.apache.ratis.thirdparty.com.google.protobuf.GeneratedMessageV3;
-import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.ratis.util.CodeInjectionForTesting;
-import org.apache.ratis.util.JavaUtils;
-import org.apache.ratis.util.PeerProxyMap;
-import org.apache.ratis.util.function.CheckedFunction;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-/** Server side Hadoop RPC service. */
-public final class HadoopRpcService extends RaftServerRpcWithProxy<Proxy<RaftServerProtocolPB>,
- PeerProxyMap<Proxy<RaftServerProtocolPB>>> {
- public static final Logger LOG = LoggerFactory.getLogger(HadoopRpcService.class);
- static final String CLASS_NAME = JavaUtils.getClassSimpleName(HadoopRpcService.class);
- public static final String SEND_SERVER_REQUEST = CLASS_NAME + ".sendServerRequest";
-
- public static final class Builder {
- private RaftServer server;
- private Configuration conf;
-
- private Builder() {}
-
- public Configuration getConf() {
- if (conf == null) {
- conf = new Configuration();
- }
- return conf;
- }
-
- public Builder setConf(Configuration conf) {
- this.conf = conf;
- return this;
- }
-
- public Builder setServer(RaftServer raftServer) {
- this.server = raftServer;
- return this;
- }
-
- public HadoopRpcService build() {
- return new HadoopRpcService(server, getConf());
- }
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- private final RPC.Server ipcServer;
- private final InetSocketAddress ipcServerAddress;
-
- private HadoopRpcService(RaftServer server, final Configuration conf) {
- super(server::getId,
- id -> new PeerProxyMap<>(id.toString(),
- p -> new Proxy<>(RaftServerProtocolPB.class, p.getAddress(), conf)));
- try {
- this.ipcServer = newRpcServer(server, conf);
- } catch (IOException e) {
- throw new RuntimeException("Failed to create Hadoop rpc server.", e);
- }
- this.ipcServerAddress = ipcServer.getListenerAddress();
-
- addRaftClientProtocol(server, conf);
-
- LOG.info(JavaUtils.getClassSimpleName(getClass()) + " created RPC.Server at " + ipcServerAddress);
- }
-
- @Override
- public SupportedRpcType getRpcType() {
- return SupportedRpcType.HADOOP;
- }
-
- @Override
- public InetSocketAddress getInetSocketAddress() {
- return ipcServerAddress;
- }
-
- private static RPC.Server newRpcServer(
- RaftServerProtocol serverProtocol, final Configuration conf)
- throws IOException {
- final int handlerCount = HadoopConfigKeys.Ipc.handlers(conf);
- final InetSocketAddress address = HadoopConfigKeys.Ipc.address(conf);
-
- final BlockingService service
- = HadoopServerProtocolService.newReflectiveBlockingService(
- new RaftServerProtocolServerSideTranslatorPB(serverProtocol));
- RPC.setProtocolEngine(conf, RaftServerProtocolPB.class, ProtobufRpcEngine.class);
- return new RPC.Builder(conf)
- .setProtocol(RaftServerProtocolPB.class)
- .setInstance(service)
- .setBindAddress(address.getHostName())
- .setPort(address.getPort())
- .setNumHandlers(handlerCount)
- .setVerbose(false)
- .build();
- }
-
- private void addRaftClientProtocol(RaftServer server, Configuration conf) {
- final Class<?> protocol = CombinedClientProtocolPB.class;
- RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class);
-
- final BlockingService service = HadoopClientProtocolService.newReflectiveBlockingService(
- new CombinedClientProtocolServerSideTranslatorPB(server));
- ipcServer.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, protocol, service);
- }
-
- @Override
- public void startImpl() {
- ipcServer.start();
- }
-
- @Override
- public void closeImpl() throws IOException {
- ipcServer.stop();
- super.closeImpl();
- }
-
- @Override
- public AppendEntriesReplyProto appendEntries(
- AppendEntriesRequestProto request) throws IOException {
- return processRequest(request, request.getServerRequest().getReplyId(),
- ServerOps.appendEntries,
- AppendEntriesReplyProto::parseFrom);
- }
-
- @Override
- public InstallSnapshotReplyProto installSnapshot(
- InstallSnapshotRequestProto request) throws IOException {
- return processRequest(request, request.getServerRequest().getReplyId(),
- ServerOps.installSnapshot, InstallSnapshotReplyProto::parseFrom);
- }
-
- @Override
- public RequestVoteReplyProto requestVote(
- RequestVoteRequestProto request) throws IOException {
- return processRequest(request, request.getServerRequest().getReplyId(),
- ServerOps.requestVote, RequestVoteReplyProto::parseFrom);
- }
-
- @Override
- public StartLeaderElectionReplyProto startLeaderElection(
- StartLeaderElectionRequestProto request) throws IOException {
- return processRequest(request, request.getServerRequest().getReplyId(),
- ServerOps.startLeaderElection, StartLeaderElectionReplyProto::parseFrom);
- }
-
- private <REQUEST extends GeneratedMessageV3, REPLY> REPLY processRequest(
- REQUEST request, org.apache.ratis.thirdparty.com.google.protobuf.ByteString replyId,
- ServerOps type, CheckedFunction<byte[], REPLY, InvalidProtocolBufferException> func)
- throws IOException {
- CodeInjectionForTesting.execute(SEND_SERVER_REQUEST, getId(), null, request);
- ServerRequestProto p = ServerRequestProto.newBuilder()
- .setRequest(ByteString.copyFrom(request.toByteArray()))
- .setType(type)
- .build();
-
- final RaftServerProtocolPB proxy = getProxies().getProxy(
- RaftPeerId.valueOf(replyId)).getProtocol();
- try {
- byte[] replyBytes = proxy.sendServer(null, p).getResponse().toByteArray();
- return func.apply(replyBytes);
- } catch (Exception se) {
- throw new IOException(se);
- }
- }
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/RaftServerProtocolPB.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/RaftServerProtocolPB.java
deleted file mode 100644
index 9caafdc..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/RaftServerProtocolPB.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.server;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-import org.apache.ratis.hadooprpc.HadoopConstants;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.HadoopServerProtocolService;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-@KerberosInfo(
- serverPrincipal = HadoopConstants.RAFT_SERVER_KERBEROS_PRINCIPAL_KEY,
- clientPrincipal = HadoopConstants.RAFT_SERVER_KERBEROS_PRINCIPAL_KEY)
-@ProtocolInfo(
- protocolName = HadoopConstants.RAFT_SERVER_PROTOCOL_NAME,
- protocolVersion = 1)
-public interface RaftServerProtocolPB extends
- HadoopServerProtocolService.BlockingInterface {
-}
diff --git a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/RaftServerProtocolServerSideTranslatorPB.java b/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/RaftServerProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 9f61207..0000000
--- a/ratis-hadoop/src/main/java/org/apache/ratis/hadooprpc/server/RaftServerProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc.server;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ServerOps;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ServerReplyProto;
-import org.apache.ratis.proto.hadoop.HadoopCompatibilityProtos.ServerRequestProto;
-import org.apache.ratis.server.protocol.RaftServerProtocol;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.ratis.proto.RaftProtos.AppendEntriesReplyProto;
-import org.apache.ratis.proto.RaftProtos.AppendEntriesRequestProto;
-import org.apache.ratis.proto.RaftProtos.InstallSnapshotReplyProto;
-import org.apache.ratis.proto.RaftProtos.InstallSnapshotRequestProto;
-import org.apache.ratis.proto.RaftProtos.RequestVoteReplyProto;
-import org.apache.ratis.proto.RaftProtos.RequestVoteRequestProto;
-import org.apache.ratis.proto.RaftProtos.StartLeaderElectionReplyProto;
-import org.apache.ratis.proto.RaftProtos.StartLeaderElectionRequestProto;
-import org.apache.ratis.thirdparty.com.google.protobuf.GeneratedMessageV3;
-
-@InterfaceAudience.Private
-public class RaftServerProtocolServerSideTranslatorPB
- implements RaftServerProtocolPB {
- private final RaftServerProtocol impl;
-
- public RaftServerProtocolServerSideTranslatorPB(RaftServerProtocol impl) {
- this.impl = impl;
- }
-
- @Override
- public ServerReplyProto sendServer(RpcController unused, ServerRequestProto requestProto) throws ServiceException {
- ServerOps type = requestProto.getType();
- ByteBuffer buffer = requestProto.getRequest().asReadOnlyByteBuffer();
- GeneratedMessageV3 respone = null;
- try {
- switch (type) {
- case requestVote:
- respone = requestVote(RequestVoteRequestProto.parseFrom(buffer));
- break;
- case startLeaderElection:
- respone = startLeaderElection(StartLeaderElectionRequestProto.parseFrom(buffer));
- break;
- case installSnapshot:
- respone = installSnapshot(InstallSnapshotRequestProto.parseFrom(buffer));
- break;
- case appendEntries:
- respone = appendEntries(AppendEntriesRequestProto.parseFrom(buffer));
- break;
- default:
- throw new IOException("Invalid Request Type:" + type);
- }
- return ServerReplyProto.newBuilder()
- .setType(type)
- .setResponse(com.google.protobuf.ByteString.copyFrom(respone.toByteArray()))
- .build();
- } catch(IOException ioe) {
- throw new ServiceException(ioe);
- }
- }
-
- public RequestVoteReplyProto requestVote(RequestVoteRequestProto request)
- throws IOException {
- return impl.requestVote(request);
- }
-
- public StartLeaderElectionReplyProto startLeaderElection(StartLeaderElectionRequestProto request) throws IOException {
- return impl.startLeaderElection(request);
- }
-
- public AppendEntriesReplyProto appendEntries(AppendEntriesRequestProto request)
- throws IOException {
- return impl.appendEntries(request);
- }
-
- public InstallSnapshotReplyProto installSnapshot(
- InstallSnapshotRequestProto request) throws IOException {
- return impl.installSnapshot(request);
- }
-}
diff --git a/ratis-hadoop/src/main/proto/HadoopCompatability.proto b/ratis-hadoop/src/main/proto/HadoopCompatability.proto
deleted file mode 100644
index f17fa2b..0000000
--- a/ratis-hadoop/src/main/proto/HadoopCompatability.proto
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-syntax = "proto2";
-option java_package = "org.apache.ratis.proto.hadoop";
-option java_outer_classname = "HadoopCompatibilityProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package ratis.hadoop;
-
-/**
- * This proto file is a replacement for Hadoop.proto.
- * The problem occurs because Hadoop's depends on protobuf 2.5.0.
- * Where as Ratis uses protbuf 3.5.0 or greater
- * This proto create a bytestring format to transfer the protos
- * and then convert the bytestring to v3.5.0 proto at the rpc endpoint.
- */
-
-enum ServerOps {
- requestVote = 1;
- appendEntries = 2;
- installSnapshot = 3;
- startLeaderElection = 4;
-}
-
-message ServerRequestProto {
- required ServerOps type = 1;
- required bytes request = 2;
-}
-
-message ServerReplyProto {
- required ServerOps type = 1;
- required bytes response = 2;
-}
-
-enum ClientOps {
- submitClientRequest = 1;
- setConfiguration = 2;
- groupManagement = 3;
- groupList = 4;
- groupInfo = 5;
- transferLeadership = 6;
- snapshotManagement = 7;
-}
-
-message ClientRequestProto {
- required ClientOps type = 1;
- required bytes request = 2;
-}
-
-message ClientReplyProto {
- required ClientOps type = 1;
- required bytes response = 2;
-}
-
-service HadoopServerProtocolService {
- rpc sendServer(ServerRequestProto) returns (ServerReplyProto);
-}
-
-service HadoopClientProtocolService {
- rpc sendClient(ClientRequestProto) returns (ClientReplyProto);
-}
\ No newline at end of file
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/MiniRaftClusterWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/MiniRaftClusterWithHadoopRpc.java
deleted file mode 100644
index e3d5920..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/MiniRaftClusterWithHadoopRpc.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.ratis.server.impl.MiniRaftCluster;
-import org.apache.ratis.RaftConfigKeys;
-import org.apache.ratis.RaftTestUtil;
-import org.apache.ratis.conf.Parameters;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.hadooprpc.server.HadoopRpcService;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.server.impl.DelayLocalExecutionInjection;
-import org.apache.ratis.util.JavaUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class MiniRaftClusterWithHadoopRpc extends MiniRaftCluster.RpcBase {
- static final Logger LOG = LoggerFactory.getLogger(MiniRaftClusterWithHadoopRpc.class);
-
- public static class Factory extends MiniRaftCluster.Factory<MiniRaftClusterWithHadoopRpc> {
- public interface Get extends MiniRaftCluster.Factory.Get<MiniRaftClusterWithHadoopRpc> {
- @Override
- default MiniRaftCluster.Factory<MiniRaftClusterWithHadoopRpc> getFactory() {
- return FACTORY;
- }
- }
-
- @Override
- public MiniRaftClusterWithHadoopRpc newCluster(String[] ids, RaftProperties prop) {
- final Configuration conf = new Configuration();
- return newCluster(ids, prop, conf);
- }
-
- public MiniRaftClusterWithHadoopRpc newCluster(
- int numServers, RaftProperties properties, Configuration conf) {
- return newCluster(generateIds(numServers, 0), properties, conf);
- }
-
- public MiniRaftClusterWithHadoopRpc newCluster(
- String[] ids, RaftProperties prop, Configuration conf) {
- RaftConfigKeys.Rpc.setType(prop, SupportedRpcType.HADOOP);
- HadoopConfigKeys.Ipc.setAddress(conf, "0.0.0.0:0");
- return new MiniRaftClusterWithHadoopRpc(ids, prop, conf);
- }
- }
-
- public static final Factory FACTORY = new Factory();
-
- public static final DelayLocalExecutionInjection sendServerRequest =
- new DelayLocalExecutionInjection(HadoopRpcService.SEND_SERVER_REQUEST);
-
- private final Configuration hadoopConf;
-
- private MiniRaftClusterWithHadoopRpc(String[] ids, RaftProperties properties,
- Configuration hadoopConf) {
- super(ids, properties, HadoopFactory.newRaftParameters(hadoopConf));
- this.hadoopConf = hadoopConf;
- }
-
- @Override
- protected Parameters setPropertiesAndInitParameters(RaftPeerId id, RaftGroup group, RaftProperties properties) {
- final Configuration hconf = new Configuration(hadoopConf);
- final String address = "0.0.0.0:" + getPort(id, group);
- HadoopConfigKeys.Ipc.setAddress(hconf, address);
- return HadoopFactory.newRaftParameters(hconf);
- }
-
- @Override
- public void blockQueueAndSetDelay(String leaderId, int delayMs)
- throws InterruptedException {
- RaftTestUtil.blockQueueAndSetDelay(getServers(), sendServerRequest,
- leaderId, delayMs, getTimeoutMax());
- }
-
- @Override
- public String toString() {
- return JavaUtils.getClassSimpleName(getClass()) + ": sendServerRequest=" + sendServerRequest;
- }
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestGroupInfoWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestGroupInfoWithHadoopRpc.java
deleted file mode 100644
index 03dda36..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestGroupInfoWithHadoopRpc.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.server.impl.GroupInfoBaseTest;
-
-public class TestGroupInfoWithHadoopRpc
- extends GroupInfoBaseTest<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get{
-}
\ No newline at end of file
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestGroupManagementWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestGroupManagementWithHadoopRpc.java
deleted file mode 100644
index d16ac38..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestGroupManagementWithHadoopRpc.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.server.impl.MiniRaftCluster;
-import org.apache.ratis.server.impl.GroupManagementBaseTest;
-
-public class TestGroupManagementWithHadoopRpc extends GroupManagementBaseTest {
- @Override
- public MiniRaftCluster.Factory<? extends MiniRaftCluster> getClusterFactory() {
- return MiniRaftClusterWithHadoopRpc.FACTORY;
- }
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestLeaderElectionWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestLeaderElectionWithHadoopRpc.java
deleted file mode 100644
index acf1a90..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestLeaderElectionWithHadoopRpc.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.log4j.Level;
-import org.apache.ratis.server.impl.BlockRequestHandlingInjection;
-import org.apache.ratis.server.impl.LeaderElectionTests;
-import org.apache.ratis.util.Log4jUtils;
-import org.junit.Test;
-
-public class TestLeaderElectionWithHadoopRpc
- extends LeaderElectionTests<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
- static {
- Log4jUtils.setLogLevel(MiniRaftClusterWithHadoopRpc.LOG, Level.DEBUG);
- }
-
- @Override
- public MiniRaftClusterWithHadoopRpc newCluster(int numPeers) {
- final Configuration conf = new Configuration();
- HadoopConfigKeys.Ipc.setHandlers(conf, 20);
- conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- conf.setInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, 1000);
- conf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
- return MiniRaftClusterWithHadoopRpc.FACTORY.newCluster(
- numPeers, getProperties(), conf);
- }
-
- @Override
- @Test
- public void testEnforceLeader() throws Exception {
- super.testEnforceLeader();
-
- MiniRaftClusterWithHadoopRpc.sendServerRequest.clear();
- BlockRequestHandlingInjection.getInstance().unblockAll();
- }
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestLogAppenderWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestLogAppenderWithHadoopRpc.java
deleted file mode 100644
index 48489ad..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestLogAppenderWithHadoopRpc.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.LogAppenderTests;
-
-public class TestLogAppenderWithHadoopRpc
- extends LogAppenderTests<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestPreAppendLeaderStepDownWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestPreAppendLeaderStepDownWithHadoopRpc.java
deleted file mode 100644
index e5e4e1d..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestPreAppendLeaderStepDownWithHadoopRpc.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.server.impl.PreAppendLeaderStepDownTest;
-
-public class TestPreAppendLeaderStepDownWithHadoopRpc
- extends PreAppendLeaderStepDownTest<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftExceptionWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftExceptionWithHadoopRpc.java
deleted file mode 100644
index f8b04f7..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftExceptionWithHadoopRpc.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.ratis.RaftExceptionBaseTest;
-
-public class TestRaftExceptionWithHadoopRpc
- extends RaftExceptionBaseTest<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
-
- @Override
- public MiniRaftClusterWithHadoopRpc newCluster(int numPeers) {
- final Configuration conf = new Configuration();
- HadoopConfigKeys.Ipc.setHandlers(conf, 20);
- conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- conf.setInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, 1000);
- conf.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
- return MiniRaftClusterWithHadoopRpc.FACTORY.newCluster(numPeers, getProperties(), conf);
- }
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftReconfigurationWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftReconfigurationWithHadoopRpc.java
deleted file mode 100644
index 6128a55..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftReconfigurationWithHadoopRpc.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.ratis.server.impl.RaftReconfigurationBaseTest;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_TIMEOUT_KEY;
-
-public class TestRaftReconfigurationWithHadoopRpc
- extends RaftReconfigurationBaseTest<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
-
- @Override
- public MiniRaftClusterWithHadoopRpc newCluster(int numPeers) {
- final Configuration hadoopConf = new Configuration();
- hadoopConf.setInt(IPC_CLIENT_CONNECT_TIMEOUT_KEY, 1000);
- hadoopConf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- return MiniRaftClusterWithHadoopRpc.FACTORY.newCluster(numPeers, getProperties(), hadoopConf);
- }
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftSnapshotWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftSnapshotWithHadoopRpc.java
deleted file mode 100644
index a2ed8ea..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftSnapshotWithHadoopRpc.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.server.impl.MiniRaftCluster;
-import org.apache.ratis.statemachine.RaftSnapshotBaseTest;
-
-public class TestRaftSnapshotWithHadoopRpc extends RaftSnapshotBaseTest {
- @Override
- public MiniRaftCluster.Factory<?> getFactory() {
- return MiniRaftClusterWithHadoopRpc.FACTORY;
- }
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftStateMachineExceptionWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftStateMachineExceptionWithHadoopRpc.java
deleted file mode 100644
index 6a8fd9d..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftStateMachineExceptionWithHadoopRpc.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.server.impl.RaftStateMachineExceptionTests;
-
-public class TestRaftStateMachineExceptionWithHadoopRpc extends
- RaftStateMachineExceptionTests<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftWithHadoopRpc.java
deleted file mode 100644
index f5c6dfa..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRaftWithHadoopRpc.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.log4j.Level;
-import org.apache.ratis.server.impl.MiniRaftCluster;
-import org.apache.ratis.RaftBasicTests;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.server.impl.BlockRequestHandlingInjection;
-import org.apache.ratis.util.Log4jUtils;
-import org.junit.Test;
-
-public class TestRaftWithHadoopRpc
- extends RaftBasicTests<MiniRaftClusterWithHadoopRpc> {
- static {
- Log4jUtils.setLogLevel(MiniRaftClusterWithHadoopRpc.LOG, Level.DEBUG);
- }
-
- static final Configuration CONF = new Configuration();
- static {
- HadoopConfigKeys.Ipc.setHandlers(CONF, 20);
- CONF.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
- CONF.setInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY, 1000);
- CONF.setInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY, 1000);
- }
- static final MiniRaftCluster.Factory<MiniRaftClusterWithHadoopRpc> FACTORY
- = new MiniRaftClusterWithHadoopRpc.Factory() {
- @Override
- public MiniRaftClusterWithHadoopRpc newCluster(String[] ids, RaftProperties prop) {
- return newCluster(ids, prop, CONF);
- }
- };
-
- @Override
- @Test
- public void testWithLoad() throws Exception {
- super.testWithLoad();
- BlockRequestHandlingInjection.getInstance().unblockAll();
- }
-
- @Override
- public MiniRaftCluster.Factory<MiniRaftClusterWithHadoopRpc> getFactory() {
- return FACTORY;
- }
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRetryCacheWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRetryCacheWithHadoopRpc.java
deleted file mode 100644
index 96caf16..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestRetryCacheWithHadoopRpc.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.RetryCacheTests;
-
-public class TestRetryCacheWithHadoopRpc
- extends RetryCacheTests<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
-}
diff --git a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestSnapshotManagementWithHadoopRpc.java b/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestSnapshotManagementWithHadoopRpc.java
deleted file mode 100644
index 7f1b85d..0000000
--- a/ratis-hadoop/src/test/java/org/apache/ratis/hadooprpc/TestSnapshotManagementWithHadoopRpc.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.hadooprpc;
-
-import org.apache.ratis.statemachine.SnapshotManagementTest;
-
-public class TestSnapshotManagementWithHadoopRpc
- extends SnapshotManagementTest<MiniRaftClusterWithHadoopRpc>
- implements MiniRaftClusterWithHadoopRpc.Factory.Get {
-}
\ No newline at end of file
diff --git a/ratis-hadoop/src/test/resources/log4j.properties b/ratis-hadoop/src/test/resources/log4j.properties
deleted file mode 100644
index ced0687..0000000
--- a/ratis-hadoop/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=info,stdout
-log4j.threshold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
diff --git a/ratis-logservice/Dockerfile b/ratis-logservice/Dockerfile
deleted file mode 100644
index 3f10cf8..0000000
--- a/ratis-logservice/Dockerfile
+++ /dev/null
@@ -1,35 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM openjdk:8-jdk-alpine
-
-######
-RUN apk update && apk upgrade
-
-ENV BASEDIR /opt/ratis-logservice/
-
-# Our ratis-logservice tarball
-ARG BINARY
-ARG VERSION
-
-# Extract, symlink, and remove the ratis-logservice tarball
-COPY $BINARY $BASEDIR
-RUN tar xf ${BASEDIR}/ratis-logservice-${VERSION}-bin.tar.gz -C $BASEDIR \
- && ln -s ${BASEDIR}/ratis-logservice-${VERSION} ${BASEDIR}/current \
- && rm ${BASEDIR}/ratis-logservice-${VERSION}-bin.tar.gz
-
-# Use the nice symlink we made as our "starting directory"
-WORKDIR $BASEDIR/current
diff --git a/ratis-logservice/README.md b/ratis-logservice/README.md
deleted file mode 100644
index c169825..0000000
--- a/ratis-logservice/README.md
+++ /dev/null
@@ -1,99 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Ratis LogService
-
-The Ratis LogService is an distributed, log implementation built on top of Apache Ratis. The LogService provides the
-ability to create named, durable, append-only data structures with the ability to perform linear reads.
-
-## Launching the LogService
-
-The LogService is compose of two Ratis quorums: the metadata quorum and the log quorum. These can be launched manually
-or via docker-compose.
-
-First, the project must be built:
-```bash
-$ mvn clean package -DskipTests
-```
-
-Then, the service can be launched.
-
-### Manual
-
-Launch Metadata daemons:
-```bash
-$ mvn exec:java -Dexec.mainClass=org.apache.ratis.logservice.server.MetadataServer -Dexec.args="-p 9991 -d $HOME/logservice1 -h localhost -q localhost:9991,localhost:9992,localhost:9993"
-```
-```bash
-$ mvn exec:java -Dexec.mainClass=org.apache.ratis.logservice.server.MetadataServer -Dexec.args="-p 9992 -d $HOME/logservice2 -h localhost -q localhost:9991,localhost:9992,localhost:9993"
-```
-```bash
-$ mvn exec:java -Dexec.mainClass=org.apache.ratis.logservice.server.MetadataServer -Dexec.args="-p 9993 -d $HOME/logservice3 -h localhost -q localhost:9991,localhost:9992,localhost:9993"
-```
-
-Above, we have started three daemons that will form a quorum to act as the "LogService Metadata". They will track what
-logs exist and the RAFT qs which service those logs.
-
-Launch Worker daemons:
-```bash
-$ mvn exec:java -Dexec.mainClass=org.apache.ratis.logservice.server.LogServer -Dexec.args="-p 9951 -d $HOME/worker1 -h localhost -q localhost:9991,localhost:9992,localhost:9993"
-```
-```bash
-$ mvn exec:java -Dexec.mainClass=org.apache.ratis.logservice.server.LogServer -Dexec.args="-p 9952 -d $HOME/worker2 -h localhost -q localhost:9991,localhost:9992,localhost:9993"
-```
-```bash
-$ mvn exec:java -Dexec.mainClass=org.apache.ratis.logservice.server.LogServer -Dexec.args="-p 9953 -d $HOME/worker3 -h localhost -q localhost:9991,localhost:9992,localhost:9993"
-```
-
-Now, we have started three daemons which can service a single LogStream. They will register to the Metadata quorum,
-and the Metadata quorum will choose three of them to form a RAFT quorum to "host" a single Log.
-
-Note: the `q` option here references to the Metadata quorum, not the worker quorum as is the case for the Metadata daemons.
-
-Then, the LogService interactive shell can be used to interact with the software:
-```bash
-$ mvn exec:java -Dexec.mainClass=org.apache.ratis.logservice.shell.LogServiceShell -Dexec.args="-q localhost:9991,localhost:9992,localhost:9993"
-```
-
-## Docker
-
-Similarly, a full quorum can be started by building a docker container and then start the docker-compose cluster:
-```bash
-$ cd ratis-logservice && mvn package assembly:single -DskipTests
-$ ./build-docker.sh
-$ docker-compose up
-```
-
-Then, a client container can be launched to connect to the cluster:
-```bash
-$ ./client-env.sh
-$ ./bin/shell -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999
-```
-
-Or, you can launch the verification tool to generate load on the cluster:
-```bash
-$ ./client-env.sh
-$ ./bin/load-test -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999
-```
-
-`client-env.sh` launches a Docker container that can communicate with the LogService cluster running from
-`docker-compose`. You can do this by hand, but take care that the correct network is provided when launching your
-container.
-
-## Service configuration
-
-The log service loads configuration values from a file named logservice.xml. The service looks up this file in an application class path.
-All service configuration options are described in logservice-example.xml file.
diff --git a/ratis-logservice/TUNING.md b/ratis-logservice/TUNING.md
deleted file mode 100644
index 9940237..0000000
--- a/ratis-logservice/TUNING.md
+++ /dev/null
@@ -1,58 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Tuning for the Log Service
-
-This is a list of Ratis configuration properties which have been
-found to be relevant/important to control how Ratis operates for
-the purposes of the LogService.
-
-## RAFT Log
-
-The default RAFT log implementation uses "segments" on disk to avoid
-a single file growing to be very large. By default, each segment is
-`8MB` in size and can be set by the API `RaftServerConfigKeys.Log.setSegmentSizeMax()`.
-When a new segment is created, Ratis will "preallocate" that segment by writing
-data into the file to reduce the risk of latency when we first try to append
-entries to the RAFT log. By default, the segment is preallocated with `4MB`
-and can be changed via `RaftServerConfigKeys.Log.setPreallocatedSize()`.
-
-Up to 2 log segments are cached in memory (including the segment actively being
-written to). This is controlled by `RaftServerConfigKeys.Log.setMaxCachedSegmentNum()`.
-Increasing this configuration would use more memory but should reduce the latency
-of reading entries from the RAFT log.
-
-Writes to the RAFT log are buffered using a Java Direct ByteBuffer (offheap). By default,
-this buffer is `64KB` in size and can be changed via `RaftServerConfigKeys.Log.setWriteBufferSize`.
-Beware that when one LogServer is hosting multiple RAFT groups (multiple "LogService Logs"), each
-will LogServer will have its own buffer. Thus, high concurrency will result in multiple buffers.
-
-## RAFT Server
-
-Every RAFT server maintains a queue of I/O actions that it needs to execute. As with
-much of Ratis, these actions are executed asynchronously and the client can block on
-completion of these tasks as necessary. To prevent saturating memory, this queue of
-items can be limited in size by both number of entries and size of the elements in the queue.
-The former defaults to 4096 elements and id controlled by `RaftServerConfigKeys.Log.setElementLimit()`,
-while the latter defaults to `64MB` and is controlled by `RaftServerConfigKeys.Log.setByteLimit()`.
-
-## Do Not Set
-
-Running a snapshot indicates that we can truncate part of the RAFT log, as the expectation is that
-a snapshot is an equivalent representation of all of the updates from the log. However, the LogService
-is written to expect that we maintain these records. As such, we must not allow snapshots to automatically
-happen as we may lose records from the RAFT log. `RaftServerConfigKeys.Snapshot.setAutoTriggerEnabled()`
-defaults to `false` and should not be set to `true`.
diff --git a/ratis-logservice/build-docker.sh b/ratis-logservice/build-docker.sh
deleted file mode 100755
index eb4ba44..0000000
--- a/ratis-logservice/build-docker.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-LOGSERVICE="$(dirname "$0")"
-LOGSERVICE="$(cd "$LOGSERVICE">/dev/null; pwd)"
-
-# Get the version of the project
-VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)"
-
-# Validate that the tarball is there
-if [[ ! -f "target/ratis-logservice-${VERSION}-bin.tar.gz" ]]; then
- echo "LogService assembly tarball missing, run 'mvn package assembly:single' first!"
- exit 1
-fi
-
-docker build -t ratis-logservice --build-arg BINARY=target/ratis-logservice-$VERSION-bin.tar.gz --build-arg VERSION=$VERSION .
diff --git a/ratis-logservice/client-env.sh b/ratis-logservice/client-env.sh
deleted file mode 100755
index e44d0bd..0000000
--- a/ratis-logservice/client-env.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-docker run --rm --network ratis-logservice_default -it ratis-logservice
diff --git a/ratis-logservice/dev-support/findbugsExcludeFile.xml b/ratis-logservice/dev-support/findbugsExcludeFile.xml
deleted file mode 100644
index 7300683..0000000
--- a/ratis-logservice/dev-support/findbugsExcludeFile.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
- <Match>
- <Package name="org.apache.ratis.logservice.proto"/>
- </Match>
-</FindBugsFilter>
diff --git a/ratis-logservice/docker-compose.yml b/ratis-logservice/docker-compose.yml
deleted file mode 100644
index 3d7b2a1..0000000
--- a/ratis-logservice/docker-compose.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-version: '3'
-
-# We already have a default network using the bridge adapter, so we don't need to redefine it.
-
-services:
- master_1:
- image: ratis-logservice:latest
- command: "java -cp 'conf:lib/*' org.apache.ratis.logservice.server.MetadataServer -h master1.logservice.ratis.org -p 9999 -d /tmp/logservice-metadata -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999"
- ports:
- - 9999
- networks:
- default:
- aliases:
- - master1.logservice.ratis.org
- master_2:
- image: ratis-logservice:latest
- command: "java -cp 'conf:lib/*' org.apache.ratis.logservice.server.MetadataServer -h master2.logservice.ratis.org -p 9999 -d /tmp/logservice-metadata -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999"
- ports:
- - 9999
- networks:
- default:
- aliases:
- - master2.logservice.ratis.org
- master_3:
- image: ratis-logservice:latest
- command: "java -cp 'conf:lib/*' org.apache.ratis.logservice.server.MetadataServer -h master3.logservice.ratis.org -p 9999 -d /tmp/logservice-metadata -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999"
- ports:
- - 9999
- networks:
- default:
- aliases:
- - master3.logservice.ratis.org
- worker_1:
- image: ratis-logservice:latest
- command: "java -cp 'conf:lib/*' org.apache.ratis.logservice.server.LogServer -p 9999 -h worker1.logservice.ratis.org -d /tmp/logservice-worker -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999"
- ports:
- - 9999
- networks:
- default:
- aliases:
- - worker1.logservice.ratis.org
- worker_2:
- image: ratis-logservice:latest
- command: "java -cp 'conf:lib/*' org.apache.ratis.logservice.server.LogServer -p 9999 -h worker2.logservice.ratis.org -d /tmp/logservice-worker -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999"
- ports:
- - 9999
- networks:
- default:
- aliases:
- - worker2.logservice.ratis.org
- worker_3:
- image: ratis-logservice:latest
- command: "java -cp 'conf:lib/*' org.apache.ratis.logservice.server.LogServer -p 9999 -h worker3.logservice.ratis.org -d /tmp/logservice-worker -q master1.logservice.ratis.org:9999,master2.logservice.ratis.org:9999,master3.logservice.ratis.org:9999"
- ports:
- - 9999
- networks:
- default:
- aliases:
- - worker3.logservice.ratis.org
- graphite:
- image: graphiteapp/graphite-statsd
- ports:
- - 8000:80
- - 2003-2004:2003-2004
- - 2023-2024:2023-2024
- - 8125:8125
- - 8126:8126
- networks:
- default:
- aliases:
- - graphite.logservice.ratis.org
diff --git a/ratis-logservice/logservice-example.xml b/ratis-logservice/logservice-example.xml
deleted file mode 100644
index ed48777..0000000
--- a/ratis-logservice/logservice-example.xml
+++ /dev/null
@@ -1,88 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration>
- <property>
- <name>logservice.metaserver.port</name>
- <value>-1</value>
- <description>Log service meta-server port number</description>
- </property>
- <property>
- <name>logservice.metaserver.hostname</name>
- <value>localhost</value>
- <description>Log service meta-server host name</description>
- </property>
- <property>
- <name>logservice.metaserver.workdir</name>
- <value>.</value>
- <description>Log service meta-server working directory</description>
- </property>
- <property>
- <name>logservice.logserver.port</name>
- <value>-1</value>
- <description>Log service log-server port number</description>
- </property>
- <property>
- <name>logservice.logserver.hostname</name>
- <value>localhost</value>
- <description>Log service log-server host name</description>
- </property>
- <property>
- <name>logservice.logserver.workdir</name>
- <value>.</value>
- <description>Log service log-server working directory</description>
- </property>
- <!--property>
- <name>logservice.metaquorum</name>
- <value></value>
- <description>Log service metaquorum list of nodes</description>
- </property-->
- <!--property>
- <name>logservice.metaserver.groupid</name>
- <value></value>
- <description>Log service metaserver stringified UUID</description>
- </property-->
- <!--property>
- <name>logservice.logserver.groupid</name>
- <value></value>
- <description>Log service log server stringified UUID</description>
- </property-->
- <property>
- <name>logservice.raft.leader.election.timeout.min</name>
- <value>1000</value>
- <description>Raft leader election timeout minimum in ms</description>
- </property>
- <property>
- <name>logservice.raft.leader.election.timeout.max</name>
- <value>1200</value>
- <description>Raft leader election timeout maximum in ms</description>
- </property>
- <property>
- <name>logservice.raft.rpc.timeout</name>
- <value>100000</value>
- <description>Raft RPC call timeout in ms</description>
- </property>
- <property>
- <name>ratis.raft.segment.size</name>
- <value>33554432</value>
- <description>Raft log segment size in bytes</description>
- </property>
-</configuration>
diff --git a/ratis-logservice/pom.xml b/ratis-logservice/pom.xml
deleted file mode 100644
index fb7aa13..0000000
--- a/ratis-logservice/pom.xml
+++ /dev/null
@@ -1,330 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <artifactId>ratis</artifactId>
- <groupId>org.apache.ratis</groupId>
- <version>2.3.0-SNAPSHOT</version>
- </parent>
-
- <artifactId>ratis-logservice</artifactId>
- <name>Apache Ratis LogService</name>
-
- <build>
- <extensions>
- <!-- Use os-maven-plugin to initialize the "os.detected" properties -->
- <extension>
- <groupId>kr.motd.maven</groupId>
- <artifactId>os-maven-plugin</artifactId>
- <version>1.5.0.Final</version>
- </extension>
- </extensions>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <compilerArgs>
- <!-- disable all javac warnings for shaded sources -->
- <arg>-Xlint:none</arg>
- <arg>-XDignore.symbol.file</arg>
- </compilerArgs>
- <showWarnings>false</showWarnings>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-checkstyle-plugin</artifactId>
- <configuration/>
- </plugin>
- <!-- Make a jar and put the sources in the jar -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-source-plugin</artifactId>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <skipAssembly>false</skipAssembly>
- <tarLongFileMode>gnu</tarLongFileMode>
- <descriptors>
- <descriptor>src/assembly/assembly.xml</descriptor>
- </descriptors>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.xolstice.maven.plugins</groupId>
- <artifactId>protobuf-maven-plugin</artifactId>
- <configuration>
- <includes>
- <include>**/*.proto</include>
- </includes>
- <protocArtifact>
- com.google.protobuf:protoc:${shaded.protobuf.version}:exe:${os.detected.classifier}
- </protocArtifact>
- <!-- Place these in a location that compiler-plugin is already looking -->
- <outputDirectory>${project.build.directory}/generated-sources</outputDirectory>
- <!-- With multiple executions, this must be `false` otherwise we wipe out the previous execution -->
- <clearOutputDirectory>false</clearOutputDirectory>
- </configuration>
- <executions>
- <execution>
- <id>compile-protobuf</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>compile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>com.google.code.maven-replacer-plugin</groupId>
- <artifactId>replacer</artifactId>
- <version>1.5.3</version>
- <executions>
- <execution>
- <phase>process-sources</phase>
- <goals>
- <goal>replace</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <basedir>${project.build.directory}/generated-sources/org/apache/ratis/logservice/proto</basedir>
- <includes>
- <include>**/*.java</include>
- </includes>
- <replacements>
- <replacement>
- <token>([^\.])com.google</token>
- <value>$1org.apache.ratis.thirdparty.com.google</value>
- </replacement>
- <replacement>
- <token>([^\.])io.grpc</token>
- <value>$1org.apache.ratis.thirdparty.io.grpc</value>
- </replacement>
- </replacements>
- </configuration>
- </plugin>
- <plugin>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-maven-plugin</artifactId>
- <configuration>
- <excludeFilterFile>${project.basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
- </configuration>
- </plugin>
- </plugins>
- <pluginManagement>
- <plugins>
- <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
- <plugin>
- <groupId>org.eclipse.m2e</groupId>
- <artifactId>lifecycle-mapping</artifactId>
- <version>1.0.0</version>
- <configuration>
- <lifecycleMappingMetadata>
- <pluginExecutions>
- <pluginExecution>
- <pluginExecutionFilter>
- <groupId>
- com.google.code.maven-replacer-plugin
- </groupId>
- <artifactId>replacer</artifactId>
- <versionRange>
- [1.5.3,)
- </versionRange>
- <goals>
- <goal>replace</goal>
- </goals>
- </pluginExecutionFilter>
- <action>
- <ignore></ignore>
- </action>
- </pluginExecution>
- </pluginExecutions>
- </lifecycleMappingMetadata>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
- </build>
-
- <dependencies>
- <!-- Ratis dependencies -->
- <dependency>
- <artifactId>ratis-proto</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-common</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-client</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-server</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-grpc</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-netty</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
-
- <!-- Hadoop dependencies -->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- CLI dependencies -->
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
- <dependency>
- <groupId>org.jline</groupId>
- <artifactId>jline</artifactId>
- </dependency>
- <!-- Third-party dependencies -->
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <!-- Test Ratis dependencies -->
- <dependency>
- <artifactId>ratis-netty</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <artifactId>ratis-grpc</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <artifactId>ratis-server</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <artifactId>ratis-common</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <artifactId>ratis-client</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <artifactId>ratis-hadoop</artifactId>
- <groupId>org.apache.ratis</groupId>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <!-- Test third-party dependencies -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
-
-
- <!-- Needed for Archive log reader and writer testing-->
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- <exclusions>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-jvm</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.github.spotbugs</groupId>
- <artifactId>spotbugs-annotations</artifactId>
- <scope>provided</scope>
- <optional>true</optional>
- </dependency>
- </dependencies>
-</project>
diff --git a/ratis-logservice/src/assembly/assembly.xml b/ratis-logservice/src/assembly/assembly.xml
deleted file mode 100644
index 21154bf..0000000
--- a/ratis-logservice/src/assembly/assembly.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0"?>
-<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
- <id>bin</id>
- <formats>
- <format>tar.gz</format>
- </formats>
- <dependencySets>
- <dependencySet>
- <outputDirectory>lib</outputDirectory>
- <includes>
- <include>*:*</include>
- </includes>
- </dependencySet>
- </dependencySets>
- <fileSets>
- <fileSet>
- <includes>
- <include>README.md</include>
- </includes>
- </fileSet>
- <fileSet>
- <!-- Source directory, keeps 'src/test/resources' out of 'conf/' -->
- <directory>src/test/resources</directory>
- <!-- Destination directory -->
- <outputDirectory>conf</outputDirectory>
- <includes>
- <include>log4j.properties</include>
- <include>hadoop-metrics2.properties</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>src/assembly/bin</directory>
- <includes>
- <include>*</include>
- </includes>
- <outputDirectory>bin</outputDirectory>
- </fileSet>
- </fileSets>
-</assembly>
diff --git a/ratis-logservice/src/assembly/bin/load-test b/ratis-logservice/src/assembly/bin/load-test
deleted file mode 100755
index 7867667..0000000
--- a/ratis-logservice/src/assembly/bin/load-test
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-LOGSERVICE="$(dirname "$0")"
-LOGSERVICE="$(cd "$LOGSERVICE/..">/dev/null; pwd)"
-
-export CLASSPATH="${LOGSERVICE}/conf:${LOGSERVICE}/lib/*"
-exec java -XX:OnOutOfMemoryError="kill -9 %p" $LOGSERVICE_OPTS org.apache.ratis.logservice.tool.VerificationTool "$@"
diff --git a/ratis-logservice/src/assembly/bin/shell b/ratis-logservice/src/assembly/bin/shell
deleted file mode 100755
index 5a78ea6..0000000
--- a/ratis-logservice/src/assembly/bin/shell
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/sh
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-LOGSERVICE="$(dirname "$0")"
-LOGSERVICE="$(cd "$LOGSERVICE/..">/dev/null; pwd)"
-
-export CLASSPATH="${LOGSERVICE}/conf:${LOGSERVICE}/lib/*"
-exec java -XX:OnOutOfMemoryError="kill -9 %p" $LOGSERVICE_OPTS org.apache.ratis.logservice.shell.LogServiceShell "$@"
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/ArchiveLogReader.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/ArchiveLogReader.java
deleted file mode 100644
index 1eb1b4f..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/ArchiveLogReader.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import org.apache.ratis.logservice.server.RaftLogReader;
-/**
- * Archive Log Reader implementation. This class is not thread-safe
- *
- */
-public interface ArchiveLogReader extends RaftLogReader, LogReader {
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/ArchiveLogWriter.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/ArchiveLogWriter.java
deleted file mode 100644
index 98038a1..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/ArchiveLogWriter.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import java.io.IOException;
-
-public interface ArchiveLogWriter extends LogWriter{
-
- /**
- * Initializes the writer
- * @param location archival location
- * @param logName
- * @throws IOException
- */
- void init(String location, LogName logName) throws IOException;
-
- /**
- * Rolls writer after number of records written crosses threshold
- * {@link org.apache.ratis.logservice.server.LogStateMachine#DEFAULT_ARCHIVE_THRESHOLD_PER_FILE}
- *
- * @throws IOException
- */
- void rollWriter() throws IOException;
-
- /**
- * Record Id of the last written record
- * @return
- */
- long getLastWrittenRecordId() throws IOException;
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogInfo.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogInfo.java
deleted file mode 100644
index 63a07d1..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogInfo.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.api;
-
-import org.apache.ratis.protocol.RaftGroup;
-
-/**
- * This classs represent the pair of LogName -> RaftGroup
- */
-public class LogInfo {
- private final RaftGroup raftGroup;
- private final LogName logName;
- public LogInfo(LogName logName, RaftGroup raftGroup) {
- this.logName = logName;
- this.raftGroup = raftGroup;
- }
-
- /**
- * Log name
- * @return
- */
- public LogName getLogName() {
- return logName;
- }
-
- /**
- * Raft group
- * @return
- */
- public RaftGroup getRaftGroup() {
- return raftGroup;
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogMessage.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogMessage.java
deleted file mode 100644
index 1fa2ad6..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogMessage.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import org.apache.ratis.protocol.Message;
-
-public abstract class LogMessage implements Message {
-
- /*
- * Log name
- */
- @SuppressFBWarnings("UWF_UNWRITTEN_FIELD")
- private LogName logName;
-
- /**
- * Get log name
- * @return log name
- */
- public LogName getLogName() {
- return logName;
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogName.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogName.java
deleted file mode 100644
index 0d61c09..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogName.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import static java.util.Objects.requireNonNull;
-
-import java.util.Objects;
-
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- * Identifier to uniquely identify a {@link LogStream}.
- */
-public final class LogName {
- // It's pretty likely that what uniquely defines a LogStream
- // to change over time. We should account for this by making an
- // API which can naturally evolve.
- private final String name;
-
- private LogName(String name) {
- this.name = requireNonNull(name);
- }
-
- /**
- * Returns the unique name which identifies a LogStream.
- *
- * Impl Note: This class uses a String to uniquely identify this LogName (and the corresponding LogStream)
- * from others. This is purely an implementation detail; the intent is that any data should be capable
- * of identifying one LogStream/LogName from another. Users need only know how to construct a {@link LogName}
- * and then use that in their application.
- */
- public String getName() {
- return name;
- }
-
- @Override public boolean equals(Object o) {
- if (!(o instanceof LogName)) {
- return false;
- }
- return Objects.equals(name, ((LogName) o).getName());
- }
-
- @Override public int hashCode() {
- return name.hashCode();
- }
-
- @Override public String toString() {
- return "LogName['" + name + "']";
- }
-
- /**
- * Length of a log's name
- * @return length
- */
- public int getLength() {
- return name.length();
- }
-
- /**
- * Creates a {@link LogName} given the provided string.
- */
- public static LogName of(String name) {
- // TODO Limit allowed characters in the name?
- return new LogName(name);
- }
-
- public static LogName parseFrom(ByteString logName)
- throws InvalidProtocolBufferException {
- LogServiceProtos.LogNameProto logNameProto = LogServiceProtos.LogNameProto.parseFrom(logName);
- return new LogName(logNameProto.getName());
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogReader.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogReader.java
deleted file mode 100644
index 2119fab..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogReader.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-/**
- * Synchronous client interface to read from a LogStream.
- */
-public interface LogReader extends AutoCloseable {
-
- /**
- * Seeks to the position before the record at the provided {@code recordId} in the LogStream.
- *
- * @param recordId A non-negative, offset in the LogStream
- * @return A future for when the operation is completed.
- */
- void seek(long recordId) throws IOException;
-
- /**
- * Reads the next record from the LogStream at the current position and advances the current position
- * to after the record which was just returned.
- *
- * @return The data for the next record.
- */
- ByteBuffer readNext() throws IOException;
-
- /**
- * Reads the next record from the LogStream at the current position into the provided {@link buffer} and
- * advances the current position to the point after the record just read.
- *
- * The provided buffer must be capable of holding one complete record from the Log. If the provided buffer is
- * too small, an exception will be thrown.
- *
- * @param buffer A buffer to read the record into
- */
- void readNext(ByteBuffer buffer) throws IOException;
-
- /**
- * Reads the next {@code numRecords} records from the LogStream, starting at the current position. This method
- * may return fewer than requested records if the LogStream does not have sufficient records to return.
- *
- * @param numRecords The number of records to return
- * @return The records, no more than the requested {@code numRecords} amount.
- */
- List<ByteBuffer> readBulk(int numRecords) throws IOException;
-
- /**
- * Fills the provided {@code List<ByteBuffer>} with records from the LogStream, starting at the current position.
- * This method will attempt to fill all of the {@code ByteBuffer}'s that were provided, as long as there are
- * records in the {@code LogStream} to support this. This method will return the number of buffers that were
- * filled.
- *
- * Each provided buffer must be capable of holding one complete record from the Log. If the provided buffer is
- * too small, an exception will be thrown.
- *
- * @param buffers A non-empty array of non-null ByteBuffers.
- * @return The number of records returns, equivalent to the number of filled buffers.
- */
- int readBulk(ByteBuffer[] buffers) throws IOException;
-
- /**
- * Returns the current position of this Reader. The position is a {@code recordId}.
- */
- long getPosition() throws IOException;
-
- /**
- * Overrides {@link #close()} in {@link AutoCloseable} to throw an IOException.
- */
- void close() throws IOException;
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogServiceClient.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogServiceClient.java
deleted file mode 100644
index 1e7c306..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogServiceClient.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.logservice.api.LogStream.State;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.impl.ArchivedLogStreamImpl;
-import org.apache.ratis.logservice.impl.ExportedLogStreamImpl;
-import org.apache.ratis.logservice.impl.LogStreamImpl;
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.*;
-import org.apache.ratis.logservice.server.ArchivalInfo;
-import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-import org.apache.ratis.logservice.util.MetaServiceProtoUtil;
-import org.apache.ratis.protocol.*;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.apache.ratis.logservice.util.LogServiceProtoUtil.toChangeStateRequestProto;
-import static org.apache.ratis.logservice.util.LogServiceUtils.getPeersFromQuorum;
-
-
-/**
- * LogServiceClient is responsible for all meta service communications such as create/get/list logs.
- * Initialized by the metaQuorum string that has list of masters as "server:port' separated by a comma.
- * An example: 'server1.example.com:9999,server2.example.com:9999,server3.example.com:9999
- */
-
-public class LogServiceClient implements AutoCloseable {
-
-
- // the raft client for meta quorum. All DML operations are going using this client.
- private final RaftClient client;
- private final LogServiceConfiguration config;
-
- /**
- * Constuctor. Build raft client for meta quorum
- * @param metaQuorum
- */
- public LogServiceClient(String metaQuorum) {
- this(metaQuorum, LogServiceConfiguration.create(), new RaftProperties());
- }
-
- /**
- * Constuctor. Build raft client for meta quorum
- * @param metaQuorum
- * @param properties
- */
- public LogServiceClient(String metaQuorum, RaftProperties properties) {
- this(metaQuorum, LogServiceConfiguration.create(), properties);
- }
-
- /**
- * Constuctor (with configuration). Build raft client for meta quorum
- * @param metaQuorum
- * @param config log service configuration
- * @param properties
- */
- public LogServiceClient(String metaQuorum, LogServiceConfiguration config, RaftProperties properties) {
- Set<RaftPeer> peers = getPeersFromQuorum(metaQuorum);
- RaftGroup meta = RaftGroup.valueOf(Constants.META_GROUP_ID, peers);
- client = RaftClient.newBuilder()
- .setRaftGroup(meta)
- .setClientId(ClientId.randomId())
- .setProperties(properties)
- .build();
- this.config = config;
- }
-
- /**
- * Create a new Log request.
- * @param logName the name of the log to create
- * @return
- * @throws IOException
- */
- public LogStream createLog(LogName logName) throws IOException {
- RaftClientReply reply = client.io().sendReadOnly(
- () -> MetaServiceProtoUtil.toCreateLogRequestProto(logName).toByteString());
- CreateLogReplyProto message =
- CreateLogReplyProto.parseFrom(reply.getMessage().getContent());
- if (message.hasException()) {
- throw MetaServiceProtoUtil.toMetaServiceException(message.getException());
- }
- LogInfo info = MetaServiceProtoUtil.toLogInfo(message.getLog());
- return new LogStreamImpl(logName, getRaftClient(info), config);
- }
-
- /**
- * Get log request.
- * @param logName the name of the log to get
- * @return
- * @throws IOException
- */
- public LogStream getLog(LogName logName) throws IOException {
- return new LogStreamImpl(logName, getRaftClient(getLogInfo(logName)), config);
- }
-
- /**
- * Get Archive log .
- * @param logName the name of the log to get
- * @return
- * @throws IOException
- */
- public LogStream getArchivedLog(LogName logName) throws IOException {
- return new ArchivedLogStreamImpl(logName, config);
- }
-
- /**
- * Get exported log .
- * @param logName the name of the log to get
- * @param location location of the exported log
- * @return
- * @throws IOException
- */
-
- public LogStream getExportLog(LogName logName, String location) throws IOException {
- return new ExportedLogStreamImpl(logName, location);
- }
-
- public List<ArchivalInfo> getExportStatus(LogName logName) throws IOException {
- try (RaftClient client = getRaftClient(getLogInfo(logName))) {
- RaftClientReply exportInfoReply = client.io().sendReadOnly(
- () -> LogServiceProtoUtil.toExportInfoRequestProto(logName).toByteString());
- LogServiceProtos.GetExportInfoReplyProto message =
- LogServiceProtos.GetExportInfoReplyProto
- .parseFrom(exportInfoReply.getMessage().getContent());
- if (message.hasException()) {
- throw new IOException(message.getException().getErrorMsg());
- }
- return message.getInfoList().stream()
- .map(infoProto -> LogServiceProtoUtil.toExportInfo(infoProto))
- .collect(Collectors.toList());
- }
- }
-
- public void deleteLog(LogName logName) throws IOException {
- RaftClientReply reply = client.io().sendReadOnly
- (() -> MetaServiceProtoUtil.toDeleteLogRequestProto(logName).toByteString());
- DeleteLogReplyProto message = DeleteLogReplyProto.parseFrom(reply.getMessage().getContent());
- if(message.hasException()) {
- throw MetaServiceProtoUtil.toMetaServiceException(message.getException());
- }
- }
-
- /**
- * Return the list of available logs
- * @return
- * @throws IOException
- */
- public List<LogInfo> listLogs() throws IOException {
- RaftClientReply reply = client.io().sendReadOnly
- (() -> MetaServiceProtoUtil.toListLogRequestProto().toByteString());
- ListLogsReplyProto message = ListLogsReplyProto.parseFrom(reply.getMessage().getContent());
- List<LogInfoProto> infoProtos = message.getLogsList();
- List<LogInfo> infos = infoProtos.stream()
- .map(proto -> MetaServiceProtoUtil.toLogInfo(proto))
- .collect(Collectors.toList());
- return infos;
- }
-
- @Override
- public void close() throws Exception {
- client.close();
- }
-
- // Internal methods
-
- /**
- * Build a raft client for the particular log. Temporary here. TODO: Should be moved to LogService part
- * @param logInfo
- * @return
- */
- private RaftClient getRaftClient(LogInfo logInfo) throws IOException {
-
- RaftProperties properties = new RaftProperties();
- return RaftClient.newBuilder().setRaftGroup(logInfo.getRaftGroup()).setProperties(properties).build();
-
- }
-
- private LogInfo getLogInfo(LogName logName) throws IOException {
- RaftClientReply reply = client.io().sendReadOnly(
- () -> MetaServiceProtoUtil.toGetLogRequestProto(logName).toByteString());
- GetLogReplyProto message = GetLogReplyProto.parseFrom(reply.getMessage().getContent());
- if (message.hasException()) {
- throw MetaServiceProtoUtil.toMetaServiceException(message.getException());
- }
- return MetaServiceProtoUtil.toLogInfo(message.getLog());
- }
-
- /**
- * Archives the given log out of the state machine and into a configurable long-term storage.
- * A log must be in {@link State#CLOSED} to archive it.
- * Archiving of the log will happen asynchronously from the client,
- * The call will return immediately after adding a request for archiving log
- * to the respective quorum
- *
- * Client can check the status of Archiving by calling getState() Method
- *
- * @param logName The name of the log to archive.
- */
- public void archiveLog(LogName logName) throws IOException {
- exportLog(logName, null, 0);
- }
-
- /**
- * Export the given log out of the state machine and into a provided location on the configured storage
- * A log must be in {@link State#CLOSED} to export it.
- * exporting of the log will happen asynchronously from the client,
- * The call will return immediately after adding a request for archiving log
- * to the respective quorum
- *
- * Client can check the status of export by calling getState() Method
- *
- * @param logName The name of the log to archive.
- */
- public void exportLog(LogName logName, String location, long recordId) throws IOException {
- try (RaftClient client = getRaftClient(getLogInfo(logName))) {
- RaftClientReply archiveLogReply =
- client.io().sendReadOnly(() -> LogServiceProtoUtil
- .toArchiveLogRequestProto(logName, location, recordId,
- location == null ? true : false, ArchivalInfo.ArchivalStatus.SUBMITTED)
- .toByteString());
- LogServiceProtos.ArchiveLogReplyProto archiveMessage =
- LogServiceProtos.ArchiveLogReplyProto
- .parseFrom(archiveLogReply.getMessage().getContent());
- if (archiveMessage.hasException()) {
- throw new IOException(archiveMessage.getException().getErrorMsg());
- }
- }
- }
-
- /**
- * Moves the {@link LogStream} identified by the {@code name} from {@link State#OPEN} to {@link State#CLOSED}.
- * If the log is not {@link State#OPEN}, this method returns an error.
- *
- * @param name The name of the log to close
- */
- // TODO this name sucks, confusion WRT the Java Closeable interface.
- public void closeLog(LogName name) throws IOException {
- try (RaftClient logClient = getRaftClient(getLogInfo(name))) {
- logClient.io().send(() -> toChangeStateRequestProto(name, State.CLOSED).toByteString());
- }
- }
-
- /**
- * Updates a log with the new configuration object, overriding
- * the previous configuration.
- *
- * @param conf The new configuration object
- */
- void updateConfiguration(LogName name, LogServiceConfiguration conf) {
- //TODO: write me
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogServiceConfiguration.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogServiceConfiguration.java
deleted file mode 100644
index b1089dd..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogServiceConfiguration.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import java.net.URL;
-import java.util.UUID;
-
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.server.MetadataServer;
-import org.apache.ratis.logservice.server.ServerOpts;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * An encapsulation of configuration for a LogService.
- * The base configuration is defined in logservice.xml file,
- * which is expected to reside in a class path.
- */
-public final class LogServiceConfiguration extends RaftProperties {
- private static final Logger LOG = LoggerFactory.getLogger(MetadataServer.class);
- public static final String RESOURCE_NAME = "logservice.xml";
-
-
- /**
- * Creates default log service configuration object
- * @return configuration object
- */
- public static LogServiceConfiguration create() {
- return new LogServiceConfiguration();
- }
-
- /**
- * Creates log service configuration object
- * with a custom configuration file (found on classpath)
- * @param resourceName name of configuration file
- * @return configuration object
- */
- public static LogServiceConfiguration create(String resourceName) {
- return new LogServiceConfiguration(resourceName);
- }
-
- /**
- * Creates log service configuration object
- * with a custom configuration URL
- * @param resourcePath path of configuration file
- * @return configuration object
- */
- public static LogServiceConfiguration create(URL resourcePath) {
- return new LogServiceConfiguration(resourcePath);
- }
-
- private LogServiceConfiguration() {
- super();
- addResource(RESOURCE_NAME);
- }
-
- private LogServiceConfiguration(String resourceName) {
- super();
- addResource(resourceName);
- }
-
- private LogServiceConfiguration(URL resourcePath) {
- super();
- addResource(resourcePath);
- }
-
- /**
- * Adds configuration options from logservice.xml
- * for LogServer
- * @param opts server options
- * @return server options
- */
- public ServerOpts addLogServerOpts(ServerOpts opts) {
- if (!opts.isHostSet()) {
- String val = get(Constants.LOG_SERVER_HOSTNAME_KEY, "localhost");
- if (val != null) {
- opts.setHost(val);
- }
- }
-
- if (!opts.isPortSet()) {
- String val = get(Constants.LOG_SERVER_PORT_KEY);
- if (val != null) {
- try {
- opts.setPort(Integer.parseInt(val));
- } catch (Exception e) {
- LOG.warn("Config value {} for {} is invaild", val, Constants.LOG_SERVER_PORT_KEY);
- }
- }
- }
-
- if (!opts.isWorkingDirSet()) {
- String val = get(Constants.LOG_SERVER_WORKDIR_KEY);
- if (val != null) {
- opts.setWorkingDir(val);
- }
- }
-
- if (!opts.isMetaQuorumSet()) {
- String val = get(Constants.LOG_SERVICE_METAQUORUM_KEY);
- if (val != null) {
- opts.setMetaQuorum(val);
- }
- }
-
- if (!opts.isLogServerGroupIdSet()) {
- String val = get(Constants.LOG_SERVICE_LOG_SERVER_GROUPID_KEY);
- if (val != null) {
- try {
- opts.setLogServerGroupId(UUID.fromString(val));
- } catch (IllegalArgumentException e) {
- LOG.warn("Config value {} for {} is invaild", val,
- Constants.LOG_SERVICE_LOG_SERVER_GROUPID_KEY);
- }
- } else {
- opts.setLogServerGroupId(Constants.SERVERS_GROUP_UUID);
- }
- }
- return opts;
- }
-
-
- /**
- * Adds configuration options from logservice.xml
- * for MetadataServer
- * @param opts server options
- * @return server options
- */
- public ServerOpts addMetaServerOpts (ServerOpts opts) {
- if (!opts.isHostSet()) {
- String val = get(Constants.META_SERVER_HOSTNAME_KEY);
- if (val != null) {
- opts.setHost(val);
- }
- }
-
- if (!opts.isPortSet()) {
- String val = get(Constants.META_SERVER_PORT_KEY);
- if (val != null) {
- try {
- opts.setPort(Integer.parseInt(val));
- } catch (Exception e) {
- LOG.warn("Config value {} for {} is invaild", val,
- Constants.META_SERVER_PORT_KEY);
- }
- }
- }
-
- if (!opts.isWorkingDirSet()) {
- String val = get(Constants.META_SERVER_WORKDIR_KEY);
- if (val != null) {
- opts.setWorkingDir(val);
- }
- }
-
- if (!opts.isMetaQuorumSet()) {
- String val = get(Constants.LOG_SERVICE_METAQUORUM_KEY);
- if (val != null) {
- opts.setMetaQuorum(val);
- }
- }
-
- if (!opts.isMetaServerGroupIdSet()) {
- String val = get(Constants.LOG_SERVICE_META_SERVER_GROUPID_KEY);
- if (val != null) {
- try {
- opts.setMetaGroupId(UUID.fromString(val));
- } catch (IllegalArgumentException e) {
- LOG.warn("Config value {} for {} is invaild", val,
- Constants.LOG_SERVICE_META_SERVER_GROUPID_KEY);
- }
- } else {
- opts.setMetaGroupId(Constants.META_GROUP_UUID);
- }
- }
- return opts;
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogStream.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogStream.java
deleted file mode 100644
index 5238ac9..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogStream.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.ratis.client.RaftClient;
-
-/**
- * A distributed log with "infinite" length that supports reads and writes.
- */
-public interface LogStream extends AutoCloseable{
-
- /**
- * An enumeration that defines the current state of a LogStream
- */
- enum State {
- /*
- Log is open to receive writes/read request
- */
- OPEN,
- /*
- Log is closed and can not be written but available for read or archival or export
- */
- CLOSED,
- /*
- Log is currently archiving but still readable and can be exported to a different locations
- */
- ARCHIVING,
- /*
- Log is archiving and available for read only
- */
- ARCHIVED,
- /*
- Log is deleted so not available for read or write or any other operation
- */
- DELETED;
- }
-
- /**
- * Returns the unique name to identify this log.
- */
- LogName getName();
-
- /**
- * Returns the current state of this log.
- */
- State getState() throws IOException;
-
- /**
- * Returns the size of this LogStream in bytes.
- * @throws IOException
- */
- long getSize() throws IOException;
-
- /**
- * Returns the number of records in this log.
- * @throws IOException
- */
- long getLength() throws IOException;
-
- /**
- * Creates a reader to read this LogStream.
- *
- * @return A synchronous reader
- */
- LogReader createReader() throws IOException;
-
- /**
- * Creates a write to write to this LogStream.
- *
- * @return A synchronous writer
- */
- LogWriter createWriter();
-
- /**
- * Returns the recordId of the last record in this LogStream. For an empty log, the recordId is {@code 0}.
- * @throws IOException
- */
- long getLastRecordId() throws IOException;
-
- /**
- * Returns the recordId of the first record in this LogStream. For an empty log, the recordId is {@code 0}.
- * @throws IOException
- */
- long getStartRecordId() throws IOException;
-
- /**
- * Returns all {@link RecordListeners} for this LogStream.
- */
- Collection<RecordListener> getRecordListeners();
-
- /**
- * Returns a copy of the Configuration for this LogStream.
- */
- LogServiceConfiguration getConfiguration();
-
- /**
- * Add new log record listener
- * @param listener listener
- */
- void addRecordListener(RecordListener listener);
-
-
- /**
- * Remove record listener
- * @param listener listener
- * @return true, if successful, false - otherwise
- */
- boolean removeRecordListener (RecordListener listener);
-
- /**
- * Get Raft Client
- * @return Raft client
- */
-
- RaftClient getRaftClient();
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogWriter.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogWriter.java
deleted file mode 100644
index fb201d9..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/LogWriter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-/**
- * Synchronous client interface to write to a LogStream.
- */
-public interface LogWriter extends AutoCloseable {
-
- /**
- * Appends the given data as a record in the LogStream.
- *
- * @param data The record to append
- * @return The recordId for the record just written
- */
- long write(ByteBuffer data) throws IOException;
-
- /**
- * Appends each entry of data as a new record in the LogStream. If this method returns
- * successfully, all records can be considered persisted. Otherwise, none can be assumed
- * to have been written.
- *
- * @param records Records to append
- * @return The recordIds assigned to the records written
- */
- List<Long> write(List<ByteBuffer> records) throws IOException;
-
- /**
- * Guarantees that all previous data appended by {@link #write(ByteBuffer)} are persisted
- * and durable in the LogStream.
- *
- * @return The recordId prior to which all records are durable
- */
- long sync() throws IOException;
-
- /**
- * Overrides {@link #close()} in {@link AutoCloseable} to throw an IOException.
- */
- void close() throws IOException;
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/RecordListener.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/RecordListener.java
deleted file mode 100644
index a62d071..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/api/RecordListener.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.api;
-
-import java.nio.ByteBuffer;
-import java.util.function.Consumer;
-
-/**
- * Interface that, when registered with a {@link LogStream}, will receive all records written
- * to that LogStream until it is removed.
- */
-public interface RecordListener extends Consumer<ByteBuffer> {
-
- /**
- * Returns a name to identify this listener from others.
- */
- String getName();
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/Constants.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/Constants.java
deleted file mode 100644
index 4afc4b9..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/Constants.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.common;
-
-import org.apache.ratis.protocol.RaftGroupId;
-
-import java.util.UUID;
-
-public final class Constants {
- private Constants() {
- }
-
- public static final UUID META_GROUP_UUID = new UUID(0,1);
- public static final RaftGroupId META_GROUP_ID = RaftGroupId.valueOf(META_GROUP_UUID);
-
- public static final UUID SERVERS_GROUP_UUID = new UUID(0,2);
- public static final RaftGroupId SERVERS_GROUP_ID =
- RaftGroupId.valueOf(SERVERS_GROUP_UUID);
- public static final String META_SERVER_PORT_KEY = "logservice.metaserver.port";
- public static final String META_SERVER_HOSTNAME_KEY = "logservice.metaserver.hostname";
- public static final String META_SERVER_WORKDIR_KEY = "logservice.metaserver.workdir";
- public static final String LOG_SERVER_PORT_KEY = "logservice.logserver.port";
- public static final String LOG_SERVER_HOSTNAME_KEY = "logservice.logserver.hostname";
- public static final String LOG_SERVER_WORKDIR_KEY = "logservice.logserver.workdir";
- public static final String LOG_SERVICE_METAQUORUM_KEY = "logservice.metaquorum";
- public static final String LOG_SERVICE_META_SERVER_GROUPID_KEY =
- "logservice.metaserver.groupid";
- public static final String LOG_SERVICE_LOG_SERVER_GROUPID_KEY = "logservice.logserver.groupid";
- public static final String LOG_SERVICE_ARCHIVAL_LOCATION_KEY = "logservice.archival.location";
- /*
- * Raft properties
- */
- public static final String LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MIN_KEY =
- "logservice.raft.leader.election.timeout.min"; // in ms
- public static final String LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MAX_KEY =
- "logservice.raft.leader.election.timeout.max"; // in ms
- public static final String LOG_SERVICE_RPC_TIMEOUT_KEY =
- "logservice.raft.rpc.timeout"; // in ms
-
- public static final long DEFAULT_LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MIN = 1000;
- public static final long DEFAULT_LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MAX = 1200;
- public static final long DEFAULT_RPC_TIMEOUT = 100000;// 100 sec (?)
-
- public static final String RATIS_RAFT_SEGMENT_SIZE_KEY = "ratis.raft.segment.size";
- public static final long DEFAULT_RATIS_RAFT_SEGMENT_SIZE = 32 * 1024 * 1024L;// 32MB
-
- public static final String LOG_SERVICE_HEARTBEAT_INTERVAL_KEY =
- "logservice.heartbeat.interval"; // in ms
- public static final long DEFAULT_HEARTBEAT_INTERVAL = 3000;// 3 seconds
-
- public static final String LOG_SERVICE_PEER_FAILURE_DETECTION_PERIOD_KEY =
- "logservice.peer.failure.detection.period"; // in ms
- public static final long DEFAULT_PEER_FAILURE_DETECTION_PERIOD = 60000;// 1 min.
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/LogAlreadyExistException.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/LogAlreadyExistException.java
deleted file mode 100644
index 74a77ba..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/LogAlreadyExistException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.common;
-
-import java.io.IOException;
-
-public class LogAlreadyExistException extends IOException {
- public LogAlreadyExistException(String name) {
- super(name);
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/LogNotFoundException.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/LogNotFoundException.java
deleted file mode 100644
index f9d672a..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/LogNotFoundException.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.common;
-
-import java.io.IOException;
-
-public class LogNotFoundException extends IOException {
-
- public LogNotFoundException(String logName) {
- super(logName);
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/NoEnoughWorkersException.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/NoEnoughWorkersException.java
deleted file mode 100644
index 163b771..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/common/NoEnoughWorkersException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.common;
-
-import java.io.IOException;
-
-public class NoEnoughWorkersException extends IOException {
-
- public NoEnoughWorkersException(String message) {
- super(message);
- }
-
- public NoEnoughWorkersException(int available) {
- this("No enough Workers to create a new Log. Currently available workers: " + available);
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchiveHdfsLogReader.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchiveHdfsLogReader.java
deleted file mode 100644
index 0b8b9b9..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchiveHdfsLogReader.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.impl;
-
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.ratis.logservice.api.ArchiveLogReader;
-import org.apache.ratis.logservice.util.LogServiceUtils;
-import org.apache.ratis.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ArchiveHdfsLogReader implements ArchiveLogReader {
- public static final Logger LOG = LoggerFactory.getLogger(ArchiveHdfsLogReader.class);
- private long fileLength;
- private List<FileStatus> files;
- private FileSystem hdfs;
- private FSDataInputStream is;
- private byte[] currentRecord;
- private int fileCounter = 0;
- private int currentRecordId;
-
- public ArchiveHdfsLogReader(String archiveLocation) throws IOException {
- this(new Configuration(), archiveLocation);
- }
-
- public ArchiveHdfsLogReader(Configuration configuration, String archiveLocation)
- throws IOException {
- this.hdfs = FileSystem.get(configuration);
- Path archiveLocationPath = new Path(archiveLocation);
- if (!hdfs.exists(archiveLocationPath)) {
- throw new FileNotFoundException(archiveLocation);
- }
- files = Arrays.asList(hdfs.listStatus(archiveLocationPath));
- if (files.size() > 0) {
- Collections.sort(files, new Comparator<FileStatus>() {
- @Override public int compare(FileStatus o1, FileStatus o2) {
- //ascending order
- //currently written file (without _recordId_) will be sorted at the last
- return LogServiceUtils.getRecordIdFromRolledArchiveFile(o1.getPath())
- .compareTo(LogServiceUtils.getRecordIdFromRolledArchiveFile(o2.getPath()));
- }
- });
- openNextFilePath();
- loadNext();
- }
- }
-
- private Path openNextFilePath() throws IOException {
- Path filePath = files.get(fileCounter).getPath();
- this.is = this.hdfs.open(filePath);
- this.fileLength = this.hdfs.getFileStatus(filePath).getLen();
- fileCounter++;
- return filePath;
-
- }
-
- @Override public void seek(long recordId) throws IOException {
- while (currentRecordId < recordId && hasNext()) {
- next();
- }
- }
-
- @Override public boolean hasNext() throws IOException {
- return currentRecord != null;
- }
-
- @Override public byte[] next() throws IOException {
- byte[] current = currentRecord;
- currentRecord = null;
- if (current != null) {
- currentRecordId++;
- }
- loadNext();
- return current;
- }
-
- @Override public long getCurrentRaftIndex() {
- throw new UnsupportedOperationException(
- "getCurrentRaftIndex() is not supported for archive hdfs log reader");
- }
-
- @Override public ByteBuffer readNext() throws IOException {
- byte[] current = next();
- if (current == null) {
- throw new NoSuchElementException();
- }
- return ByteBuffer.wrap(current);
- }
-
- private int readLength() throws IOException {
- int length;
- try {
- length = is.readInt();
- } catch (EOFException e) {
- if (files.size() <= fileCounter) {
- LOG.trace("EOF and no more file to read, throwing back", e);
- throw e;
- } else {
- LOG.trace("EOF.. Opening next file: {}!!", files.get(fileCounter).getPath());
- openNextFilePath();
- length = is.readInt();
- }
- }
- return length;
- }
-
- @Override public void readNext(ByteBuffer buffer) throws IOException {
- Preconditions.checkNotNull(buffer, "buffer is NULL");
- byte[] current = next();
- if (current == null) {
- throw new NoSuchElementException();
- }
- buffer.put(current);
- }
-
-
- @Override public List<ByteBuffer> readBulk(int numRecords) throws IOException {
- Preconditions.checkArgument(numRecords > 0, "number of records must be greater than 0");
- List<ByteBuffer> ret = new ArrayList<ByteBuffer>();
- try {
-
- for (int i = 0; i < numRecords; i++) {
- ByteBuffer buffer = readNext();
- ret.add(buffer);
- }
-
- } catch (EOFException eof) {
- } catch (Exception e) {
- throw new IOException(e);
- } finally {
- return ret;
- }
- }
-
- @Override public int readBulk(ByteBuffer[] buffers) throws IOException {
- Preconditions.checkNotNull(buffers, "list of buffers is NULL");
- Preconditions.checkArgument(buffers.length > 0, "list of buffers is empty");
- int count = 0;
- try {
- for (int i = 0; i < buffers.length; i++) {
- readNext(buffers[i]);
- count++;
- }
- } catch (EOFException eof) {
-
- }
- return count;
- }
-
- @Override public long getPosition() throws IOException {
- return currentRecordId;
- }
-
- @Override public void close() throws IOException {
- if (this.is != null) {
- this.is.close();
- this.is = null;
- }
- }
-
- private void loadNext() throws IOException {
- int length;
- try {
- length = readLength();
- } catch (EOFException e) {
- currentRecord = null;
- return;
- }
- byte[] bytes = new byte[length];
- if (is.read(bytes) != length) {
- throw new EOFException(
- "File seems to be corrupted, Encountered EOF before reading the complete record");
- }
- currentRecord = bytes;
- }
-
- //Only for testing
- public List<FileStatus> getFiles(){
- return files;
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchiveHdfsLogWriter.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchiveHdfsLogWriter.java
deleted file mode 100644
index 3a92f73..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchiveHdfsLogWriter.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.impl;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.ratis.logservice.api.ArchiveLogWriter;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.util.LogServiceUtils;
-
-public class ArchiveHdfsLogWriter implements ArchiveLogWriter {
- private final Configuration configuration;
- private FileSystem hdfs;
- private FSDataOutputStream os;
- private Path currentPath;
- private long currentRecordId;
- private long lastRollRecordId;
-
- public ArchiveHdfsLogWriter(Configuration conf) {
- this.configuration = conf;
- }
-
- public ArchiveHdfsLogWriter() {
- this.configuration = new Configuration();
- }
-
- @Override public void init(String archiveLocation, LogName logName) throws IOException {
- hdfs = FileSystem.get(configuration);
- Path loc = new Path(LogServiceUtils.getArchiveLocationForLog(archiveLocation, logName));
- if (!hdfs.exists(loc)) {
- hdfs.mkdirs(loc);
- }
- currentPath = new Path(loc, logName.getName());
- os = hdfs.create(currentPath, true);
- }
-
- @Override public long write(ByteBuffer buffer) throws IOException {
- if (buffer.hasArray()) {
- int startIndex = buffer.arrayOffset();
- int curIndex = buffer.arrayOffset() + buffer.position();
- int endIndex = curIndex + buffer.remaining();
- int length = endIndex - startIndex;
- os.writeInt(length);
- os.write(buffer.array(), startIndex, length);
- } else {
- throw new IllegalArgumentException(
- "Currently array backed byte buffer is only supported for archive write !!");
- }
- currentRecordId++;
- return currentRecordId;
- }
-
- @Override public List<Long> write(List<ByteBuffer> records) throws IOException {
- List<Long> list = new ArrayList<Long>();
- for (ByteBuffer record : records) {
- list.add(write(record));
- }
- return list;
- }
-
- @Override public long sync() throws IOException {
- return 0;
- }
-
- @Override public void close() throws IOException {
- os.close();
- if (lastRollRecordId != currentRecordId) {
- hdfs.rename(currentPath, new Path(currentPath + "_recordId_" + currentRecordId));
- }
- }
-
- @Override public void rollWriter() throws IOException {
- if (lastRollRecordId != currentRecordId) {
- //close old file
- os.close();
- hdfs.rename(currentPath,
- new Path(LogServiceUtils.getRolledPathForArchiveWriter(currentPath, currentRecordId)));
- lastRollRecordId = currentRecordId;
- //create new file
- os = hdfs.create(currentPath, true);
- }
- }
-
- @Override public long getLastWrittenRecordId() throws IOException {
- os.hflush();
- return currentRecordId;
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchivedLogStreamImpl.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchivedLogStreamImpl.java
deleted file mode 100644
index 5648185..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ArchivedLogStreamImpl.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.impl;
-
-import java.io.IOException;
-import java.util.Collection;
-
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.logservice.api.ArchiveLogReader;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogServiceConfiguration;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogWriter;
-import org.apache.ratis.logservice.api.RecordListener;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.util.LogServiceUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ArchivedLogStreamImpl implements LogStream {
- public static final Logger LOG = LoggerFactory.getLogger(ArchivedLogStreamImpl.class);
- /*
- * Directory of the archived files
- */
- private String location;
-
- /*
- * Log stream name
- */
- private LogName name;
- /*
- * Log stream configuration
- */
- private LogServiceConfiguration config;
-
- /*
- * State
- */
- private State state;
-
- protected void setState(State state) {
- this.state = state;
- }
-
- public ArchivedLogStreamImpl(LogName name, LogServiceConfiguration config) {
- this(name, config.get(Constants.LOG_SERVICE_ARCHIVAL_LOCATION_KEY));
- this.config = config;
- init();
- }
-
- protected ArchivedLogStreamImpl(LogName name, String location) {
- this.name = name;
- this.location = location;
- }
-
- protected void init() {
- this.state = State.ARCHIVED;
- }
-
- @Override
- public LogName getName() {
- return name;
- }
-
- @Override public State getState() throws IOException {
- return state;
- }
-
- @Override
- public long getSize() throws IOException{
- throw new UnsupportedOperationException("getSize()");
- }
-
- @Override
- public long getLength() throws IOException {
- throw new UnsupportedOperationException("getLength()");
- }
-
- @Override
- public ArchiveLogReader createReader() throws IOException {
- return new ArchiveHdfsLogReader(LogServiceUtils.getArchiveLocationForLog(location, name));
- }
-
- @Override
- public LogWriter createWriter() {
- throw new UnsupportedOperationException("Archived log cannot be written");
- }
-
- @Override
- public long getLastRecordId() throws IOException {
- throw new UnsupportedOperationException("getLastRecordId()");
- }
-
- @Override
- public long getStartRecordId() throws IOException {
- throw new UnsupportedOperationException("getStartRecordId()");
- }
-
- @Override public Collection<RecordListener> getRecordListeners() {
- throw new UnsupportedOperationException("get record listeners");
- }
-
- @Override
- public LogServiceConfiguration getConfiguration() {
- return config;
- }
-
- @Override
- public void close() throws Exception {
- }
-
- @Override public void addRecordListener(RecordListener listener) {
- throw new UnsupportedOperationException("Add record listener");
- }
-
- @Override public boolean removeRecordListener(RecordListener listener) {
- throw new UnsupportedOperationException("remove record listener");
- }
-
- @Override
- public RaftClient getRaftClient() {
- throw new UnsupportedOperationException("getRaftClient()");
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ExportedLogStreamImpl.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ExportedLogStreamImpl.java
deleted file mode 100644
index 5b7db61..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/ExportedLogStreamImpl.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.impl;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ExportedLogStreamImpl extends ArchivedLogStreamImpl {
- public static final Logger LOG = LoggerFactory.getLogger(ExportedLogStreamImpl.class);
-
- public ExportedLogStreamImpl(LogName name, String location) {
- super(name, location);
- }
-
- @Override
- protected void init() {
- setState(State.CLOSED);
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogReaderImpl.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogReaderImpl.java
deleted file mode 100644
index 477fb29..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogReaderImpl.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.impl;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.logservice.api.LogReader;
-import org.apache.ratis.logservice.api.LogServiceConfiguration;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.proto.LogServiceProtos.*;
-import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.apache.ratis.thirdparty.com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Log Reader implementation. This class is not thread-safe
- *
- */
-
-public class LogReaderImpl implements LogReader {
- public static final Logger LOG = LoggerFactory.getLogger(LogReaderImpl.class);
-
- /*
- * Parent log stream
- */
- private LogStream parent;
- /*
- * Raft client
- */
- private RaftClient raftClient;
- /*
- * Log service configuration object
- */
- private LogServiceConfiguration config;
-
- /*
- * offset
- */
- private long currentRecordId;
-
- public LogReaderImpl(LogStream logStream) {
- this.parent = logStream;
- this.raftClient = logStream.getRaftClient();
- this.config = logStream.getConfiguration();
- }
-
- @Override
- public void seek(long recordId) throws IOException {
- Preconditions.checkArgument(recordId >= 0, "recordId must be >= 0");
- this.currentRecordId = recordId;
- }
-
- @Override
- public ByteBuffer readNext() throws IOException {
-
- try {
- RaftClientReply reply =
- raftClient
- .io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toReadLogRequestProto(parent.getName(), currentRecordId, 1).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- ReadLogReplyProto proto = ReadLogReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
-
- currentRecordId++;
-
- if (proto.getLogRecordCount() > 0) {
- return ByteBuffer.wrap(proto.getLogRecord(0).toByteArray());
- } else {
- return null;
- }
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void readNext(ByteBuffer buffer) throws IOException {
-
- Preconditions.checkNotNull(buffer, "buffer is NULL" );
- try {
- RaftClientReply reply = raftClient.io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toReadLogRequestProto(parent.getName(), currentRecordId, 1).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- ReadLogReplyProto proto = ReadLogReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- currentRecordId++;
- if (proto.getLogRecordCount() > 0) {
- // TODO limits
- buffer.put(proto.getLogRecord(0).toByteArray());
- }
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<ByteBuffer> readBulk(int numRecords) throws IOException {
- Preconditions.checkArgument(numRecords > 0, "number of records must be greater than 0");
-
- try {
- RaftClientReply reply = raftClient
- .io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toReadLogRequestProto(parent.getName(), currentRecordId, numRecords).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- ReadLogReplyProto proto = ReadLogReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- int n = proto.getLogRecordCount();
-
- // TODO correct current record
- currentRecordId += n;
- List<ByteBuffer> ret = new ArrayList<ByteBuffer>();
- for (int i = 0; i < n; i++) {
- ret.add(ByteBuffer.wrap(proto.getLogRecord(i).toByteArray()));
- }
- return ret;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public int readBulk(ByteBuffer[] buffers) throws IOException {
- Preconditions.checkNotNull(buffers, "list of buffers is NULL" );
- Preconditions.checkArgument(buffers.length > 0, "list of buffers is empty");
-
- try {
- RaftClientReply reply = raftClient.io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toReadLogRequestProto(parent.getName(), currentRecordId, buffers.length).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- ReadLogReplyProto proto = ReadLogReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- // TODO correct current record
- int n = proto.getLogRecordCount();
- currentRecordId += n;
- for (int i = 0; i < n; i++) {
- buffers[i] = ByteBuffer.wrap(proto.getLogRecord(i).toByteArray());
- }
- return n;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public long getPosition() {
- return currentRecordId;
- }
-
- @Override
- public void close() throws IOException {
- // TODO Auto-generated method stub
-
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogStreamImpl.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogStreamImpl.java
deleted file mode 100644
index 8eff2e1..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogStreamImpl.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogReader;
-import org.apache.ratis.logservice.api.LogServiceConfiguration;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogWriter;
-import org.apache.ratis.logservice.api.RecordListener;
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.logservice.proto.LogServiceProtos.GetLogLastCommittedIndexReplyProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.GetLogLengthReplyProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.GetLogSizeReplyProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.GetLogStartIndexReplyProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.LogServiceException;
-import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LogStreamImpl implements LogStream {
- public static final Logger LOG = LoggerFactory.getLogger(LogStreamImpl.class);
-
- /*
- * Log stream listeners
- */
- private List<RecordListener> listeners;
- /*
- * Log stream name
- */
- private LogName name;
- /*
- * Parent log service instance
- */
- private RaftClient raftClient;
- /*
- * Log stream configuration
- */
- private LogServiceConfiguration config;
- /*
- * State
- */
- private LogStream.State state;
-
- /*
- * Length
- */
- private long length;
-
-
- public LogStreamImpl(LogName name, RaftClient raftClient) {
- this.raftClient = raftClient;
- this.name = name;
- this.config = LogServiceConfiguration.create();
- init();
- }
-
- public LogStreamImpl(LogName name, RaftClient raftClient, LogServiceConfiguration config) {
- this.raftClient = raftClient;
- this.name = name;
- this.config = config;
- init();
- }
-
- private void init() {
- // TODO create new state machine. etc
- this.state = State.OPEN;
- this.listeners = Collections.synchronizedList(new ArrayList<RecordListener>());
- }
-
- @Override
- public LogName getName() {
- return name;
- }
-
- @Override public State getState() throws IOException {
- RaftClientReply reply = raftClient.io().sendReadOnly(
- Message.valueOf(LogServiceProtoUtil.toGetStateRequestProto(name).toByteString()));
- LogServiceProtos.GetStateReplyProto proto =
- LogServiceProtos.GetStateReplyProto.parseFrom(reply.getMessage().getContent());
- return State.valueOf(proto.getState().name());
- }
-
- @Override
- public long getSize() throws IOException{
- RaftClientReply reply = raftClient
- .io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toGetSizeRequestProto(name).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- GetLogSizeReplyProto proto =
- GetLogSizeReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- return proto.getSize();
- }
-
- @Override
- public long getLength() throws IOException {
- RaftClientReply reply = raftClient
- .io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toGetLengthRequestProto(name).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- GetLogLengthReplyProto proto =
- GetLogLengthReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- return proto.getLength();
- }
-
- @Override
- public LogReader createReader() throws IOException{
- return new LogReaderImpl(this);
- }
-
- @Override
- public LogWriter createWriter() {
- return new LogWriterImpl(this);
- }
-
- @Override
- public long getLastRecordId() throws IOException {
- try {
- RaftClientReply reply = raftClient
- .io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toGetLastCommittedIndexRequestProto(name).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- GetLogLastCommittedIndexReplyProto proto =
- GetLogLastCommittedIndexReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- return proto.getLastCommittedIndex();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public long getStartRecordId() throws IOException {
- try {
- RaftClientReply reply = raftClient
- .io().sendReadOnly(Message.valueOf(LogServiceProtoUtil
- .toGetStartIndexProto(name).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- GetLogStartIndexReplyProto proto =
- GetLogStartIndexReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- return proto.getStartIndex();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public Collection<RecordListener> getRecordListeners() {
- return listeners;
- }
-
- @Override
- public LogServiceConfiguration getConfiguration() {
- return config;
- }
-
- @Override
- public void close() throws Exception {
- // TODO Auto-generated method stub
- raftClient.close();
- state = State.CLOSED;
- }
-
- @Override
- public void addRecordListener(RecordListener listener) {
- synchronized (listeners) {
- if (!listeners.contains(listener)) {
- listeners.add(listener);
- }
- }
- }
-
- @Override
- public boolean removeRecordListener(RecordListener listener) {
- return listeners.remove(listener);
- }
-
- @Override
- public RaftClient getRaftClient() {
- return raftClient;
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogWriterImpl.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogWriterImpl.java
deleted file mode 100644
index 39f1a43..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/impl/LogWriterImpl.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.impl;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.logservice.api.LogServiceConfiguration;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogWriter;
-import org.apache.ratis.logservice.proto.LogServiceProtos.AppendLogEntryReplyProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.LogServiceException;
-import org.apache.ratis.logservice.proto.LogServiceProtos.SyncLogReplyProto;
-import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LogWriterImpl implements LogWriter {
- public static final Logger LOG = LoggerFactory.getLogger(LogWriterImpl.class);
-
- /*
- * Parent log stream
- */
- private LogStream parent;
- /*
- * Raft client
- */
- private RaftClient raftClient;
- /*
- * Log service configuration object
- * TODO: usage of custom configuration
- */
- private LogServiceConfiguration config;
-
- public LogWriterImpl(LogStream logStream) {
- this.parent = logStream;
- this.raftClient = logStream.getRaftClient();
- this.config = logStream.getConfiguration();
- }
-
- @Override
- public long write(ByteBuffer data) throws IOException {
- // One record to write is always one recordId returned
- return write(Collections.singletonList(data)).get(0);
- }
-
- @Override
- public List<Long> write(List<ByteBuffer> list) throws IOException {
- try {
- RaftClientReply reply = raftClient.io().send(
- Message.valueOf(LogServiceProtoUtil.toAppendBBEntryLogRequestProto(parent.getName(), list).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- AppendLogEntryReplyProto proto = AppendLogEntryReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- List<Long> ids = proto.getRecordIdList();
- // The above call Always returns one id (regardless of a batch size)
- return ids;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public long sync() throws IOException {
- try {
- RaftClientReply reply = raftClient.io().send(Message
- .valueOf(LogServiceProtoUtil.toSyncLogRequestProto(parent.getName()).toByteString()));
- if (reply.getException() != null) {
- throw new IOException(reply.getException());
- }
-
- SyncLogReplyProto proto = SyncLogReplyProto.parseFrom(reply.getMessage().getContent());
- if (proto.hasException()) {
- LogServiceException e = proto.getException();
- throw new IOException(e.getErrorMsg());
- }
- return proto.getLastRecordId();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
- @Override
- public void close() throws IOException {
- //TODO
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetaDataMetrics.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetaDataMetrics.java
deleted file mode 100644
index eeb274c..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetaDataMetrics.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.metrics;
-
-import org.apache.ratis.metrics.MetricRegistryInfo;
-import org.apache.ratis.metrics.RatisMetricRegistry;
-import org.apache.ratis.metrics.RatisMetrics;
-
-import com.codahale.metrics.Timer;
-
-import static org.apache.ratis.logservice.metrics.LogServiceMetrics.RATIS_LOG_SERVICE_METRICS;
-
-public final class LogServiceMetaDataMetrics extends RatisMetrics {
- public static final String RATIS_LOG_SERVICE_META_DATA_METRICS = "metadata_statemachine";
- public static final String RATIS_LOG_SERVICE_META_DATA_METRICS_DESC =
- "Ratis log service metadata metrics";
-
- public LogServiceMetaDataMetrics(String serverId) {
- registry = getMetricRegistryForLogServiceMetaData(serverId);
- }
-
- private RatisMetricRegistry getMetricRegistryForLogServiceMetaData(String serverId) {
- return create(new MetricRegistryInfo(serverId,
- RATIS_LOG_SERVICE_METRICS,
- RATIS_LOG_SERVICE_META_DATA_METRICS, RATIS_LOG_SERVICE_META_DATA_METRICS_DESC));
- }
-
- public Timer getTimer(String name) {
- return registry.timer(name);
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetrics.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetrics.java
deleted file mode 100644
index b92655b..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/metrics/LogServiceMetrics.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.metrics;
-
-import org.apache.ratis.metrics.MetricRegistryInfo;
-import org.apache.ratis.metrics.RatisMetricRegistry;
-import org.apache.ratis.metrics.RatisMetrics;
-
-import com.codahale.metrics.Timer;
-
-public final class LogServiceMetrics extends RatisMetrics {
- public static final String RATIS_LOG_STATEMACHINE_METRICS = "log_statemachine";
- public static final String RATIS_LOG_SERVICE_METRICS = "ratis_log_service";
- public static final String RATIS_LOG_SERVICE_METRICS_DESC = "Ratis log service metrics";
-
- public LogServiceMetrics(String logName, String serverId) {
- registry = getMetricRegistryForLogService(logName, serverId);
- }
-
- private RatisMetricRegistry getMetricRegistryForLogService(String logName, String serverId) {
- return create(new MetricRegistryInfo(logName + "." + serverId,
- RATIS_LOG_SERVICE_METRICS,
- RATIS_LOG_STATEMACHINE_METRICS, RATIS_LOG_SERVICE_METRICS_DESC));
- }
-
- public Timer getTimer(String name) {
- return registry.timer(name);
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ArchivalInfo.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ArchivalInfo.java
deleted file mode 100644
index 0c2b460..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ArchivalInfo.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.server;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-
-public class ArchivalInfo {
-
- public enum ArchivalStatus{
- /*
- Initial state when the archival/export request is submitted and request
- is recorded at the leader but the single thread responsible for archiving
- has not started processing it
- */
- SUBMITTED,
- /*
- Archiving/exporting of the particular request has been started and
- file will appear soon in archival location during this state
- */
- STARTED,
- /*
- Archival is ongoing and at least one file is rolled as well
- */
- RUNNING,
- /*
- Archiving on the current leader will get interrupted
- when it become a follower after re-election.
- and a request to new leader will be submitted after some delay
- to avoid leader election storm, if a new request fails , archival
- status will be changed to FAILED and log state back to CLOSED so that
- user can submit request again
- */
- INTERRUPTED,
- /*
- Archival/export request is successfully completed
- */
- COMPLETED,
- /*
- Archival/export request is failed due to the error,
- worker logs should have trace for it
- After fixing the issue , archival request can be resubmitted
- */
- FAILED
- }
- private String archiveLocation;
- private LogName archiveLogName;
- private long lastArchivedIndex;
- private ArchivalStatus status;
-
- public ArchivalInfo(String location) {
- this.archiveLocation = location;
- }
-
-
- public ArchivalInfo updateArchivalInfo(LogServiceProtos.ArchiveLogRequestProto archiveLog) {
- this.archiveLogName = LogServiceProtoUtil.toLogName(archiveLog.getLogName());
- this.lastArchivedIndex = archiveLog.getLastArchivedRaftIndex();
- this.status = ArchivalStatus.valueOf(archiveLog.getStatus().name());
- return this;
- }
-
- public String getArchiveLocation() {
- return archiveLocation;
- }
-
- public LogName getArchiveLogName() {
- return archiveLogName;
- }
-
- public long getLastArchivedIndex() {
- return lastArchivedIndex;
- }
-
- public ArchivalStatus getStatus(){
- return status;
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/BaseServer.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/BaseServer.java
deleted file mode 100644
index 30a938e..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/BaseServer.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.server;
-
-import java.io.Closeable;
-import java.net.InetSocketAddress;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.logservice.api.LogServiceConfiguration;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.util.LogServiceUtils;
-import org.apache.ratis.netty.NettyConfigKeys;
-import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.util.NetUtils;
-import org.apache.ratis.util.TimeDuration;
-
-/**
- * A base class to encapsulate functionality around a long-lived Java process which runs a state machine.
- */
-public abstract class BaseServer implements Closeable {
-
- private final ServerOpts opts;
- private final LogServiceConfiguration config;
-
- public BaseServer(ServerOpts opts) {
- this.opts = Objects.requireNonNull(opts);
- this.config = LogServiceConfiguration.create();
- }
-
- public ServerOpts getServerOpts() {
- return opts;
- }
-
- public LogServiceConfiguration getConfig() {
- return config;
- }
- /**
- * Sets common Ratis server properties for both the log and metadata state machines.
- */
- void setRaftProperties(RaftProperties properties) {
- // Set the ports for the server
- GrpcConfigKeys.Server.setPort(properties, opts.getPort());
- NettyConfigKeys.Server.setPort(properties, opts.getPort());
-
- // Ozone sets the leader election timeout (min) to 1second.
- long leaderElectionTimeoutMinVal = getLeaderElectionTimeoutMin();
- TimeDuration leaderElectionTimeoutMin = TimeDuration.valueOf(leaderElectionTimeoutMinVal,
- TimeUnit.MILLISECONDS);
- RaftServerConfigKeys.Rpc.setTimeoutMin(properties, leaderElectionTimeoutMin);
- long leaderElectionTimeoutMaxVal = getLeaderElectionTimeoutMax();
-
- TimeDuration leaderElectionMaxTimeout = TimeDuration.valueOf(
- leaderElectionTimeoutMaxVal,
- TimeUnit.MILLISECONDS);
- RaftServerConfigKeys.Rpc.setTimeoutMax(properties, leaderElectionMaxTimeout);
- }
-
- private long getLeaderElectionTimeoutMin() {
- return config.getLong(Constants.LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MIN_KEY,
- Constants.DEFAULT_LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MIN);
- }
-
- private long getLeaderElectionTimeoutMax() {
- return config.getLong(Constants.LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MAX_KEY,
- Constants.DEFAULT_LOG_SERVICE_LEADER_ELECTION_TIMEOUT_MAX);
- }
-
- /**
- * Validates that there are no properties set which are in conflict with the LogService.
- */
- void validateRaftProperties(RaftProperties properties) {
- if (RaftServerConfigKeys.Snapshot.autoTriggerEnabled(properties)) {
- throw new IllegalStateException("Auto triggering snapshots is disallowed by the LogService");
- }
- }
-
- static ServerOpts buildOpts(String hostname, String metaQuorum, int port, String workingDir) {
- ServerOpts opts = new ServerOpts();
- opts.setHost(hostname);
- opts.setMetaQuorum(metaQuorum);
- opts.setPort(port);
- opts.setWorkingDir(workingDir);
- return opts;
- }
-
- public abstract static class Builder<T extends BaseServer> {
- private ServerOpts opts = new ServerOpts();
-
- protected ServerOpts getOpts() {
- return opts;
- }
-
- public abstract T build();
-
- public Builder<T> validate() {
- if (!opts.isPortSet()) {
- InetSocketAddress addr = NetUtils.createLocalServerAddress();
- opts.setPort(addr.getPort());
- }
- if (!opts.isHostSet()) {
- opts.setHost(LogServiceUtils.getHostName());
- }
- if (!opts.isWorkingDirSet()) {
- throw new IllegalArgumentException("Working directory was not specified");
- }
- return this;
- }
-
- public Builder<T> setMetaQuorum(String meta) {
- opts.setMetaQuorum(meta);
- return this;
- }
-
- public Builder<T> setPort(int port) {
- opts.setPort(port);
- return this;
- }
-
- public Builder<T> setWorkingDir(String workingDir) {
- opts.setWorkingDir(workingDir);
- return this;
- }
-
- public Builder<T> setHostName(String hostName) {
- opts.setHost(hostName);
- return this;
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogServer.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogServer.java
deleted file mode 100644
index 1a4da83..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogServer.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.server;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Collections;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.client.RaftClientConfigKeys;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.logservice.api.LogServiceConfiguration;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.util.LogServiceUtils;
-import org.apache.ratis.logservice.util.MetaServiceProtoUtil;
-import org.apache.ratis.protocol.ClientId;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftGroupId;
-import org.apache.ratis.protocol.RaftPeer;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.statemachine.StateMachine;
-import org.apache.ratis.util.Daemon;
-import org.apache.ratis.util.SizeInBytes;
-import org.apache.ratis.util.TimeDuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-
-public class LogServer extends BaseServer {
- private static final Logger LOG = LoggerFactory.getLogger(LogServer.class);
-
- private RaftServer raftServer = null;
- private RaftClient metaClient = null;
-
- private Daemon daemon = null;
- private long heartbeatInterval = Constants.DEFAULT_HEARTBEAT_INTERVAL;
- public LogServer(ServerOpts opts) {
- super(opts);
- LOG.debug("Log Server options: {}", opts);
- }
-
- public RaftServer getServer() {
- return raftServer;
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- @Override
- void setRaftProperties(RaftProperties properties) {
- super.setRaftProperties(properties);
-
- // Increase the client timeout
- long rpcTimeout = getConfig().getLong(Constants.LOG_SERVICE_RPC_TIMEOUT_KEY,
- Constants.DEFAULT_RPC_TIMEOUT);
- RaftClientConfigKeys.Rpc.setRequestTimeout(properties,
- TimeDuration.valueOf(rpcTimeout, TimeUnit.MILLISECONDS));
-
- // Increase the segment size to avoid rolling so quickly
- long segmentSize = getConfig().getLong(Constants.RATIS_RAFT_SEGMENT_SIZE_KEY,
- Constants.DEFAULT_RATIS_RAFT_SEGMENT_SIZE);
- SizeInBytes segmentSizeBytes = SizeInBytes.valueOf(segmentSize);
- String archiveLocation = getConfig().get(Constants.LOG_SERVICE_ARCHIVAL_LOCATION_KEY);
- if (archiveLocation != null) {
- properties.set(Constants.LOG_SERVICE_ARCHIVAL_LOCATION_KEY, archiveLocation);
- }
- heartbeatInterval = getConfig().getLong(Constants.LOG_SERVICE_HEARTBEAT_INTERVAL_KEY,
- Constants.DEFAULT_HEARTBEAT_INTERVAL);
- if(heartbeatInterval <= 0) {
- LOG.warn("Heartbeat interval configuration is invalid." +
- " Setting default value "+ Constants.DEFAULT_HEARTBEAT_INTERVAL);
- heartbeatInterval = Constants.DEFAULT_HEARTBEAT_INTERVAL;
- }
- RaftServerConfigKeys.Log.setSegmentSizeMax(properties, segmentSizeBytes);
- RaftServerConfigKeys.Log.setPreallocatedSize(properties, segmentSizeBytes);
-
- // TODO this seems to cause errors, not sure if pushing Ratis too hard?
- // SizeInBytes writeBufferSize = SizeInBytes.valueOf("128KB");
- // RaftServerConfigKeys.Log.setWriteBufferSize(properties, writeBufferSize);
- }
-
- public void start() throws IOException {
- final ServerOpts opts = getServerOpts();
- Set<RaftPeer> peers = LogServiceUtils.getPeersFromQuorum(opts.getMetaQuorum());
- RaftProperties properties = new RaftProperties();
-
- // Set properties for the log server state machine
- setRaftProperties(properties);
-
- InetSocketAddress addr = new InetSocketAddress(opts.getHost(), opts.getPort());
- if(opts.getWorkingDir() != null) {
- RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(new File(opts.getWorkingDir())));
- }
- String id = opts.getHost() +"_" + opts.getPort();
- final RaftPeer peer = RaftPeer.newBuilder().setId(id).setAddress(addr).build();
- final RaftGroupId logServerGroupId = RaftGroupId.valueOf(opts.getLogServerGroupId());
- RaftGroup all = RaftGroup.valueOf(logServerGroupId, peer);
- RaftGroup meta = RaftGroup.valueOf(RaftGroupId.valueOf(opts.getMetaGroupId()), peers);
-
- // Make sure that we aren't setting any invalid/harmful properties
- validateRaftProperties(properties);
-
- raftServer = RaftServer.newBuilder()
- .setStateMachineRegistry(new StateMachine.Registry() {
- @Override
- public StateMachine apply(RaftGroupId raftGroupId) {
- // TODO this looks wrong. Why isn't this metaGroupId?
- if(raftGroupId.equals(logServerGroupId)) {
- return new ManagementStateMachine();
- }
- return new LogStateMachine(properties);
- }
- })
- .setProperties(properties)
- .setServerId(RaftPeerId.valueOf(id))
- .setGroup(all)
- .build();
- raftServer.start();
-
- metaClient = RaftClient.newBuilder()
- .setRaftGroup(meta)
- .setClientId(ClientId.randomId())
- .setProperties(properties)
- .build();
- metaClient.io().send(() -> MetaServiceProtoUtil.toPingRequestProto(peer).toByteString());
- daemon = new Daemon(new HeartbeatSender(RaftPeer.newBuilder().setId(raftServer.getId()).build()),
- "heartbeat-Sender"+raftServer.getId());
- daemon.start();
- }
-
- public static void main(String[] args) throws IOException {
- ServerOpts opts = new ServerOpts();
- JCommander.newBuilder()
- .addObject(opts)
- .build()
- .parse(args);
- // Add config from log service configuration file
- LogServiceConfiguration config = LogServiceConfiguration.create();
- opts = config.addLogServerOpts(opts);
-
- try (LogServer worker = new LogServer(opts)) {
- worker.start();
- while (true) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return;
- }
- }
- }
- }
-
-
- public void close() throws IOException {
- metaClient.close();
- raftServer.close();
- daemon.interrupt();
- }
-
- public static class Builder extends BaseServer.Builder<LogServer> {
- public LogServer build() {
- validate();
- return new LogServer(getOpts());
- }
- }
-
- private class HeartbeatSender implements Runnable {
-
- private RaftPeer peer;
- HeartbeatSender(RaftPeer peer) {
- this.peer = peer;
- }
-
- @Override
- public void run() {
-
- while (true) {
- try {
- metaClient.io().send(() -> MetaServiceProtoUtil.
- toHeartbeatRequestProto(peer).toByteString());
- Thread.sleep(heartbeatInterval);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return;
- } catch (IOException e) {
- LOG.warn("Heartbeat request failed with exception", e);
- }
- }
-
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogServiceRaftLogReader.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogServiceRaftLogReader.java
deleted file mode 100644
index 42e6d53..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogServiceRaftLogReader.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.server;
-
-import static java.util.Objects.requireNonNull;
-
-import java.util.NoSuchElementException;
-
-import org.apache.ratis.logservice.proto.LogServiceProtos.AppendLogEntryRequestProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.LogServiceRequestProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.LogServiceRequestProto.RequestCase;
-import org.apache.ratis.proto.RaftProtos.LogEntryProto;
-import org.apache.ratis.server.raftlog.RaftLog;
-import org.apache.ratis.server.raftlog.RaftLogIOException;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * A reader for the {@link RaftLog} which is accessed using LogService recordId's instead
- * of Raft log indexes. Not thread-safe.
- */
-public class LogServiceRaftLogReader implements RaftLogReader{
- private static final Logger LOG = LoggerFactory.getLogger(LogServiceRaftLogReader.class);
- private final RaftLog raftLog;
-
- private long currentRecordId = -1;
- private long currentRaftIndex = -1;
- private AppendLogEntryRequestProto currentLogEntry = null;
- private int currentLogEntryOffset = -1;
- private ByteString currentRecord = null;
-
- public LogServiceRaftLogReader(RaftLog raftLog) {
- this.raftLog = requireNonNull(raftLog);
- }
-
- /**
- * Positions this reader just before the current recordId. Use {@link #next()} to get that
- * element, but take care to check if a value is present using {@link #hasNext()} first.
- */
- @Override
- public void seek(long recordId) throws RaftLogIOException, InvalidProtocolBufferException {
- LOG.trace("Seeking to recordId={}", recordId);
- // RaftLog starting index
- currentRaftIndex = raftLog.getStartIndex();
- currentRecordId = 0;
-
- currentLogEntry = null;
- currentLogEntryOffset = -1;
- currentRecord = null;
-
- loadNext();
- while (currentRecordId < recordId && hasNext()) {
- next();
- currentRecordId++;
- }
- }
-
- /**
- * Returns true if there is a log entry to read.
- */
- @Override
- public boolean hasNext() throws RaftLogIOException, InvalidProtocolBufferException {
- return currentRecord != null;
- }
-
- /**
- * Returns the next log entry. Ensure {@link #hasNext()} returns true before
- * calling this method.
- */
- @Override
- public byte[] next() throws RaftLogIOException, InvalidProtocolBufferException {
- if (currentRecord == null) {
- throw new NoSuchElementException();
- }
- ByteString current = currentRecord;
- currentRecord = null;
- loadNext();
- return current.toByteArray();
- }
-
- /**
- * Finds the next record from the RaftLog and sets it as {@link #currentRecord}.
- */
- private void loadNext() throws RaftLogIOException, InvalidProtocolBufferException {
- // Clear the old "current" record
- currentRecord = null;
- if (LOG.isTraceEnabled()) {
- LOG.trace("Loading next value: raftIndex={}, recordId={}, proto='{}', offset={}",
- currentRaftIndex, currentRecordId,
- currentLogEntry == null ? "null" : TextFormat.shortDebugString(currentLogEntry),
- currentLogEntryOffset);
- }
- // Continue iterating over the current entry.
- if (currentLogEntry != null) {
- assert currentLogEntryOffset != -1;
- currentLogEntryOffset++;
-
- // We have an element to read from our current entry
- if (currentLogEntryOffset < currentLogEntry.getDataCount()) {
- currentRecord = currentLogEntry.getData(currentLogEntryOffset);
- return;
- }
- // We don't have an element in our current entry so null it out.
- currentLogEntry = null;
- currentLogEntryOffset = -1;
- // Also, increment to the next element in the RaftLog
- currentRaftIndex++;
- }
-
- // Make sure we don't read off the end of the Raft log
- for (; currentRaftIndex <= raftLog.getLastCommittedIndex(); currentRaftIndex++) {
- try {
- LogEntryProto entry = raftLog.get(currentRaftIndex);
- if (LOG.isTraceEnabled()) {
- LOG.trace("Raft Index: {} Entry: {}", currentRaftIndex,
- TextFormat.shortDebugString(entry));
- }
- if (entry == null || entry.hasConfigurationEntry()) {
- continue;
- }
-
- LogServiceRequestProto logServiceProto =
- LogServiceRequestProto.parseFrom(entry.getStateMachineLogEntry().getLogData());
- // TODO is it possible to get LogService messages that aren't appends?
- if (RequestCase.APPENDREQUEST != logServiceProto.getRequestCase()) {
- continue;
- }
-
- currentLogEntry = logServiceProto.getAppendRequest();
- currentLogEntryOffset = 0;
- if (currentLogEntry.getDataCount() > 0) {
- currentRecord = currentLogEntry.getData(currentLogEntryOffset);
- return;
- }
- currentLogEntry = null;
- currentLogEntryOffset = -1;
- } catch (RaftLogIOException e) {
- LOG.error("Caught exception reading from RaftLog", e);
- throw e;
- } catch (InvalidProtocolBufferException e) {
- LOG.error("Caught exception reading LogService protobuf from RaftLog", e);
- throw e;
- }
- }
- // If we make it here, we've read off the end of the RaftLog.
- }
-
- @Override
- public long getCurrentRaftIndex(){
- return currentRaftIndex;
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogStateMachine.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogStateMachine.java
deleted file mode 100644
index 7d307f2..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/LogStateMachine.java
+++ /dev/null
@@ -1,787 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.server;
-
-import static org.apache.ratis.logservice.api.LogStream.State;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
-
-import com.codahale.metrics.Timer;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.logservice.api.ArchiveLogWriter;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.impl.ArchiveHdfsLogReader;
-import org.apache.ratis.logservice.impl.ArchiveHdfsLogWriter;
-import org.apache.ratis.logservice.metrics.LogServiceMetrics;
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.logservice.proto.LogServiceProtos.AppendLogEntryRequestProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.GetLogLengthRequestProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.GetLogSizeRequestProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.LogServiceRequestProto;
-import org.apache.ratis.logservice.proto.LogServiceProtos.ReadLogRequestProto;
-import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-import org.apache.ratis.logservice.util.LogServiceUtils;
-import org.apache.ratis.proto.RaftProtos;
-import org.apache.ratis.proto.RaftProtos.LogEntryProto;
-import org.apache.ratis.protocol.ClientId;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftGroupId;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.logservice.server.ArchivalInfo.ArchivalStatus;
-import org.apache.ratis.server.protocol.TermIndex;
-import org.apache.ratis.server.raftlog.RaftLog;
-import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.statemachine.StateMachineStorage;
-import org.apache.ratis.statemachine.TransactionContext;
-import org.apache.ratis.statemachine.impl.BaseStateMachine;
-import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
-import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo;
-import org.apache.ratis.thirdparty.com.google.protobuf.TextFormat;
-import org.apache.ratis.util.AutoCloseableLock;
-import org.apache.ratis.util.JavaUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class LogStateMachine extends BaseStateMachine {
- public static final Logger LOG = LoggerFactory.getLogger(LogStateMachine.class);
- public static final long DEFAULT_ARCHIVE_THRESHOLD_PER_FILE = 1000000;
- private final RaftProperties properties;
- private LogServiceMetrics logServiceMetrics;
- private Timer sizeRequestTimer;
- private Timer readNextQueryTimer;
- private Timer getStateTimer;
- private Timer lastIndexQueryTimer;
- private Timer lengthQueryTimer;
- private Timer startIndexTimer;
- private Timer appendRequestTimer;
- private Timer syncRequesTimer;
- private Timer archiveLogRequestTimer;
- private Timer getCloseLogTimer;
- private RaftClient client;
- //Archival information
- /*
- * State is a log's length, size, and state (closed/open);
- */
- private long length;
-
- /**
- * The size (number of bytes) of the log records. Does not include Ratis storage overhead
- */
- private long dataRecordsSize;
-
- private State state = State.OPEN;
-
- private final SimpleStateMachineStorage storage = new SimpleStateMachineStorage();
-
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
-
- private RaftLog log;
-
-
- private RaftServer proxy;
- private ExecutorService executorService;
- private boolean isArchivalRequest;
- private ArchivalInfo archivalInfo;
- private Map<String,ArchivalInfo> exportMap = new HashMap<String,ArchivalInfo>();
- private Map<String, Future<Boolean>> archiveExportFutures = new HashMap<>();
- private Timer archiveLogTimer;
-
- public LogStateMachine(RaftProperties properties) {
- this.properties = properties;
- }
-
- private AutoCloseableLock readLock() {
- return AutoCloseableLock.acquire(lock.readLock());
- }
-
- private AutoCloseableLock writeLock() {
- return AutoCloseableLock.acquire(lock.writeLock());
- }
-
- /**
- * Reset state machine
- */
- void reset() {
- this.length = 0;
- this.dataRecordsSize = 0;
- setLastAppliedTermIndex(TermIndex.valueOf(0, -1));
- }
-
- @Override
- public void initialize(RaftServer server, RaftGroupId groupId,
- RaftStorage raftStorage) throws IOException {
- super.initialize(server, groupId, raftStorage);
- this.storage.init(raftStorage);
- this.proxy = server;
- //TODO: using groupId for metric now but better to tag it with LogName
- this.logServiceMetrics = new LogServiceMetrics(groupId.toString(),
- server.getId().toString());
- this.readNextQueryTimer = logServiceMetrics.getTimer("readNextQueryTime");
- this.startIndexTimer= logServiceMetrics.getTimer("startIndexTime");
- this.sizeRequestTimer = logServiceMetrics.getTimer("sizeRequestTime");
- this.getStateTimer = logServiceMetrics.getTimer("getStateTime");
- this.lastIndexQueryTimer = logServiceMetrics.getTimer("lastIndexQueryTime");
- this.lengthQueryTimer = logServiceMetrics.getTimer("lengthQueryTime");
- this.syncRequesTimer = logServiceMetrics.getTimer("syncRequesTime");
- this.appendRequestTimer = logServiceMetrics.getTimer("appendRequestTime");
- this.getCloseLogTimer = logServiceMetrics.getTimer("getCloseLogTime");
- //archiving request time not the actual archiving time
- this.archiveLogRequestTimer = logServiceMetrics.getTimer("archiveLogRequestTime");
- this.archiveLogTimer = logServiceMetrics.getTimer("archiveLogTime");
- loadSnapshot(storage.getLatestSnapshot());
- executorService = Executors.newSingleThreadExecutor();
- this.archivalInfo =
- new ArchivalInfo(properties.get(Constants.LOG_SERVICE_ARCHIVAL_LOCATION_KEY));
-
-
- }
-
- private void checkInitialization() throws IOException {
- if (this.log == null) {
- this.log = proxy.getDivision(getGroupId()).getRaftLog();
- }
- }
-
- @Override
- public void reinitialize() throws IOException {
- close();
- loadSnapshot(storage.getLatestSnapshot());
- }
-
- @Override
- public long takeSnapshot() {
- final TermIndex last;
- try(AutoCloseableLock readLock = readLock()) {
- last = getLastAppliedTermIndex();
- }
-
- final File snapshotFile = storage.getSnapshotFile(last.getTerm(), last.getIndex());
- LOG.info("Taking a snapshot to file {}", snapshotFile);
-
- try(AutoCloseableLock readLock = readLock();
- ObjectOutputStream out = new ObjectOutputStream(
- new BufferedOutputStream(new FileOutputStream(snapshotFile)))) {
- out.writeLong(length);
- out.writeLong(dataRecordsSize);
- out.writeObject(state);
- } catch(IOException ioe) {
- LOG.warn("Failed to write snapshot file \"" + snapshotFile
- + "\", last applied index=" + last);
- }
-
- return last.getIndex();
- }
-
- private long loadSnapshot(SingleFileSnapshotInfo snapshot) throws IOException {
- return load(snapshot, false);
- }
-
- private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException {
- if (snapshot == null) {
- LOG.warn("The snapshot info is null.");
- return RaftLog.INVALID_LOG_INDEX;
- }
- final File snapshotFile = snapshot.getFile().getPath().toFile();
- if (!snapshotFile.exists()) {
- LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot);
- return RaftLog.INVALID_LOG_INDEX;
- }
-
- final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile);
- try(AutoCloseableLock writeLock = writeLock();
- ObjectInputStream in = new ObjectInputStream(
- new BufferedInputStream(new FileInputStream(snapshotFile)))) {
- if (reload) {
- reset();
- }
- setLastAppliedTermIndex(last);
- this.length = in.readLong();
- this.dataRecordsSize = in.readLong();
- this.state = (State) in.readObject();
- } catch (ClassNotFoundException e) {
- throw new IllegalStateException(e);
- }
- return last.getIndex();
- }
-
- @Override
- public StateMachineStorage getStateMachineStorage() {
- return storage;
- }
-
- @Override
- public CompletableFuture<Message> query(Message request) {
-
- try {
-
- checkInitialization();
- LogServiceRequestProto logServiceRequestProto =
- LogServiceRequestProto.parseFrom(request.getContent());
- if (LOG.isTraceEnabled()) {
- LOG.trace("Processing LogService query: {}", TextFormat.shortDebugString(logServiceRequestProto));
- }
-
- switch (logServiceRequestProto.getRequestCase()) {
-
- case READNEXTQUERY:
- return recordTime(readNextQueryTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processReadRequest(logServiceRequestProto);
- }
- });
- case SIZEREQUEST:
- return recordTime(sizeRequestTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processGetSizeRequest(logServiceRequestProto);
- }
- });
- case STARTINDEXQUERY:
- return recordTime(startIndexTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processGetStartIndexRequest(logServiceRequestProto);
- }
- });
- case GETSTATE:
- return recordTime(getStateTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processGetStateRequest();
- }
- });
- case LASTINDEXQUERY:
- return recordTime(lastIndexQueryTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processGetLastCommittedIndexRequest(logServiceRequestProto);
- }
- });
- case LENGTHQUERY:
- return recordTime(lengthQueryTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processGetLengthRequest(logServiceRequestProto);
- }
- });
- case ARCHIVELOG:
- return recordTime(archiveLogRequestTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processArchiveLog(logServiceRequestProto);
- }});
- case EXPORTINFO:
- return processExportInfo();
- default:
- // TODO
- throw new RuntimeException(
- "Wrong message type for query: " + logServiceRequestProto.getRequestCase());
- }
-
- } catch (IOException e) {
- // TODO exception handling
- throw new RuntimeException(e);
- }
-
- }
-
- private CompletableFuture<Message> processExportInfo() {
- LogServiceProtos.GetExportInfoReplyProto.Builder exportBuilder =
- LogServiceProtos.GetExportInfoReplyProto.newBuilder();
- exportMap.values().forEach(
- archInfo -> exportBuilder.addInfo(LogServiceProtoUtil.toExportInfoProto(archInfo)));
-
- return CompletableFuture.completedFuture(Message.valueOf(exportBuilder.build().toByteString()));
- }
-
- /**
- * Process get start index request
- * @param proto message
- * @return reply message
- */
- private CompletableFuture<Message>
- processGetStartIndexRequest(LogServiceRequestProto proto) {
-
- Throwable t = verifyState(State.OPEN);
- long startIndex = log.getStartIndex();
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtoUtil.toGetLogStartIndexReplyProto(startIndex, t).toByteString()));
- }
-
- /**
- * Process get last committed record index
- * @param proto message
- * @return reply message
- */
- private CompletableFuture<Message>
- processGetLastCommittedIndexRequest(LogServiceRequestProto proto) {
- Throwable t = verifyState(State.OPEN);
- long lastIndex = log.getLastCommittedIndex();
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtoUtil.toGetLogLastIndexReplyProto(lastIndex, t).toByteString()));
- }
-
- /**
- * Process get length request
- * @param proto message
- * @return reply message
- */
- private CompletableFuture<Message> processGetSizeRequest(LogServiceRequestProto proto) {
- GetLogSizeRequestProto msgProto = proto.getSizeRequest();
- Throwable t = verifyState(State.OPEN);
- LOG.trace("Size query: {}, Result: {}", msgProto, this.dataRecordsSize);
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtoUtil.toGetLogSizeReplyProto(this.dataRecordsSize, t).toByteString()));
- }
-
- private CompletableFuture<Message> processGetLengthRequest(LogServiceRequestProto proto) {
- GetLogLengthRequestProto msgProto = proto.getLengthQuery();
- Throwable t = verifyState(State.OPEN);
- LOG.trace("Length query: {}, Result: {}", msgProto, this.length);
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtoUtil.toGetLogLengthReplyProto(this.length, t).toByteString()));
- }
- /**
- * Process read log entries request
- * @param proto message
- * @return reply message
- */
- private CompletableFuture<Message> processReadRequest(LogServiceRequestProto proto) {
- ReadLogRequestProto msgProto = proto.getReadNextQuery();
- // Get the recordId the user wants to start reading at
- long startRecordId = msgProto.getStartRecordId();
- // And the number of records they want to read
- int numRecordsToRead = msgProto.getNumRecords();
- //Log must have been closed while Archiving , so we can let user only to
- // read when the log is either OPEN or ARCHIVED
- Throwable t = verifyState(State.OPEN, State.ARCHIVING, State.CLOSED, State.ARCHIVED);
- List<byte[]> list = null;
-
- if (t == null) {
- RaftLogReader reader = null;
- try {
- if (this.state == State.OPEN || this.state == State.CLOSED
- || this.state == State.ARCHIVING) {
- reader = new LogServiceRaftLogReader(log);
- } else if (this.state == State.ARCHIVED) {
- reader = new ArchiveHdfsLogReader(LogServiceUtils
- .getArchiveLocationForLog(archivalInfo.getArchiveLocation(),
- archivalInfo.getArchiveLogName()));
- } else {
- //could be a race condition
- t = verifyState(State.OPEN, State.ARCHIVED);
- }
- if (t == null && reader != null) {
- list = new ArrayList<byte[]>();
- reader.seek(startRecordId);
- for (int i = 0; i < numRecordsToRead; i++) {
- if (!reader.hasNext()) {
- break;
- }
- list.add(reader.next());
- }
- }
- } catch (Exception e) {
- LOG.error("Failed to execute ReadNextQuery", e);
- t = e;
- list = null;
- }
- }
- return CompletableFuture.completedFuture(
- Message.valueOf(LogServiceProtoUtil.toReadLogReplyProto(list, t).toByteString()));
- }
-
- /**
- * Process sync request
- * @param trx transaction
- * @param logMessage message
- * @return reply message
- */
- private CompletableFuture<Message> processSyncRequest(TransactionContext trx,
- LogServiceRequestProto logMessage) {
- long index = trx.getLogEntry().getIndex();
- // TODO: Do we really need this call?
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtoUtil.toSyncLogReplyProto(index, null).toByteString()));
-
- }
-
- private CompletableFuture<Message> processAppendRequest(TransactionContext trx,
- LogServiceRequestProto logProto) {
-
- final LogEntryProto entry = trx.getLogEntry();
- AppendLogEntryRequestProto proto = logProto.getAppendRequest();
- final long index = entry.getIndex();
- long newSize = 0;
- Throwable t = verifyState(State.OPEN);
- final List<Long> ids = new ArrayList<Long>();
- if (t == null) {
- try (AutoCloseableLock writeLock = writeLock()) {
- List<byte[]> entries = LogServiceProtoUtil.toListByteArray(proto.getDataList());
- for (byte[] bb : entries) {
- ids.add(this.length);
- newSize += bb.length;
- this.length++;
- }
- this.dataRecordsSize += newSize;
- // TODO do we need this for other write request (close, sync)
- updateLastAppliedTermIndex(entry.getTerm(), index);
- }
- }
- final CompletableFuture<Message> f =
- CompletableFuture.completedFuture(
- Message.valueOf(LogServiceProtoUtil.toAppendLogReplyProto(ids, t).toByteString()));
- final RaftProtos.RaftPeerRole role = trx.getServerRole();
- if (LOG.isTraceEnabled()) {
- LOG.trace("{}:{}-{}: {} new length {}", role, getId(), index,
- TextFormat.shortDebugString(proto), dataRecordsSize);
- }
- return f;
- }
-
- @Override
- public void close() {
- reset();
- logServiceMetrics.unregister();
- if (client != null) {
- try {
- client.close();
- } catch (Exception ignored) {
- LOG.warn("{} is ignored", JavaUtils.getClassSimpleName(ignored.getClass()), ignored);
- }
- }
- }
-
- @Override
- public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
- try {
- checkInitialization();
- final LogEntryProto entry = trx.getLogEntry();
- LogServiceRequestProto logServiceRequestProto =
- LogServiceRequestProto.parseFrom(entry.getStateMachineLogEntry().getLogData());
- switch (logServiceRequestProto.getRequestCase()) {
- case CHANGESTATE:
- return recordTime(getCloseLogTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processChangeState(logServiceRequestProto);
- }});
- case APPENDREQUEST:
- return recordTime(appendRequestTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processAppendRequest(trx, logServiceRequestProto);
- }});
- case SYNCREQUEST:
- return recordTime(syncRequesTimer, new Task(){
- @Override public CompletableFuture<Message> run() {
- return processSyncRequest(trx, logServiceRequestProto);
- }});
- case ARCHIVELOG:
- return updateArchiveLogInfo(logServiceRequestProto);
- default:
- //TODO
- return null;
- }
- } catch (IOException e) {
- // TODO exception handling
- throw new RuntimeException(e);
- }
- }
-
-
-
- private CompletableFuture<Message> processChangeState(LogServiceRequestProto logServiceRequestProto) {
- LogServiceProtos.ChangeStateLogRequestProto changeState = logServiceRequestProto.getChangeState();
- // Need to check whether the file is opened if opened close it.
- // TODO need to handle exceptions while operating with files.
-
- State targetState = State.valueOf(changeState.getState().name());
- Throwable t = null;
- //if forced skip checking states
- if(!changeState.getForce()) {
- switch (targetState) {
- case OPEN:
- if (state != null) {
- t = verifyState(State.OPEN, State.CLOSED);
- }
- break;
- case CLOSED:
- t = verifyState(State.OPEN);
- break;
- case ARCHIVED:
- t = verifyState(State.ARCHIVING);
- break;
- case ARCHIVING:
- t = verifyState(State.CLOSED);
- break;
- case DELETED:
- t = verifyState(State.CLOSED);
- break;
- default:
- }
- }
- if(t != null) {
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtos.ChangeStateReplyProto.newBuilder().
- setException(LogServiceProtoUtil.toLogException(t)).build().toByteString()));
- }
- this.state = targetState;
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtos.ChangeStateReplyProto.newBuilder().build().toByteString()));
- }
-
- private CompletableFuture<Message> processGetStateRequest() {
- return CompletableFuture.completedFuture(Message
- .valueOf(LogServiceProtoUtil.toGetStateReplyProto(state).toByteString()));
- }
-
- private Throwable verifyState(State... states) {
- for (State st : states) {
- if (this.state == st) {
- return null;
- }
- }
- return new IOException("Wrong state: " + this.state);
- }
-
- private CompletableFuture<Message> updateArchiveLogInfo(
- LogServiceRequestProto logServiceRequestProto) {
- LogServiceProtos.ArchiveLogRequestProto archiveLog = logServiceRequestProto.getArchiveLog();
- this.isArchivalRequest = !archiveLog.getIsExport();
- Throwable t = null;
- if(isArchivalRequest) {
- archivalInfo.updateArchivalInfo(archiveLog);
- t = verifyState(State.ARCHIVING);
- }else{
- t = verifyState(State.OPEN, State.CLOSED);
- ArchivalInfo info = exportMap.get(archiveLog.getLocation());
- if(info==null) {
- info = new ArchivalInfo(archiveLog.getLocation());
- exportMap.put(archiveLog.getLocation(),info);
- }
- info.updateArchivalInfo(archiveLog);
- }
- return CompletableFuture.completedFuture(
- Message.valueOf(LogServiceProtoUtil.toArchiveLogReplyProto(t).toByteString()));
- }
-
- private CompletableFuture<Message> processArchiveLog(
- LogServiceRequestProto logServiceRequestProto) {
- LogServiceProtos.ArchiveLogRequestProto archiveLog = logServiceRequestProto.getArchiveLog();
- LogName logName = LogServiceProtoUtil.toLogName(archiveLog.getLogName());
- Throwable t = null;
- try {
- String loc = null;
- this.isArchivalRequest = !archiveLog.getIsExport();
- if (isArchivalRequest) {
- loc = archivalInfo.getArchiveLocation();
- archivalInfo.updateArchivalInfo(archiveLog);
- } else {
- loc = archiveLog.getLocation();
- ArchivalInfo exportInfo =
- exportMap.putIfAbsent(loc, new ArchivalInfo(loc));
- if (exportInfo != null) {
- if (exportInfo.getLastArchivedIndex() == archiveLog
- .getLastArchivedRaftIndex()) {
- throw new IllegalStateException("Export of " + logName + "for the given location " + loc
- + "is already present and in " + exportInfo.getStatus());
- } else {
- exportInfo.updateArchivalInfo(archiveLog);
- }
- }
- }
- if (loc == null) {
- throw new IllegalArgumentException(isArchivalRequest ?
- "Location for archive is not configured" :
- "Location for export provided is null");
- }
- final String location = loc;
- long recordId = archiveLog.getLastArchivedRaftIndex();
- if (isArchivalRequest) {
- t = verifyState(State.CLOSED);
- } else {
- t = verifyState(State.OPEN, State.CLOSED);
- }
- if (t == null) {
- Callable<Boolean> callable = () -> {
- final Timer.Context timerContext = archiveLogTimer.time();
- try {
- startArchival(recordId, logName, location);
- //Init ArchiveLogWriter for writing in export/archival location
- ArchiveLogWriter writer = new ArchiveHdfsLogWriter();
- writer.init(location, logName);
-
- LogServiceRaftLogReader reader = new LogServiceRaftLogReader(log);
- reader.seek(recordId);
- long records = 0;
- boolean isInterrupted = false;
- while (reader.hasNext()) {
- writer.write(ByteBuffer.wrap(reader.next()));
- isInterrupted = Thread.currentThread().isInterrupted();
- if (records >= DEFAULT_ARCHIVE_THRESHOLD_PER_FILE || isInterrupted) {
- //roll writer when interuppted or no. of records threshold per file is met
- commit(writer, logName, location);
- if (isInterrupted) {
- break;
- }
- records = 0;
- }
- records++;
- }
- writer.close();
- if (!isInterrupted) {
- //It means archival is successfully completed on this leader
- completeArchival(writer.getLastWrittenRecordId(), logName, location);
- } else {
- //Thread is interuppted either leader is going down or it become follower
- try {
- //Sleeping here before sending archival request to new leader to
- // avoid causing problem during leader election storm
- Thread.sleep(10000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- sendArchiveLogrequestToNewLeader(writer.getLastWrittenRecordId(), logName, location);
- }
- return true;
- } catch (Exception e) {
- LOG.error("Archival failed for the log:" + logName, e);
- failArchival(recordId, logName, location);
- } finally {
- timerContext.stop();
- }
- return false;
- };
-
- archiveExportFutures.put(location, executorService.submit(callable));
- }
- }catch (Exception e){
- LOG.warn("Exception while processing archival request for " + logName, e);
- t = e;
- }
- return CompletableFuture.completedFuture(
- Message.valueOf(LogServiceProtoUtil.toArchiveLogReplyProto(t).toByteString()));
- }
-
- private void failArchival(long recordId, LogName logName, String location) throws IOException {
- updateArchivingInfo(recordId, logName, location, isArchivalRequest,
- ArchivalStatus.FAILED);
- if(isArchivalRequest) {
- sendChangeStateRequest(State.CLOSED, true);
- }
- }
-
- private void startArchival(long recordId, LogName logName, String location) throws IOException {
- if (isArchivalRequest) {
- sendChangeStateRequest(State.ARCHIVING, false);
- }
- updateArchivingInfo(recordId, logName, location, isArchivalRequest, ArchivalStatus.STARTED);
- }
-
- private void sendArchiveLogrequestToNewLeader(long recordId, LogName logName, String location)
- throws IOException {
- getClient().io().sendReadOnly(() -> LogServiceProtoUtil
- .toArchiveLogRequestProto(logName, location, recordId, isArchivalRequest,
- ArchivalStatus.INTERRUPTED).toByteString());
- }
-
- public void completeArchival(long recordId, LogName logName, String location) throws IOException {
- if (isArchivalRequest) {
- sendChangeStateRequest(State.ARCHIVED, false);
- }
- updateArchivingInfo(recordId, logName, location, isArchivalRequest, ArchivalStatus.COMPLETED);
- }
-
- private void commit(ArchiveLogWriter writer, LogName logName, String location)
- throws IOException {
- writer.rollWriter();
- updateArchivingInfo(writer.getLastWrittenRecordId(), logName, location, isArchivalRequest,
- ArchivalStatus.RUNNING);
- }
-
- private void updateArchivingInfo(long recordId, LogName logName, String location,
- boolean isArchival, ArchivalStatus status)
- throws IOException {
- RaftClientReply archiveLogReply = getClient().io().send(() -> LogServiceProtoUtil
- .toArchiveLogRequestProto(logName, location, recordId, isArchival, status).toByteString());
- LogServiceProtos.ArchiveLogReplyProto message=LogServiceProtos.ArchiveLogReplyProto
- .parseFrom(archiveLogReply.getMessage().getContent());
- if (message.hasException()) {
- throw new IOException(message.getException().getErrorMsg());
- }
- }
-
- private void sendChangeStateRequest(State st, boolean force) throws IOException {
- getClient().io().send(
- () -> LogServiceProtoUtil.toChangeStateRequestProto(LogName.of("Dummy"), st, force)
- .toByteString());
- }
-
- private RaftClient getClient() throws IOException {
- if (client == null) {
- try {
- RaftServer raftServer = getServer().get();
- client = RaftClient.newBuilder().setRaftGroup(getGroupFromGroupId(raftServer, getGroupId()))
- .setClientId(ClientId.randomId())
- .setProperties(raftServer.getProperties()).build();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
- return client;
- }
-
- private RaftGroup getGroupFromGroupId(RaftServer raftServer, RaftGroupId raftGroupId)
- throws IOException {
- List<RaftGroup> x = StreamSupport.stream(raftServer.getGroups().spliterator(), false)
- .filter(group -> group.getGroupId().equals(raftGroupId)).collect(Collectors.toList());
- if (x.size() == 1) {
- return x.get(0);
- } else {
- throw new IOException(x.size() + " are group found for group id:" + raftGroupId);
- }
- }
-
- @Override public void notifyNotLeader(Collection<TransactionContext> pendingEntries)
- throws IOException {
- for(Future<Boolean> archiveFuture:archiveExportFutures.values()) {
- if (archiveFuture != null && !archiveFuture.isCancelled()) {
- archiveFuture.cancel(true);
- }
- }
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ManagementStateMachine.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ManagementStateMachine.java
deleted file mode 100644
index 2b018bb..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ManagementStateMachine.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.server;
-
-import org.apache.ratis.statemachine.impl.BaseStateMachine;
-
-
-/**
- * This is the statemachine for the default group serversGroupID. At the moment it's empty, but would
- * have some logic related to the heartbeat functionality.
- */
-public class ManagementStateMachine extends BaseStateMachine {
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetaStateMachine.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetaStateMachine.java
deleted file mode 100644
index b88a048..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetaStateMachine.java
+++ /dev/null
@@ -1,526 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.server;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.PriorityBlockingQueue;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.StreamSupport;
-
-import com.codahale.metrics.Timer;
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.logservice.api.LogInfo;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.common.LogAlreadyExistException;
-import org.apache.ratis.logservice.common.LogNotFoundException;
-import org.apache.ratis.logservice.common.NoEnoughWorkersException;
-import org.apache.ratis.logservice.metrics.LogServiceMetaDataMetrics;
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.logservice.proto.MetaServiceProtos;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.CreateLogRequestProto;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.DeleteLogRequestProto;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.LogServicePingRequestProto;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.LogServiceRegisterLogRequestProto;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.LogServiceUnregisterLogRequestProto;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.MetaSMRequestProto;
-import org.apache.ratis.logservice.util.LogServiceProtoUtil;
-import org.apache.ratis.logservice.util.MetaServiceProtoUtil;
-import org.apache.ratis.proto.RaftProtos;
-import org.apache.ratis.protocol.*;
-
-
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.statemachine.TransactionContext;
-import org.apache.ratis.statemachine.impl.BaseStateMachine;
-import org.apache.ratis.thirdparty.com.google.common.annotations.VisibleForTesting;
-import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.ratis.util.AutoCloseableLock;
-import org.apache.ratis.util.Daemon;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * State Machine serving meta data for LogService. It persists the pairs 'log name' -> RaftGroup
- * During the start basing on the persisted data it would be able to build a list of the existing servers.
- * Requests from clients for DDL operations are handled by query mechanism (so only the leader accept that.
- * It performs the operation (such as Log creation) and sends a message with the log -> group pair to itself
- * to persis this data internally and on followers.
- */
-
-public class MetaStateMachine extends BaseStateMachine {
-
- private static final Logger LOG = LoggerFactory.getLogger(MetaStateMachine.class);
-
-
- //Persisted map between log and RaftGroup
- private Map<LogName, RaftGroup> map = new ConcurrentHashMap<>();
- // List of the currently known peers.
- private final Set<RaftPeer> peers = new HashSet<>();
-
- // keep a copy of raftServer to get group information.
- private RaftServer raftServer;
-
- private Map<RaftPeer, Set<LogName>> peerLogs = new ConcurrentHashMap<>();
-
- private Map<RaftPeer, Long> heartbeatInfo = new ConcurrentHashMap<>();
-
- private RaftGroup currentGroup = null;
-
- private Daemon peerHealthChecker = null;
- // MinHeap queue for load balancing groups across the peers
- private long failureDetectionPeriod = Constants.DEFAULT_PEER_FAILURE_DETECTION_PERIOD;
- private PriorityBlockingQueue<PeerGroups> avail = new PriorityBlockingQueue<PeerGroups>();
-
- //Properties
- private RaftProperties properties = new RaftProperties();
-
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
-
- private RaftGroupId metadataGroupId;
- private RaftGroupId logServerGroupId;
- private LogServiceMetaDataMetrics logServiceMetaDataMetrics;
-
- public MetaStateMachine(RaftGroupId metadataGroupId, RaftGroupId logServerGroupId,
- long failureDetectionPeriod) {
- this.metadataGroupId = metadataGroupId;
- this.logServerGroupId = logServerGroupId;
- this.failureDetectionPeriod = failureDetectionPeriod;
- }
-
- @Override
- public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage storage) throws IOException {
- this.raftServer = server;
- this.logServiceMetaDataMetrics = new LogServiceMetaDataMetrics(server.getId().toString());
- super.initialize(server, groupId, storage);
- peerHealthChecker = new Daemon(new PeerHealthChecker(),"peer-Health-Checker");
- peerHealthChecker.start();
- }
-
- @VisibleForTesting
- public void setProperties(RaftProperties properties) {
- this.properties = properties;
- }
-
- @Override
- public TransactionContext applyTransactionSerial(TransactionContext trx) throws InvalidProtocolBufferException {
- RaftProtos.LogEntryProto x = trx.getLogEntry();
- MetaSMRequestProto req = null;
- try {
- req = MetaSMRequestProto.parseFrom(x.getStateMachineLogEntry().getLogData());
- } catch (InvalidProtocolBufferException e) {
- e.printStackTrace();
- throw e;
- }
- switch (req.getTypeCase()) {
- case REGISTERREQUEST:
- LogServiceRegisterLogRequestProto r = req.getRegisterRequest();
- LogName logname = LogServiceProtoUtil.toLogName(r.getLogname());
- RaftGroup rg = MetaServiceProtoUtil.toRaftGroup(r.getRaftGroup());
- rg.getPeers().stream().forEach(raftPeer -> {
- Set<LogName> logNames;
- if(!peerLogs.containsKey(raftPeer)) {
- logNames = new HashSet<>();
- peerLogs.put(raftPeer, logNames);
- } else {
- logNames = peerLogs.get(raftPeer);
- }
- logNames.add(logname);
-
- });
- map.put(logname, rg);
-
- LOG.info("Log {} registered at {} with group {} ", logname, getId(), rg );
- break;
- case UNREGISTERREQUEST:
- LogServiceUnregisterLogRequestProto unregReq = req.getUnregisterRequest();
- logname = LogServiceProtoUtil.toLogName(unregReq.getLogname());
- map.remove(logname);
- break;
- case PINGREQUEST:
- LogServicePingRequestProto pingRequest = req.getPingRequest();
- RaftPeer peer = MetaServiceProtoUtil.toRaftPeer(pingRequest.getPeer());
- //If Set<RaftPeer> contains peer then do nothing as that's just heartbeat else add the peer to the set.
- if (!peers.contains(peer)) {
- peers.add(peer);
- avail.add(new PeerGroups(peer));
- heartbeatInfo.put(peer, System.currentTimeMillis());
- }
- break;
- case HEARTBEATREQUEST:
- MetaServiceProtos.LogServiceHeartbeatRequestProto heartbeatRequest = req.getHeartbeatRequest();
- RaftPeer heartbeatPeer = MetaServiceProtoUtil.toRaftPeer(heartbeatRequest.getPeer());
- heartbeatInfo.put(heartbeatPeer, System.currentTimeMillis());
- break;
- default:
- }
- return super.applyTransactionSerial(trx);
- }
-
- @Override
- public TransactionContext startTransaction(RaftClientRequest request) throws IOException {
- return super.startTransaction(request);
- }
-
- @Override
- public TransactionContext preAppendTransaction(TransactionContext trx) throws IOException {
- return super.preAppendTransaction(trx);
- }
-
- @Override
- public CompletableFuture<Message> queryStale(Message request, long minIndex) {
- return super.queryStale(request, minIndex);
- }
-
- @Override
- public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
- return super.applyTransaction(trx);
- }
-
-
- @Override
- public CompletableFuture<Message> query(Message request) {
- Timer.Context timerContext = null;
- MetaServiceProtos.MetaServiceRequestProto.TypeCase type = null;
- if (currentGroup == null) {
- try {
- List<RaftGroup> x =
- StreamSupport.stream(raftServer.getGroups().spliterator(), false)
- .filter(group -> group.getGroupId().equals(metadataGroupId)).collect(Collectors.toList());
- if (x.size() == 1) {
- currentGroup = x.get(0);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- MetaServiceProtos.MetaServiceRequestProto req;
- try {
- req = MetaServiceProtos.MetaServiceRequestProto.parseFrom(request.getContent());
- } catch (InvalidProtocolBufferException e) {
- e.printStackTrace();
- return null;
- }
- type = req.getTypeCase();
- // Main purpose of this try catch block is to make sure that
- // timerContext.stop() is run after return.
- try {
- timerContext = logServiceMetaDataMetrics.getTimer(type.name()).time();
- switch (type) {
-
- case CREATELOG:
- return processCreateLogRequest(req);
- case LISTLOGS:
- return processListLogsRequest();
- case GETLOG:
- return processGetLogRequest(req);
- case DELETELOG:
- return processDeleteLog(req);
- default:
- CompletableFuture<Message> reply = super.query(request);
- return reply;
- }
- } catch (Exception e) {
- LOG.error("Exception during Meta State Machine query");
- throw e;
- } finally {
- if (timerContext != null) {
- timerContext.stop();
- }
- }
- }
-
-
-
- private CompletableFuture<Message>
- processDeleteLog(MetaServiceProtos.MetaServiceRequestProto logServiceRequestProto) {
- DeleteLogRequestProto deleteLog = logServiceRequestProto.getDeleteLog();
- LogName logName = LogServiceProtoUtil.toLogName(deleteLog.getLogName());
- RaftGroup raftGroup = map.get(logName);
- if (raftGroup == null) {
- return CompletableFuture.completedFuture(Message.valueOf(
- MetaServiceProtoUtil.toDeleteLogExceptionReplyProto(
- new LogNotFoundException(logName.getName())).build().toByteString()));
- } else {
- Collection<RaftPeer> raftPeers = raftGroup.getPeers();
- raftPeers.stream().forEach(peer -> {
- try (RaftClient client = RaftClient.newBuilder().setProperties(properties)
- .setClientId(ClientId.randomId()).setRaftGroup(RaftGroup.valueOf(logServerGroupId, peer)).build()){
- client.getGroupManagementApi(peer.getId()).remove(raftGroup.getGroupId(), true, false);
- } catch (IOException e) {
- e.printStackTrace();
- }
- });
- try (RaftClient client = RaftClient.newBuilder().setRaftGroup(currentGroup)
- .setClientId(ClientId.randomId()).setProperties(properties).build()){
- client.io().send(() -> MetaServiceProtos.MetaSMRequestProto.newBuilder()
- .setUnregisterRequest(
- LogServiceUnregisterLogRequestProto.newBuilder()
- .setLogname(LogServiceProtoUtil.toLogNameProto(logName)))
- .build().toByteString());
- } catch (IOException e) {
- LOG.error(
- "Exception while unregistring raft group with Metadata Service during deletion of log");
- e.printStackTrace();
- }
- }
- return CompletableFuture.completedFuture(Message.valueOf(
- MetaServiceProtoUtil.toDeleteLogReplyProto().toByteString()));
- }
-
-
- private CompletableFuture<Message> processCreateLogRequest(
- MetaServiceProtos.MetaServiceRequestProto logServiceRequestProto) {
- LogName name;
- try (AutoCloseableLock writeLock = writeLock()) {
- CreateLogRequestProto createLog = logServiceRequestProto.getCreateLog();
- name = LogServiceProtoUtil.toLogName(createLog.getLogName());
- if(map.containsKey(name)) {
- return CompletableFuture.completedFuture(Message.valueOf(MetaServiceProtoUtil
- .toCreateLogExceptionReplyProto(new LogAlreadyExistException(name.getName()))
- .build()
- .toByteString()));
- }
- // Check that we have at least 3 nodes
- if (avail.size() < 3) {
- return CompletableFuture.completedFuture(Message.valueOf(MetaServiceProtoUtil
- .toCreateLogExceptionReplyProto(new NoEnoughWorkersException(avail.size()))
- .build()
- .toByteString()));
- } else {
- List<PeerGroups> peerGroup =
- IntStream.range(0, 3).mapToObj(i -> avail.poll()).collect(Collectors.toList());
- List<RaftPeer> peersFromGroup =
- peerGroup.stream().map(obj -> obj.getPeer()).collect(Collectors.toList());
- RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.randomId(), peersFromGroup);
- peerGroup.stream().forEach(pg -> {
- pg.getGroups().add(raftGroup);
- avail.add(pg);
- });
- int provisionedPeers = 0;
- Exception originalException = null;
- for (RaftPeer peer : peers) {
- try (RaftClient client = RaftClient.newBuilder().setProperties(properties)
- .setRaftGroup(RaftGroup.valueOf(logServerGroupId, peer)).build()) {
- client.getGroupManagementApi(peer.getId()).add(raftGroup);
- } catch (IOException e) {
- LOG.error("Failed to add Raft group ({}) for new Log({})",
- raftGroup.getGroupId(), name, e);
- originalException = e;
- break;
- }
- provisionedPeers++;
- }
- // If we fail to add the group on all three peers, try to remove the group(s) which
- // failed to be added.
- if (provisionedPeers != peers.size()) {
- int tornDownPeers = 0;
- for (RaftPeer peer : peers) {
- if (tornDownPeers >= provisionedPeers) {
- break;
- }
- try (RaftClient client = RaftClient.newBuilder().setProperties(properties)
- .setRaftGroup(RaftGroup.valueOf(logServerGroupId, peer)).build()) {
- client.getGroupManagementApi(peer.getId()).remove(raftGroup.getGroupId(), true, false);
- } catch (IOException e) {
- LOG.error("Failed to clean up Raft group ({}) for peer ({}), "
- + "ignoring exception", raftGroup.getGroupId(), peer, e);
- }
- tornDownPeers++;
- }
- // Make sure to send the original exception back to the client.
- return CompletableFuture.completedFuture(Message.valueOf(
- MetaServiceProtoUtil.toCreateLogExceptionReplyProto(originalException)
- .build().toByteString()));
- }
- try (RaftClient client = RaftClient.newBuilder().setRaftGroup(currentGroup)
- .setClientId(ClientId.randomId()).setProperties(properties).build()){
- client.io().send(() -> MetaServiceProtos.MetaSMRequestProto.newBuilder()
- .setRegisterRequest(LogServiceRegisterLogRequestProto.newBuilder()
- .setLogname(LogServiceProtoUtil.toLogNameProto(name))
- .setRaftGroup(MetaServiceProtoUtil
- .toRaftGroupProto(raftGroup)))
- .build().toByteString());
- } catch (IOException e) {
- LOG.error(
- "Exception while registering raft group with Metadata Service during creation of log");
- // Make sure to send the original exception back to the client.
- return CompletableFuture.completedFuture(Message.valueOf(
- MetaServiceProtoUtil.toCreateLogExceptionReplyProto(e).build()
- .toByteString()));
- }
- return CompletableFuture.completedFuture(Message.valueOf(MetaServiceProtoUtil
- .toCreateLogReplyProto(new LogInfo((name), raftGroup)).build().toByteString()));
- }
- }
- }
-
-
- private AutoCloseableLock writeLock() {
- return AutoCloseableLock.acquire(lock.writeLock());
- }
-
- private CompletableFuture<Message> processListLogsRequest() {
- return CompletableFuture.completedFuture(Message.valueOf(MetaServiceProtoUtil
- .toListLogLogsReplyProto(
- map.entrySet()
- .stream()
- .map(log -> new LogInfo(log.getKey(), log.getValue()))
- .collect(Collectors.toList())).toByteString()));
- }
-
- private CompletableFuture<Message> processGetLogRequest(
- MetaServiceProtos.MetaServiceRequestProto logServiceRequestProto) {
- MetaServiceProtos.GetLogRequestProto getLog = logServiceRequestProto.getGetLog();
- LogName logName = LogServiceProtoUtil.toLogName(getLog.getLogName());
- RaftGroup raftGroup = map.get(logName);
- if (raftGroup != null) {
- return CompletableFuture.completedFuture(Message.valueOf(
- MetaServiceProtoUtil.toGetLogReplyProto(new LogInfo(logName, raftGroup))
- .toByteString()));
- } else {
- return CompletableFuture.completedFuture(Message.valueOf(
- MetaServiceProtoUtil.toGetLogExceptionReplyProto(
- new LogNotFoundException(logName.getName())).build().toByteString()));
- }
- }
-
-
- static class PeerGroups implements Comparable{
- private RaftPeer peer;
- private Set<RaftGroup> groups = new HashSet<>();
-
- PeerGroups(RaftPeer peer) {
- this.peer = peer;
- }
-
- public Set<RaftGroup> getGroups () {
- return groups;
- }
-
- public RaftPeer getPeer() {
- return peer;
- }
-
- @Override
- @SuppressFBWarnings("EQ_COMPARETO_USE_OBJECT_EQUALS")
- public int compareTo(Object o) {
- return groups.size() - ((PeerGroups) o).groups.size();
- }
- }
-
- private class PeerHealthChecker implements Runnable {
- @Override
- public void run() {
- while(true) {
- try {
- Thread.sleep(1000);
- long now = System.currentTimeMillis();
- heartbeatInfo.keySet().stream().forEach(raftPeer -> {
- Long heartbeatTimestamp = heartbeatInfo.get(raftPeer);
- // Introduce configuration for period to detect the failure.
- if((now - heartbeatTimestamp) > failureDetectionPeriod) {
- // Close the logs serve by peer if any.
- if (peerLogs.containsKey(raftPeer)) {
- LOG.warn("Closing all logs hosted by peer {} because last heartbeat ({}ms) exceeds " +
- "the threshold ({}ms)", raftPeer, now - heartbeatTimestamp, failureDetectionPeriod);
- peers.remove(raftPeer);
- Set<LogName> logNames = peerLogs.get(raftPeer);
- Iterator<LogName> itr = logNames.iterator();
- while(itr.hasNext()) {
- LogName logName = itr.next();
- RaftGroup group = map.get(logName);
- try (RaftClient client = RaftClient.newBuilder()
- .setRaftGroup(group).setProperties(properties).build()) {
- LOG.warn(String.format("Peer %s in the group %s went down." +
- " Hence closing the log %s serve by the group.",
- raftPeer.toString(), group.toString(), logName.toString()));
- RaftClientReply reply = client.io().send(
- () -> LogServiceProtoUtil.
- toChangeStateRequestProto(logName, LogStream.State.CLOSED, true)
- .toByteString());
- LogServiceProtos.ChangeStateReplyProto message =
- LogServiceProtos.ChangeStateReplyProto.parseFrom(
- reply.getMessage().getContent());
- if(message.hasException()) {
- throw new IOException(message.getException().getErrorMsg());
- }
- itr.remove();
- } catch (IOException e) {
- LOG.warn(String.format("Failed to close log %s on peer %s failure.",
- logName, raftPeer.toString()), e);
- }
- }
- if(logNames.isEmpty()) {
- peerLogs.remove(raftPeer);
- heartbeatInfo.remove(raftPeer);
- } // else retry closing failed logs on next period.
- }
- final List<PeerGroups> peerGroupsToRemove = new ArrayList<>();
- // remove peer groups from avail.
- avail.stream().forEach(peerGroup -> {
- if(peerGroup.getPeer().equals(raftPeer)) {
- peerGroupsToRemove.add(peerGroup);
- }
- });
- for(PeerGroups peerGroups: peerGroupsToRemove) {
- avail.remove(peerGroups);
- }
- }
- });
- } catch (Exception e) {
- LOG.error(
- "Exception while closing logs and removing peer" +
- " from raft groups with Metadata Service on node failure", e);
- }
- }
- }
- }
-
-
- // This method need to be used for testing only.
- public boolean checkPeersAreSame() {
- if(!peers.equals(peerLogs.keySet()) || !peers.equals(heartbeatInfo.keySet())) {
- return false;
- }
- Set<RaftPeer> availPeers = new HashSet<>();
- avail.stream().forEach(peerGroups -> {
- availPeers.add(peerGroups.getPeer());
- });
- if(!peers.equals(availPeers)) {
- return false;
- }
- return true;
- }
-
- @Override
- public void close() {
- logServiceMetaDataMetrics.unregister();
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetadataServer.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetadataServer.java
deleted file mode 100644
index 41e4433..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/MetadataServer.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.server;
-
-import com.beust.jcommander.JCommander;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.logservice.api.LogServiceConfiguration;
-import org.apache.ratis.logservice.common.Constants;
-import org.apache.ratis.logservice.util.LogServiceUtils;
-import org.apache.ratis.protocol.*;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.statemachine.StateMachine;
-import org.apache.ratis.util.FileUtils;
-import org.apache.ratis.util.LifeCycle;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Set;
-
-import static org.apache.ratis.logservice.common.Constants.META_GROUP_ID;
-import static org.apache.ratis.logservice.util.LogServiceUtils.getPeersFromQuorum;
-
-/**
- * Master quorum is responsible for tracking all available quorum members
- */
-public class MetadataServer extends BaseServer {
- private static final Logger LOG = LoggerFactory.getLogger(MetadataServer.class);
-
- // RaftServer internal server. Has meta raft group and MetaStateMachine
- private RaftServer server;
-
- private String id;
-
- private StateMachine metaStateMachine;
-
- private LifeCycle lifeCycle;
-
- public StateMachine getMetaStateMachine() {
- return metaStateMachine;
- }
-
- public MetadataServer(ServerOpts opts) {
- super(opts);
- LOG.debug("Metadata Server options: {}", opts);
- this.id = opts.getHost() + "_" + opts.getPort();
- this.lifeCycle = new LifeCycle(this.id);
- }
-
- public void start() throws IOException {
- final ServerOpts opts = getServerOpts();
- if (opts.getHost() == null) {
- opts.setHost(LogServiceUtils.getHostName());
- }
- this.lifeCycle = new LifeCycle(this.id);
- RaftProperties properties = new RaftProperties();
- if(opts.getWorkingDir() != null) {
- RaftServerConfigKeys.setStorageDir(properties,
- Collections.singletonList(new File(opts.getWorkingDir())));
- } else {
- String path = getConfig().get(Constants.META_SERVER_WORKDIR_KEY);
- if (path != null) {
- RaftServerConfigKeys.setStorageDir(properties,
- Collections.singletonList(new File(path)));
- }
- }
-
- // Set properties common to all log service state machines
- setRaftProperties(properties);
- long failureDetectionPeriod = getConfig().
- getLong(Constants.LOG_SERVICE_PEER_FAILURE_DETECTION_PERIOD_KEY,
- Constants.DEFAULT_PEER_FAILURE_DETECTION_PERIOD);
- Set<RaftPeer> peers = getPeersFromQuorum(opts.getMetaQuorum());
- RaftGroupId raftMetaGroupId = RaftGroupId.valueOf(opts.getMetaGroupId());
- RaftGroup metaGroup = RaftGroup.valueOf(raftMetaGroupId, peers);
- metaStateMachine = new MetaStateMachine(raftMetaGroupId, RaftGroupId.valueOf(opts.getLogServerGroupId()),
- failureDetectionPeriod);
-
- // Make sure that we aren't setting any invalid/harmful properties
- validateRaftProperties(properties);
-
- server = RaftServer.newBuilder()
- .setGroup(metaGroup)
- .setServerId(RaftPeerId.valueOf(id))
- .setStateMachineRegistry(raftGroupId -> {
- if(raftGroupId.equals(META_GROUP_ID)) {
- return metaStateMachine;
- }
- return null;
- })
- .setProperties(properties).build();
- lifeCycle.startAndTransition(() -> {
- server.start();
- }, IOException.class);
- }
-
- public static void main(String[] args) throws IOException {
- ServerOpts opts = new ServerOpts();
- JCommander.newBuilder()
- .addObject(opts)
- .build()
- .parse(args);
- // Add config from log service configuration file
- LogServiceConfiguration config = LogServiceConfiguration.create();
- opts = config.addMetaServerOpts(opts);
-
- try (MetadataServer master = new MetadataServer(opts)) {
- master.start();
- while (true) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return;
- }
- }
- }
- }
-
- public static MetadataServer.Builder newBuilder() {
- return new MetadataServer.Builder();
- }
-
- @Override
- public void close() throws IOException {
- server.close();
- }
-
- public String getId() {
- return id;
- }
-
- public String getAddress() {
- return getServerOpts().getHost() + ":" + getServerOpts().getPort();
- }
-
- public void cleanUp() throws IOException {
- FileUtils.deleteFully(new File(getServerOpts().getWorkingDir()));
- }
-
- public static class Builder extends BaseServer.Builder<MetadataServer> {
- /**
- * @return a {@link MetadataServer} object.
- */
- public MetadataServer build() {
- validate();
- return new MetadataServer(getOpts());
- }
- }
-
- public RaftServer getServer() {
- return server;
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/RaftLogReader.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/RaftLogReader.java
deleted file mode 100644
index b92bedf..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/RaftLogReader.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.server;
-
-import java.io.IOException;
-
-public interface RaftLogReader {
-
- /**
- * Positions this reader just before the current recordId. Use {@link #next()} to get that
- * element, but take care to check if a value is present using {@link #hasNext()} first.
- */
- void seek(long recordId) throws IOException;
-
- /**
- * Returns true if there is a log entry to read.
- */
- boolean hasNext() throws IOException;
-
- /**
- * Returns the next log entry. Ensure {@link #hasNext()} returns true before
- * calling this method.
- */
- byte[] next() throws IOException;
-
- /**
- * Returns current raft index read
- * @return
- */
- long getCurrentRaftIndex();
-
- }
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ServerOpts.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ServerOpts.java
deleted file mode 100644
index 3e76620..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/server/ServerOpts.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.server;
-
-import java.util.UUID;
-
-import org.apache.ratis.logservice.common.Constants;
-
-import com.beust.jcommander.IStringConverter;
-import com.beust.jcommander.Parameter;
-
-/**
- * Options that are common across metadata and log server classes.
- */
-public class ServerOpts {
- /**
- * Converts a string into a UUID
- */
- static class UUIDConverter implements IStringConverter<UUID> {
- @Override public UUID convert(String value) {
- return UUID.fromString(value);
- }
- }
-
- @Parameter(names = {"-h", "--hostname"}, description = "Hostname")
- private String host = null;
-
- @Parameter(names = {"-p", "--port"}, description = "Port number")
- private int port = -1;
-
- @Parameter(names = {"-d", "--dir"}, description = "Working directory")
- private String workingDir = null;
-
- @Parameter(names = {"-q", "--metaQuorum"}, description = "Metadata Service Quorum", required = true)
- private String metaQuorum = null;
-
- @Parameter(names = {"--metadataServerGroupId"}, description = "UUID corresponding to the RAFT metadata servers group",
- converter = UUIDConverter.class)
- private UUID metaGroupId = Constants.META_GROUP_UUID;
-
- @Parameter(names = {"--logServerGroupId"}, description = "UUID corresponding to the RAFT log servers group",
- converter = UUIDConverter.class)
- private UUID logServerGroupId = Constants.SERVERS_GROUP_UUID;
-
- public String getHost() {
- return host;
- }
-
- public void setHost(String host) {
- this.host = host;
- }
-
- public boolean isHostSet() {
- return this.host != null;
- }
-
- public int getPort() {
- return port;
- }
-
- public void setPort(int port) {
- this.port = port;
- }
-
- public boolean isPortSet() {
- return this.port != -1;
- }
-
- public String getWorkingDir() {
- return workingDir;
- }
-
- public void setWorkingDir(String workingDir) {
- this.workingDir = workingDir;
- }
-
- public boolean isWorkingDirSet() {
- return this.workingDir != null;
- }
-
- public String getMetaQuorum() {
- return metaQuorum;
- }
-
- public void setMetaQuorum(String metaQuorum) {
- this.metaQuorum = metaQuorum;
- }
-
- public boolean isMetaQuorumSet() {
- return this.metaQuorum != null;
- }
-
- public UUID getMetaGroupId() {
- return metaGroupId;
- }
-
- public void setMetaGroupId(UUID metaGroupId) {
- this.metaGroupId = metaGroupId;
- }
-
- public boolean isMetaServerGroupIdSet() {
- return this.metaGroupId != null && !this.metaGroupId.equals(Constants.META_GROUP_UUID);
- }
-
- public UUID getLogServerGroupId() {
- return logServerGroupId;
- }
-
- public void setLogServerGroupId(UUID logServerGroupId) {
- this.logServerGroupId = logServerGroupId;
- }
-
- public boolean isLogServerGroupIdSet() {
- return this.logServerGroupId != null &&
- !this.logServerGroupId.equals(Constants.SERVERS_GROUP_UUID);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("Hostname=").append(host);
- sb.append(", port=").append(port);
- sb.append(", dir=").append(workingDir);
- sb.append(", metaQuorum=").append(metaQuorum);
- sb.append(", metaGroupId=").append(metaGroupId);
- sb.append(", logServerGroupId=").append(logServerGroupId);
- return sb.toString();
- }
-}
\ No newline at end of file
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/Command.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/Command.java
deleted file mode 100644
index 5b2fcb1..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/Command.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell;
-
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-/**
- * Represents a single command in the shell. Each command must be stateless.
- */
-public interface Command {
-
- String getHelpMessage();
-
- void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args);
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/CommandFactory.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/CommandFactory.java
deleted file mode 100644
index 660efa3..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/CommandFactory.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-
-import org.apache.ratis.logservice.shell.commands.ArchiveLogCommand;
-import org.apache.ratis.logservice.shell.commands.CreateLogCommand;
-import org.apache.ratis.logservice.shell.commands.DeleteLogCommand;
-import org.apache.ratis.logservice.shell.commands.ExitCommand;
-import org.apache.ratis.logservice.shell.commands.ExportLogCommand;
-import org.apache.ratis.logservice.shell.commands.HelpCommand;
-import org.apache.ratis.logservice.shell.commands.ListLogsCommand;
-import org.apache.ratis.logservice.shell.commands.PutToLogCommand;
-import org.apache.ratis.logservice.shell.commands.ReadLogCommand;
-
-public final class CommandFactory {
- private static final Map<String,Command> KNOWN_COMMANDS = cacheCommands();
-
- private static Map<String,Command> cacheCommands() {
- Map<String,Command> commands = new HashMap<>();
- ExitCommand exitCommand = new ExitCommand();
-
- // Ensure all keys are lowercase -- see the same logic in #create(String).
- commands.put("create", new CreateLogCommand());
- commands.put("put", new PutToLogCommand());
- commands.put("read", new ReadLogCommand());
- commands.put("delete", new DeleteLogCommand());
- commands.put("exit", exitCommand);
- commands.put("quit", exitCommand);
- commands.put("help", new HelpCommand());
- commands.put("list", new ListLogsCommand());
- commands.put("archive", new ArchiveLogCommand());
- commands.put("export", new ExportLogCommand());
-
- return Collections.unmodifiableMap(commands);
- }
-
- public static Map<String,Command> getCommands() {
- return KNOWN_COMMANDS;
- }
-
- private CommandFactory() {}
-
- /**
- * Returns an instance of the command to run given the name, else null.
- * Be aware that Command instances are singletons and do not retain state.
- */
- public static Command create(String commandName) {
- // Normalize the command name to lowercase
- return KNOWN_COMMANDS.get(Objects.requireNonNull(commandName).toLowerCase());
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/LogServiceShell.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/LogServiceShell.java
deleted file mode 100644
index f7124e9..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/LogServiceShell.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell;
-
-import java.io.IOException;
-import java.util.AbstractMap;
-import java.util.Arrays;
-import java.util.Map.Entry;
-import java.util.Objects;
-
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.commands.ExitCommand;
-import org.jline.reader.EndOfFileException;
-import org.jline.reader.History;
-import org.jline.reader.LineReader;
-import org.jline.reader.LineReaderBuilder;
-import org.jline.reader.UserInterruptException;
-import org.jline.reader.impl.DefaultHighlighter;
-import org.jline.reader.impl.history.DefaultHistory;
-import org.jline.terminal.Terminal;
-import org.jline.terminal.TerminalBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.JCommander;
-
-/**
- * An interactive shell that can interact with a LogService instance.
- */
-public class LogServiceShell {
- private static final Logger LOG = LoggerFactory.getLogger(LogServiceShell.class);
-
- private static final String PROMPT = "logservice> ";
-
- private final Terminal terminal;
- private final LineReader lineReader;
- private final LogServiceClient client;
-
- public LogServiceShell(Terminal terminal, LineReader reader, LogServiceClient client) {
- this.terminal = Objects.requireNonNull(terminal);
- this.lineReader = Objects.requireNonNull(reader);
- this.client = Objects.requireNonNull(client);
- }
-
- public void run() {
- while (true) {
- // Read and sanitize the input
- String line;
- try {
- line = lineReader.readLine(PROMPT);
- } catch (UserInterruptException e) {
- continue;
- } catch (EndOfFileException e) {
- break;
- }
- if (line == null) {
- continue;
- }
- line = line.trim();
-
- // Construct the Command and args to pass to that command
- Entry<Command,String[]> pair = parseCommand(line);
- if (pair == null) {
- continue;
- }
- Command command = pair.getKey();
- String[] commandArgs = pair.getValue();
-
- // Our "exit" or "quit"
- if (command instanceof ExitCommand) {
- break;
- }
-
- // Run the command
- command.run(terminal, lineReader, client, commandArgs);
-
- // Flush a newline to the screen.
- terminal.writer().flush();
- }
- terminal.writer().println("Bye!");
- terminal.writer().flush();
- }
-
- Entry<Command,String[]> parseCommand(String line) {
- String[] words = line.split("\\s+");
- if (words.length == 0) {
- return null;
- }
-
- String commandWord = words[0];
- Command command = CommandFactory.create(commandWord);
- // If we have no command to run, return null.
- if (command == null) {
- return null;
- }
-
- // Pull off the "command" name
- String[] args = Arrays.copyOfRange(words, 1, words.length);
-
- return new AbstractMap.SimpleEntry<>(command, args);
- }
-
- public static void main(String[] args) throws Exception {
- final Terminal terminal = TerminalBuilder.builder()
- .system(true)
- .build();
-
- History defaultHistory = new DefaultHistory();
- // Register a shutdown-hook per JLine documentation to save history
- Runtime.getRuntime().addShutdownHook(new Thread(() -> {
- try {
- defaultHistory.save();
- } catch (IOException e) {
- LOG.debug("Failed to save terminal history", e);
- }
- }));
-
- final LineReader lineReader = LineReaderBuilder.builder()
- .terminal(terminal)
- .highlighter(new DefaultHighlighter())
- .history(defaultHistory)
- .build();
-
- LogServiceShellOpts opts = new LogServiceShellOpts();
- JCommander.newBuilder()
- .addObject(opts)
- .build()
- .parse(args);
-
- try (LogServiceClient logServiceClient = new LogServiceClient(opts.getMetaQuorum())) {
- LogServiceShell client = new LogServiceShell(terminal, lineReader, logServiceClient);
- client.run();
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/LogServiceShellOpts.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/LogServiceShellOpts.java
deleted file mode 100644
index 8c77f18..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/LogServiceShellOpts.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell;
-
-import com.beust.jcommander.Parameter;
-
-public class LogServiceShellOpts {
- @Parameter(names = {"--meta-quorum", "-q"}, required = true)
- private String metaQuorum;
-
- public String getMetaQuorum() {
- return metaQuorum;
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ArchiveLogCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ArchiveLogCommand.java
deleted file mode 100644
index c4f9f98..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ArchiveLogCommand.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class ArchiveLogCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`archive` - archive the given log at already configured location";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- if (args.length != 1) {
- terminal.writer().println("ERROR - Usage: archive <name> ");
- return;
- }
- String logName = args[0];
- try {
- client.archiveLog(LogName.of(logName));
- terminal.writer().println("Archive Log request is submitted successfully!!");
- } catch (Exception e) {
- terminal.writer().println("Failed to archive log!!");
- e.printStackTrace(terminal.writer());
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/CreateLogCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/CreateLogCommand.java
deleted file mode 100644
index 31fae65..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/CreateLogCommand.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class CreateLogCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`create` - Creates a new log with the given name";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- if (args.length != 1) {
- terminal.writer().println("ERROR - Usage: create <name>");
- return;
- }
- String logName = args[0];
- try (LogStream stream = client.createLog(LogName.of(logName))) {
- terminal.writer().println("Created log '" + logName + "'");
- } catch (Exception e) {
- terminal.writer().println("Error creating log '" + logName + "'");
- e.printStackTrace(terminal.writer());
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/DeleteLogCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/DeleteLogCommand.java
deleted file mode 100644
index 96081f9..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/DeleteLogCommand.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import java.io.IOException;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class DeleteLogCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`delete` - deletes the log with the given name.";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- if (args.length != 1) {
- terminal.writer().println("ERROR - Usage: delete <name>");
- return;
- }
- String logName = args[0];
- try {
- client.deleteLog(LogName.of(logName));
- terminal.writer().println("Deleted log '" + logName + "'");
- } catch (IOException e) {
- terminal.writer().println("Error deleting log '" + logName + "'");
- e.printStackTrace(terminal.writer());
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ExitCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ExitCommand.java
deleted file mode 100644
index 11a30d1..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ExitCommand.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class ExitCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`exit` -- Exits this shell.";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- throw new IllegalStateException("This command should be be invoked");
- }
-
-}
-
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ExportLogCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ExportLogCommand.java
deleted file mode 100644
index 02b0a6f..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ExportLogCommand.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class ExportLogCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`export` - export the given log at given location starting from provided recordId";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- if (args.length != 3) {
- terminal.writer().println("ERROR - Usage: export <name> <location> <recordId>");
- return;
- }
- String logName = args[0];
- String location = args[1];
- long recordId = Long.parseLong(args[2]);
- try {
- client.exportLog(LogName.of(logName), location,recordId);
- terminal.writer().println("Export Log request is submitted successfully!!");
- } catch (Exception e) {
- terminal.writer().println("Failed to export log!!");
- e.printStackTrace(terminal.writer());
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/HelpCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/HelpCommand.java
deleted file mode 100644
index c139f59..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/HelpCommand.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.apache.ratis.logservice.shell.CommandFactory;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class HelpCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`help` - Prints this help message";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- CommandFactory.getCommands()
- .values()
- .stream()
- .forEach((c) -> terminal.writer().println(c.getHelpMessage()));
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ListLogsCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ListLogsCommand.java
deleted file mode 100644
index 8b8e03b..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ListLogsCommand.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.ratis.logservice.api.LogInfo;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class ListLogsCommand implements Command {
-
- @Override
- public String getHelpMessage() {
- return "`list` - List the available logs";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- if (args.length != 0) {
- terminal.writer().println("ERROR - Usage: list");
- return;
- }
-
- try {
- List<LogInfo> logs = client.listLogs();
- StringBuilder sb = new StringBuilder();
- for (LogInfo log : logs) {
- if (sb.length() > 0) {
- sb.append("\n");
- }
- sb.append(log.getLogName().getName());
- }
- terminal.writer().println(sb.toString());
- } catch (IOException e) {
- terminal.writer().println("Failed to list available logs");
- e.printStackTrace(terminal.writer());
- }
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/PutToLogCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/PutToLogCommand.java
deleted file mode 100644
index 8f0a617..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/PutToLogCommand.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogWriter;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class PutToLogCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`put` - Appends a value to a log";
- }
-
- @Override
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- if (args.length != 2) {
- terminal.writer().println("ERROR - Usage: put <name> <value>");
- return;
- }
- String name = args[0];
- String value = args[1];
- try (LogStream stream = client.getLog(LogName.of(name));
- LogWriter writer = stream.createWriter()) {
- writer.write(ByteBuffer.wrap(value.getBytes(StandardCharsets.UTF_8)));
- } catch (Exception e) {
- terminal.writer().println("Error writing to log");
- e.printStackTrace(terminal.writer());
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ReadLogCommand.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ReadLogCommand.java
deleted file mode 100644
index 8819806..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/shell/commands/ReadLogCommand.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.shell.commands;
-
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogReader;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.shell.Command;
-import org.jline.reader.LineReader;
-import org.jline.terminal.Terminal;
-
-public class ReadLogCommand implements Command {
-
- @Override public String getHelpMessage() {
- return "`read` - Reads all values from the given log";
- }
-
- @Override
- @SuppressFBWarnings("REC_CATCH_EXCEPTION")
- public void run(Terminal terminal, LineReader lineReader, LogServiceClient client, String[] args) {
- if (args.length != 1) {
- terminal.writer().println("ERROR - Usage: read <name>");
- return;
- }
- String logName = args[0];
- try (LogStream stream = client.getLog(LogName.of(logName));
- LogReader reader = stream.createReader()) {
- long firstId = stream.getStartRecordId();
- long lastId = stream.getLastRecordId();
- StringBuilder sb = new StringBuilder();
- List<ByteBuffer> records = reader.readBulk((int) (lastId - firstId));
- for (ByteBuffer record : records) {
- if (sb.length() > 0) {
- sb.append(", ");
- }
- sb.append("\"");
- if (record != null) {
- String strData = new String(record.array(), record.arrayOffset(), record.remaining(), StandardCharsets.UTF_8);
- sb.append(strData);
- }
- sb.append("\"");
- }
- sb.insert(0, "[");
- sb.append("]");
- terminal.writer().println(sb.toString());
- } catch (Exception e) {
- terminal.writer().println("Failed to read from log");
- e.printStackTrace(terminal.writer());
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/tool/VerificationTool.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/tool/VerificationTool.java
deleted file mode 100644
index 2c3d3f2..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/tool/VerificationTool.java
+++ /dev/null
@@ -1,420 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.tool;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import org.apache.ratis.logservice.api.LogInfo;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogReader;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogWriter;
-import org.apache.ratis.logservice.api.LogServiceClient;
-import org.apache.ratis.logservice.common.LogNotFoundException;
-import org.apache.ratis.logservice.server.LogStateMachine;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.beust.jcommander.IParameterValidator;
-import com.beust.jcommander.JCommander;
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.ParameterException;
-import com.beust.jcommander.validators.PositiveInteger;
-
-/**
- * LogService utility to write data and validate its presence.
- */
-public class VerificationTool {
- /**
- * Validator to check that the provided value is positive and non-zero.
- */
- public static class NonZeroPositiveInteger implements IParameterValidator {
- @Override
- public void validate(String name, String value) throws ParameterException {
- int i = Integer.parseInt(value);
- if (i < 1) {
- throw new ParameterException("Parameter '" + name + "' must be positive and non-zero"
- + ", but was " + value);
- }
- }
- }
-
- public static final Logger LOG = LoggerFactory.getLogger(LogStateMachine.class);
-
- @Parameter(names = {"-q", "--metaQuorum"},
- description = "Metadata Service Quorum",
- required = true)
- private String metaQuorum;
-
- @Parameter(names = {"-nl", "--numLogs"},
- description = "Number of logs",
- validateWith = NonZeroPositiveInteger.class)
- private int numLogs = 10;
-
- @Parameter(names = {"-nr", "--numRecords"},
- description = "Number of records to write per log",
- validateWith = NonZeroPositiveInteger.class)
- private int numRecords = 1000;
-
- @Parameter(names = {"-w", "--write"},
- description = "Write to the logs",
- arity = 1)
- private boolean write = true;
-
- @Parameter(names = {"-r", "--read"},
- description = "Read the logs",
- arity = 1)
- private boolean read = true;
-
- @Parameter(names = {"-l", "--logFrequency"},
- description = "Print update every N operations",
- validateWith = NonZeroPositiveInteger.class)
- private int logFrequency = 50;
-
- @Parameter(names = {"-h", "--help"},
- description = "Help",
- help = true)
- private boolean help = false;
-
- @Parameter(names = {"-s", "--size"},
- description = "Size in bytes of each value")
- private int recordSize = -1;
-
- @Parameter(names = {"-bs", "--batchSize"},
- description = "Number of records in a batch, a value of 0 disables batching",
- validateWith = PositiveInteger.class)
- private int batchSize = 0;
-
- public static final String LOG_NAME_PREFIX = "testlog";
- public static final String MESSAGE_PREFIX = "message";
-
- public static void main(String[] args) throws Exception {
- VerificationTool tool = new VerificationTool();
- JCommander jc = JCommander.newBuilder()
- .addObject(tool)
- .build();
- jc.parse(args);
- if (tool.help) {
- jc.usage();
- return;
- }
- System.out.println(tool.metaQuorum);
- try (LogServiceClient client = new LogServiceClient(tool.metaQuorum)) {
- ExecutorService executor = Executors.newCachedThreadPool();
- List<Future<?>> futures = new ArrayList<Future<?>>(tool.numLogs);
-
- if (tool.write) {
- LOG.info("Executing parallel writes");
- // Delete any logs that already exist first
- final Set<LogName> logsInSystem = new HashSet<>();
- List<LogInfo> listOfLogs = client.listLogs();
- for (LogInfo logInfo : listOfLogs) {
- logsInSystem.add(logInfo.getLogName());
- }
-
- LOG.info("Observed logs already in system: {}", logsInSystem);
- for (int i = 0; i < tool.numLogs; i++) {
- LogName logName = getLogName(i);
- if (logsInSystem.contains(logName)) {
- LOG.info("Deleting {}", logName);
- client.deleteLog(logName);
- }
- }
-
- // First write batch entries to log.
- if(tool.batchSize > 0) {
- // Compute the number of batches to write given the batch size.
- int numBatches = tool.numRecords / tool.batchSize;
- for (int i = 0; i < tool.numLogs; i++) {
- BatchWriter writer = new BatchWriter(getLogName(i), client, tool.numRecords,
- tool.logFrequency, tool.recordSize, tool.batchSize, numBatches);
- futures.add(executor.submit(writer));
- }
- } else {
- // Write single entries to log.
- for (int i = 0; i < tool.numLogs; i++) {
- BulkWriter writer = new BulkWriter(getLogName(i), client, tool.numRecords,
- tool.logFrequency, tool.recordSize);
- futures.add(executor.submit(writer));
- }
- }
- waitForCompletion(futures);
- }
-
- if (tool.read) {
- LOG.info("Executing parallel reads");
- futures = new ArrayList<Future<?>>(tool.numLogs);
- for (int i = 0; i < tool.numLogs; i++) {
- BulkReader reader = new BulkReader(getLogName(i), client, tool.numRecords, tool.logFrequency,
- tool.recordSize);
- futures.add(executor.submit(reader));
- }
- waitForCompletion(futures);
- }
- executor.shutdownNow();
- }
-
- }
-
- private static LogName getLogName(int id) {
- return LogName.of(LOG_NAME_PREFIX + id);
- }
-
- @SuppressFBWarnings("DM_EXIT")
- private static void waitForCompletion(List<Future<?>> futures) {
- for (Future<?> future : futures) {
- try {
- Object object = future.get();
- if (object != null) {
- LOG.error("Operation failed with error ", object);
- System.exit(-1);
- }
- } catch (Exception e) {
- LOG.error("Got exception ", e);
- System.exit(-1);
- }
- }
- LOG.info("All operations finished");
- }
-
- abstract static class Operation implements Runnable {
- static final byte DIVIDER_BYTE = '_';
- private final LogName logName;
- private final LogServiceClient client;
- private final int numRecords;
- private final int logFreq;
- private final int valueSize;
-
- public LogName getLogName() {
- return logName;
- }
-
- public int getNumRecords() {
- return numRecords;
- }
-
- public int getLogFreq() {
- return logFreq;
- }
-
- public LogServiceClient getClient() {
- return client;
- }
-
- Operation(LogName logName, LogServiceClient client, int numRecords, int logFreq, int valueSize) {
- this.logName = logName;
- this.client = client;
- this.numRecords = numRecords;
- this.logFreq = logFreq;
- this.valueSize = valueSize;
- }
-
- ByteBuffer createValue(String prefix) {
- if (valueSize == -1) {
- return ByteBuffer.wrap(prefix.getBytes(StandardCharsets.UTF_8));
- }
- byte[] value = new byte[valueSize];
- byte[] prefixBytes = prefix.getBytes(StandardCharsets.UTF_8);
- // Write as much of the prefix as possible
- if (prefixBytes.length > valueSize) {
- System.arraycopy(prefixBytes, 0, value, 0, valueSize);
- return ByteBuffer.wrap(value);
- }
-
- // Write the full prefix
- System.arraycopy(prefixBytes, 0, value, 0, prefixBytes.length);
- int bytesWritten = prefixBytes.length;
-
- // Write the divider (but only if we can write the whole thing)
- if (bytesWritten + 1 > valueSize) {
- return ByteBuffer.wrap(value);
- }
-
- value[bytesWritten] = DIVIDER_BYTE;
- bytesWritten += 1;
-
- // Generate random data to pad the value
- int bytesToGenerate = valueSize - bytesWritten;
- Random r = new Random();
- byte[] suffix = new byte[bytesToGenerate];
- r.nextBytes(suffix);
- System.arraycopy(suffix, 0, value, bytesWritten, suffix.length);
-
- return ByteBuffer.wrap(value);
- }
-
- String parseValue(ByteBuffer buff) {
- if (!buff.hasArray()) {
- throw new IllegalArgumentException("Require a ByteBuffer with a backing array");
- }
- if (valueSize == -1) {
- return new String(buff.array(), buff.arrayOffset(), buff.remaining(), StandardCharsets.UTF_8);
- }
- int length = buff.limit() - buff.arrayOffset();
- byte[] value = new byte[length];
- System.arraycopy(buff.array(), buff.arrayOffset(), value, 0, length);
-
- int dividerOffset = -1;
- for (int i = 0; i < value.length; i++) {
- if (value[i] == DIVIDER_BYTE) {
- dividerOffset = i;
- break;
- }
- }
- // We didn't have enough space to write the divider, return all of the bytes
- if (dividerOffset < 0) {
- return new String(value, StandardCharsets.UTF_8);
- }
-
- return new String(value, 0, dividerOffset, StandardCharsets.UTF_8);
- }
-
- LogWriter getLogWriter() throws IOException {
- LogStream logStream = null;
- try {
- logStream = this.client.getLog(logName);
- } catch (LogNotFoundException e) {
- LOG.info("Creating {}", logName);
- logStream = this.client.createLog(logName);
- }
- return logStream.createWriter();
- }
- }
-
- static class BulkWriter extends Operation {
- BulkWriter(LogName logName, LogServiceClient client, int numRecords, int logFreq,
- int valueSize) {
- super(logName, client, numRecords, logFreq, valueSize);
- }
-
- public void run() {
- try {
- LogWriter writer = getLogWriter();
- for (int i = 0; i < getNumRecords(); i++) {
- String message = MESSAGE_PREFIX + i;
- if (i % getLogFreq() == 0) {
- LOG.info(getLogName() + " Writing " + message);
- }
- writer.write(createValue(message));
- }
- writer.close();
- LOG.info("{} entries written to {} successfully.", getNumRecords(), getLogName());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class BatchWriter extends Operation {
- private int batchSize;
- private int numBatches;
- BatchWriter(LogName logName, LogServiceClient client, int numRecords, int logFreq,
- int valueSize, int batchSize, int numBatches) {
- super(logName, client, numRecords, logFreq, valueSize);
- this.batchSize = batchSize;
- this.numBatches = numBatches;
- }
-
- public void run() {
- try {
- LogWriter writer = getLogWriter();
- for(int i = 0; i < numBatches; i++) {
- List<ByteBuffer> messages = new ArrayList<ByteBuffer>(batchSize);
- for(int j = 0; j < batchSize; j++) {
- String message = MESSAGE_PREFIX + (i * batchSize + j);
- messages.add(createValue(message));
- if((i * batchSize + j) % getLogFreq() == 0) {
- LOG.info(getLogName() + " batching write " + message);
- }
- }
- try {
- writer.write(messages);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- // Catch the last bit that didn't evenly fit into the batch sizes
- if (getNumRecords() % batchSize != 0) {
- List<ByteBuffer> lastBatch = new ArrayList<>();
- for (int i = numBatches * batchSize; i < getNumRecords(); i++) {
- String message = MESSAGE_PREFIX + i;
- lastBatch.add(createValue(message));
- }
- LOG.info(getLogName() + " writing last mini-batch of " + lastBatch.size() + " records");
- try {
- writer.write(lastBatch);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- LOG.info("{} entries written in batches to {} successfully.",
- getNumRecords(), getLogName());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- static class BulkReader extends Operation {
- BulkReader(LogName logName, LogServiceClient client, int numRecords, int logFreq, int valueSize) {
- super(logName, client, numRecords, logFreq, valueSize);
- }
-
- @SuppressFBWarnings("DM_EXIT")
- public void run() {
- try {
- LogStream logStream = getClient().getLog(getLogName());
- LogReader reader = logStream.createReader();
- long size = logStream.getLength();
- if(size != getNumRecords()) {
- LOG.error("There is mismatch is number of records. Expected Records: "+
- getNumRecords() +", Actual Records: " + size);
- System.exit(-1);
- }
- for (int i = 0; i < size; i++) {
- ByteBuffer buffer = reader.readNext();
- String message = parseValue(buffer);
- if (i % getLogFreq() == 0) {
- LOG.info(getLogName() + " Read " + message);
- }
- if(!message.equals(MESSAGE_PREFIX + i)) {
- LOG.error("Message is not correct. Expected: "+(MESSAGE_PREFIX + i)
- +". Actual:" +message);
- System.exit(-1);
- }
- }
- LOG.info("{} log entries read from log {} successfully.", getNumRecords(), getLogName());
- reader.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/LogServiceProtoUtil.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/LogServiceProtoUtil.java
deleted file mode 100644
index a743e89..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/LogServiceProtoUtil.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.util;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.api.LogStream;
-import org.apache.ratis.logservice.api.LogStream.State;
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.logservice.proto.LogServiceProtos.*;
-import org.apache.ratis.logservice.server.ArchivalInfo;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-
-public class LogServiceProtoUtil {
-
- public static LogNameProto toLogNameProto(LogName logName) {
- return LogNameProto.newBuilder().setName(logName.getName()).build();
- }
-
- public static LogName toLogName(LogServiceProtos.LogNameProto logNameProto) {
- return LogName.of(logNameProto.getName());
- }
-
- public static LogStreamProto toLogStreamProto(LogStream logStream) throws IOException {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(logStream.getName().getName()).build();
- LogStreamProto logStreamProto =
- LogStreamProto
- .newBuilder()
- .setLogName(logNameProto)
- .setSize(logStream.getSize())
- .setState(
- logStream.getState().equals(State.OPEN) ? LogStreamState.OPEN : LogStreamState.CLOSED)
- .build();
- return logStreamProto;
- }
-
- public static LogServiceRequestProto toChangeStateRequestProto(LogName logName, State state,
- boolean force) {
- LogNameProto logNameProto = LogNameProto.newBuilder().setName(logName.getName()).build();
- ChangeStateLogRequestProto changeLog =
- ChangeStateLogRequestProto.newBuilder().setLogName(logNameProto)
- .setState(LogStreamState.valueOf(state.name())).setForce(force).build();
- return LogServiceRequestProto.newBuilder().setChangeState(changeLog).build();
- }
-
- public static LogServiceRequestProto toChangeStateRequestProto(LogName logName, State state) {
- return toChangeStateRequestProto(logName, state, false);
- }
-
- public static LogServiceRequestProto toGetStateRequestProto(LogName logName) {
- LogNameProto logNameProto = LogNameProto.newBuilder().setName(logName.getName()).build();
- GetStateRequestProto getState =
- GetStateRequestProto.newBuilder().setLogName(logNameProto).build();
- return LogServiceRequestProto.newBuilder().setGetState(getState).build();
- }
-
- public static ArchiveLogReplyProto toArchiveLogReplyProto(Throwable t) {
- ArchiveLogReplyProto.Builder builder = ArchiveLogReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- }
- return builder.build();
- }
-
- public static LogServiceRequestProto toGetSizeRequestProto(LogName name) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- GetLogSizeRequestProto getLogSize = GetLogSizeRequestProto.newBuilder()
- .setLogName(logNameProto).build();
- return LogServiceRequestProto.newBuilder().setSizeRequest(getLogSize).build();
- }
-
- public static LogServiceRequestProto toGetLengthRequestProto(LogName name) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- GetLogLengthRequestProto.Builder builder = GetLogLengthRequestProto.newBuilder();
- builder.setLogName(logNameProto);
- return LogServiceRequestProto.newBuilder().setLengthQuery(builder.build()).build();
- }
-
- public static LogServiceRequestProto toGetLastCommittedIndexRequestProto(LogName name) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- GetLogLastCommittedIndexRequestProto.Builder builder =
- GetLogLastCommittedIndexRequestProto.newBuilder();
- builder.setLogName(logNameProto);
- return LogServiceRequestProto.newBuilder().setLastIndexQuery(builder.build()).build();
- }
-
- public static LogServiceRequestProto toGetStartIndexProto(LogName name) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- GetLogStartIndexRequestProto.Builder builder = GetLogStartIndexRequestProto.newBuilder();
- builder.setLogName(logNameProto);
- return LogServiceRequestProto.newBuilder().setStartIndexQuery(builder.build()).build();
- }
-
- public static LogServiceRequestProto toReadLogRequestProto(LogName name, long start, int total) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- ReadLogRequestProto.Builder builder = ReadLogRequestProto.newBuilder();
- builder.setLogName(logNameProto);
- builder.setStartRecordId(start);
- builder.setNumRecords(total);
- return LogServiceRequestProto.newBuilder().setReadNextQuery(builder.build()).build();
- }
-
- public static LogServiceRequestProto toSyncLogRequestProto(LogName name) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- SyncLogRequestProto.Builder builder = SyncLogRequestProto.newBuilder();
- builder.setLogName(logNameProto);
- return LogServiceRequestProto.newBuilder().setSyncRequest(builder.build()).build();
- }
-
- public static LogServiceRequestProto toAppendEntryLogRequestProto(LogName name,
- List<byte[]> entries) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- AppendLogEntryRequestProto.Builder builder = AppendLogEntryRequestProto.newBuilder();
- builder.setLogName(logNameProto);
- for (int i=0; i < entries.size(); i++) {
- builder.addData(ByteString.copyFrom(entries.get(i)));
- }
- return LogServiceRequestProto.newBuilder().setAppendRequest(builder.build()).build();
- }
-
- public static LogServiceRequestProto toAppendBBEntryLogRequestProto(LogName name,
- List<ByteBuffer> entries) {
- LogNameProto logNameProto =
- LogNameProto.newBuilder().setName(name.getName()).build();
- AppendLogEntryRequestProto.Builder builder = AppendLogEntryRequestProto.newBuilder();
- builder.setLogName(logNameProto);
- for (int i=0; i < entries.size(); i++) {
- ByteBuffer currentBuf = entries.get(i);
- // Save the current position
- int pos = currentBuf.position();
- builder.addData(ByteString.copyFrom(currentBuf));
- // Reset it after we're done reading the bytes
- currentBuf.position(pos);
- }
- return LogServiceRequestProto.newBuilder().setAppendRequest(builder.build()).build();
- }
-
- public static List<byte[]> toListByteArray(List<ByteString> list) {
- List<byte[]> retVal = new ArrayList<byte[]>(list.size());
- for(int i=0; i < list.size(); i++) {
- retVal.add(list.get(i).toByteArray());
- }
- return retVal;
- }
-
- public static GetStateReplyProto toGetStateReplyProto(State state) {
- return GetStateReplyProto.newBuilder().setState(LogStreamState.valueOf(state.name())).build();
- }
-
- public static GetLogLengthReplyProto toGetLogLengthReplyProto(long length, Throwable t) {
- GetLogLengthReplyProto.Builder builder = GetLogLengthReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- } else {
- builder.setLength(length);
- }
- return builder.build();
- }
-
- public static GetLogSizeReplyProto toGetLogSizeReplyProto(long size, Throwable t) {
- GetLogSizeReplyProto.Builder builder = GetLogSizeReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- } else {
- builder.setSize(size);
- }
- return builder.build();
- }
-
- public static GetLogStartIndexReplyProto toGetLogStartIndexReplyProto(long length, Throwable t) {
- GetLogStartIndexReplyProto.Builder builder = GetLogStartIndexReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- } else {
- builder.setStartIndex(length);
- }
- return builder.build();
- }
-
- public static GetLogLastCommittedIndexReplyProto
- toGetLogLastIndexReplyProto(long lastIndex, Throwable t) {
-
- GetLogLastCommittedIndexReplyProto.Builder builder =
- GetLogLastCommittedIndexReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- } else {
- builder.setLastCommittedIndex(lastIndex);
- }
- return builder.build();
- }
-
- public static ReadLogReplyProto toReadLogReplyProto(List<byte[]> entries, Throwable t) {
- ReadLogReplyProto.Builder builder = ReadLogReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- } else {
- for(byte[] record: entries) {
- builder.addLogRecord( ByteString.copyFrom(record));
- }
- }
- return builder.build();
- }
-
- public static AppendLogEntryReplyProto toAppendLogReplyProto(List<Long> ids, Throwable t) {
- AppendLogEntryReplyProto.Builder builder = AppendLogEntryReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- } else if (ids != null) {
- for (long id: ids) {
- builder.addRecordId(id);
- }
- }
- return builder.build();
- }
-
- public static SyncLogReplyProto toSyncLogReplyProto(long index, Throwable t) {
- SyncLogReplyProto.Builder builder = SyncLogReplyProto.newBuilder();
- if (t != null) {
- builder.setException(toLogException(t));
- } else {
- builder.setLastRecordId(index);
- }
- return builder.build();
- }
-
- public static ArchiveLogRequestProto toExportInfoProto(ArchivalInfo info) {
- return ArchiveLogRequestProto.newBuilder().setIsExport(true)
- .setLastArchivedRaftIndex(info.getLastArchivedIndex())
- .setLocation(info.getArchiveLocation()).setLogName(
- LogServiceProtos.LogNameProto.newBuilder().setName(info.getArchiveLogName().getName())
- .build()).setStatus(ArchivalStatus.valueOf(info.getStatus().name())).build();
- }
-
- public static ArchivalInfo toExportInfo(ArchiveLogRequestProto proto){
- return new ArchivalInfo(proto.getLocation()).updateArchivalInfo(proto);
-
- }
-
- public GetLogLengthReplyProto toGetLogLengthReplyProto(long length) {
- GetLogLengthReplyProto.Builder builder = GetLogLengthReplyProto.newBuilder();
- builder.setLength(length);
- return builder.build();
- }
-
- public static LogServiceException toLogException(Throwable t) {
- LogServiceException.Builder builder = LogServiceException.newBuilder();
- builder.setExceptionClassName(t.getClass().getName());
- builder.setErrorMsg(t.getMessage());
- StackTraceElement[] trace = t.getStackTrace();
- StringBuffer buf = new StringBuffer();
- for (StackTraceElement el: trace) {
- buf.append(el.toString()).append("\n");
- }
- String strace = buf.toString();
- builder.setStacktrace(ByteString.copyFrom(strace, Charset.defaultCharset()));
- return builder.build();
- }
-
- public static LogServiceRequestProto toExportInfoRequestProto(LogName logName){
- LogServiceProtos.LogNameProto logNameProto =
- LogServiceProtos.LogNameProto.newBuilder().setName(logName.getName()).build();
- GetExportInfoRequestProto exportInfoRequestProto =
- GetExportInfoRequestProto.newBuilder().setLogName(logNameProto).build();
- return LogServiceRequestProto.newBuilder().setExportInfo(exportInfoRequestProto).build();
- }
-
- public static LogServiceRequestProto toArchiveLogRequestProto(LogName logName, String location,
- long raftIndex, boolean isArchival, ArchivalInfo.ArchivalStatus status) {
- LogServiceProtos.LogNameProto logNameProto =
- LogServiceProtos.LogNameProto.newBuilder().setName(logName.getName()).build();
- ArchiveLogRequestProto.Builder builder =
- ArchiveLogRequestProto.newBuilder().setLogName(logNameProto)
- .setLastArchivedRaftIndex(raftIndex).setStatus(ArchivalStatus.valueOf(status.name()));
- builder.setIsExport(!isArchival);
- if (location != null) {
- builder.setLocation(location);
- }
- ArchiveLogRequestProto archiveLog = builder.build();
- return LogServiceRequestProto.newBuilder().setArchiveLog(archiveLog).build();
- }
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/LogServiceUtils.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/LogServiceUtils.java
deleted file mode 100644
index 05e585c..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/LogServiceUtils.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.logservice.util;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.protocol.RaftPeer;
-
-import java.io.IOException;
-import java.net.DatagramSocket;
-import java.net.InetAddress;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public final class LogServiceUtils {
- private LogServiceUtils() {
- }
-
- public static Set<RaftPeer> getPeersFromIds(String identity) {
-
- return Stream.of(identity.split(",")).map(elem ->
- RaftPeer.newBuilder().setId(elem).setAddress(elem.replace('_', ':')).build()
- ).collect(Collectors.toSet());
- }
-
- public static Set<RaftPeer> getPeersFromQuorum(String identity) {
- return Stream.of(identity.split(",")).map(elem ->
- RaftPeer.newBuilder().setId(elem.replace(':', '_')).setAddress(elem).build()
- ).collect(Collectors.toSet());
- }
-
- public static String getHostName() {
- try (DatagramSocket socket = new DatagramSocket()) {
- socket.connect(InetAddress.getByName("8.8.8.8"), 10002);
- return socket.getLocalAddress().getHostName();
- } catch (IOException e) {
- return "localhost";
- }
-
- }
-
- public static String getArchiveLocationForLog(String location, LogName logName) {
- return location + "/" + logName.getName();
- }
-
- public static String getRolledPathForArchiveWriter(Path path, long lastWrittenId) {
- return path + "_recordId_" + lastWrittenId;
- }
-
- public static Integer getRecordIdFromRolledArchiveFile(Path path) {
- String[] splits = path.getName().split("_recordId_");
- if (splits.length != 2) {
- //currently written file, should be read last
- return Integer.MAX_VALUE;
- }
- return Integer.parseInt(splits[1]);
- }
-
-}
diff --git a/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/MetaServiceProtoUtil.java b/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/MetaServiceProtoUtil.java
deleted file mode 100644
index 4dd09a6..0000000
--- a/ratis-logservice/src/main/java/org/apache/ratis/logservice/util/MetaServiceProtoUtil.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis.logservice.util;
-
-
-import org.apache.ratis.logservice.api.LogInfo;
-import org.apache.ratis.logservice.api.LogName;
-import org.apache.ratis.logservice.proto.LogServiceProtos;
-import org.apache.ratis.logservice.proto.MetaServiceProtos;
-import org.apache.ratis.logservice.proto.MetaServiceProtos.*;
-import org.apache.ratis.protocol.*;
-import org.apache.ratis.util.ProtoUtils;
-import org.apache.ratis.util.ReflectionUtils;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.apache.ratis.logservice.util.LogServiceProtoUtil.toLogNameProto;
-
-public final class MetaServiceProtoUtil {
- private MetaServiceProtoUtil() {
- }
-
- public static RaftPeerProto toRaftPeerProto(RaftPeer peer) {
- RaftPeerProto.Builder builder = RaftPeerProto.newBuilder()
- .setId(peer.getId().toByteString());
- if (peer.getAddress() != null) {
- builder.setAddress(peer.getAddress());
- }
- return builder.build();
- }
-
- public static RaftPeer toRaftPeer(RaftPeerProto p) {
- return RaftPeer.newBuilder().setId(p.getId()).setAddress(p.getAddress()).build();
- }
-
- public static RaftGroup toRaftGroup(RaftGroupProto proto) {
- return RaftGroup.valueOf(RaftGroupId.valueOf(proto.getId()), toRaftPeerArray(proto.getPeersList()));
- }
-
- public static RaftGroupProto toRaftGroupProto(RaftGroup group) {
- return RaftGroupProto.newBuilder()
- .setId(group.getGroupId().toByteString())
- .addAllPeers(toRaftPeerProtos(group.getPeers())).build();
- }
-
- public static LogInfo toLogInfo(LogInfoProto logProto) {
- return new LogInfo(LogServiceProtoUtil.toLogName(logProto.getLogname()), toRaftGroup(logProto.getRaftGroup()));
- }
-
- public static LogInfoProto toLogInfoProto (LogInfo logInfo) {
- return LogInfoProto.newBuilder()
- .setLogname(toLogNameProto(logInfo.getLogName()))
- .setRaftGroup(toRaftGroupProto(logInfo.getRaftGroup()))
- .build();
- }
-
- public static MetaSMRequestProto toPingRequestProto(RaftPeer peer) {
- return MetaServiceProtos.MetaSMRequestProto
- .newBuilder()
- .setPingRequest(
- MetaServiceProtos.LogServicePingRequestProto
- .newBuilder()
- .setPeer(MetaServiceProtoUtil.toRaftPeerProto(peer)).build()).build();
- }
-
- public static MetaSMRequestProto toHeartbeatRequestProto(RaftPeer peer) {
- return MetaServiceProtos.MetaSMRequestProto
- .newBuilder()
- .setHeartbeatRequest(
- MetaServiceProtos.LogServiceHeartbeatRequestProto
- .newBuilder()
- .setPeer(MetaServiceProtoUtil.toRaftPeerProto(peer)).build()).build();
- }
-
- public static MetaServiceRequestProto toCreateLogRequestProto(LogName logName) {
- LogServiceProtos.LogNameProto logNameProto = LogServiceProtos.LogNameProto.newBuilder()
- .setName(logName.getName())
- .build();
- CreateLogRequestProto createLog =
- CreateLogRequestProto.newBuilder().setLogName(logNameProto).build();
- return MetaServiceRequestProto.newBuilder().setCreateLog(createLog).build();
- }
-
- public static MetaServiceRequestProto toListLogRequestProto() {
- ListLogsRequestProto listLogs = ListLogsRequestProto.newBuilder().build();
- return MetaServiceRequestProto.newBuilder().setListLogs(listLogs).build();
- }
-
- public static MetaServiceRequestProto toGetLogRequestProto(LogName name) {
- GetLogRequestProto getLog =
- GetLogRequestProto.newBuilder().setLogName(toLogNameProto(name)).build();
- return MetaServiceRequestProto.newBuilder().setGetLog(getLog).build();
- }
-
- public static MetaServiceRequestProto toDeleteLogRequestProto(LogName logName) {
- LogServiceProtos.LogNameProto logNameProto = LogServiceProtos.LogNameProto.newBuilder()
- .setName(logName.getName())
- .build();
- DeleteLogRequestProto deleteLog =
- DeleteLogRequestProto.newBuilder().setLogName(logNameProto).build();
- return MetaServiceRequestProto.newBuilder().setDeleteLog(deleteLog).build();
- }
-
- public static CreateLogReplyProto.Builder toCreateLogReplyProto(LogInfo logInfo) {
- return CreateLogReplyProto.newBuilder().setLog(toLogInfoProto(logInfo));
- }
-
- public static ListLogsReplyProto toListLogLogsReplyProto(List<LogInfo> logInfos) {
- return ListLogsReplyProto.newBuilder().addAllLogs(
- logInfos.stream()
- .map(log -> toLogInfoProto(log))
- .collect(Collectors.toList())).build();
- }
-
- public static GetLogReplyProto toGetLogReplyProto(LogInfo logInfo) {
- return GetLogReplyProto.newBuilder().setLog(toLogInfoProto(logInfo)).build();
- }
-
- public static MetaServiceExceptionProto toMetaServiceExceptionProto(Exception exception) {
- final Throwable t = exception.getCause() != null ? exception.getCause() : exception;
- return MetaServiceExceptionProto.newBuilder()
- .setExceptionClassName(t.getClass().getName())
- .setErrorMsg(t.getMessage())
- .setStacktrace(ProtoUtils.writeObject2ByteString(t.getStackTrace()))
- .build();
- }
-
- public static CreateLogReplyProto.Builder toCreateLogExceptionReplyProto(Exception e) {
- return CreateLogReplyProto.newBuilder().setException(toMetaServiceExceptionProto(e));
- }
-
- public static GetLogReplyProto.Builder toGetLogExceptionReplyProto(Exception e) {
- return GetLogReplyProto.newBuilder().setException(toMetaServiceExceptionProto(e));
- }
-
- public static DeleteLogReplyProto.Builder toDeleteLogExceptionReplyProto(Exception e) {
- return DeleteLogReplyProto.newBuilder().setException(toMetaServiceExceptionProto(e));
- }
-
- public static IOException toMetaServiceException(MetaServiceExceptionProto exceptionProto) {
- try {
- final IOException result;
- Class<?> clazz = Class.forName(exceptionProto.getExceptionClassName());
- Exception e = ReflectionUtils.instantiateException(
- clazz.asSubclass(Exception.class), exceptionProto.getErrorMsg());
- if(e instanceof IOException) {
- result = (IOException)e;
- } else {
- result = new IOException(e);
- }
- StackTraceElement[] stacktrace =
- (StackTraceElement[]) ProtoUtils.toObject(exceptionProto.getStacktrace());
- result.setStackTrace(stacktrace);
-
- return result;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- // Internal methods
- static RaftPeer[] toRaftPeerArray(List<RaftPeerProto> protos) {
- final RaftPeer[] peers = new RaftPeer[protos.size()];
- for (int i = 0; i < peers.length; i++) {
- peers[i] = toRaftPeer(protos.get(i));
- }
- return peers;
- }
-
- static Iterable<RaftPeerProto> toRaftPeerProtos(
- final Collection<RaftPeer> peers) {
- return () -> new Iterator<RaftPeerProto>() {
- private final Iterator<RaftPeer> i = peers.iterator();
-
- @Override
- public boolean hasNext() {
- return i.hasNext();
- }
-
- @Override
- public RaftPeerProto next() {
- return toRaftPeerProto(i.next());
- }
- };
- }
-
- public static DeleteLogReplyProto toDeleteLogReplyProto() {
- return DeleteLogReplyProto.newBuilder().build();
- }
-}
diff --git a/ratis-logservice/src/main/proto/LogService.proto b/ratis-logservice/src/main/proto/LogService.proto
deleted file mode 100644
index eb27842..0000000
--- a/ratis-logservice/src/main/proto/LogService.proto
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-syntax = "proto3";
-option java_package = "org.apache.ratis.logservice.proto";
... 5261 lines suppressed ...